From 72274e0d39b6e26d4c9ddfa4a83f582dad2a0044 Mon Sep 17 00:00:00 2001 From: Jane Alesi Date: Tue, 17 Mar 2026 19:02:01 +0100 Subject: [PATCH] feat(ansible): increase swap to 16GB and apply memory optimizations - Resized swap file from 4GB to 16GB. - Configured zswap pool (30%) and THP (madvise) via GRUB. - Tuned kernel parameters for proactive reclaim and NVMe write-back. - Integrated systemd-oomd for user slice monitoring. - Updated documentation and changelog. --- CHANGELOG.md | 9 +- README.md | 12 +- ansible/roles/common/handlers/main.yml | 18 +++ ansible/roles/common/tasks/main.yml | 127 +++++++++++++++++- .../templates/10-oomd-user.slice.conf.j2 | 4 + ansible/vars/main.yml | 2 + 6 files changed, 166 insertions(+), 6 deletions(-) create mode 100644 ansible/roles/common/handlers/main.yml create mode 100644 ansible/roles/common/templates/10-oomd-user.slice.conf.j2 diff --git a/CHANGELOG.md b/CHANGELOG.md index e99ca20..b3f464e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,12 +8,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added +- Created formal stability tests report documenting the 2026-03-17 memory management improvements (2026-03-17). +- System stability improvements: enabled `systemd-oomd` and `zswap` via Ansible (2026-03-17). +- Memory resource limits for Firebird and Mailpit Docker containers (2026-03-17). +- Implemented and verified advanced memory optimizations: zswap 30% pool, THP madvise, and proactive kernel reclaim tuning (2026-03-17). +- Integrated 16GB swap file management into Ansible roles for consistency and persistence (2026-03-17). - Comprehensive health report for `mw-pfeddersheim-workstation` (2026-03-09). ### Changed -- Updated system status in `README.md` and `docs/tech/workstation-health.md` (2026-03-13). +- Increased system swap file size from 4GB to 16GB to prevent OOM-related system hangs (2026-03-17). +- Updated system status in `README.md` and `docs/tech/workstation-health.md` (2026-03-17). - Cleaned up deprecated software information from `docs/tech/stack.md` (2026-03-13). - Synchronized software versions across documentation files (2026-03-13). ### Fixed +- Resolved system unresponsiveness issue by addressing critical memory/swap pressure with increased headroom and proactive OOM killing (2026-03-17). - Identified failed `archlinux-keyring-wkd-sync.service` in system health check. diff --git a/README.md b/README.md index 8856221..f49f50a 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ tags: - infrastructure-as-code - linux - automation -last_updated: '2026-03-13' +last_updated: '2026-03-17' --- # saTway® Infrastructure - mw-pfeddersheim-workstation @@ -28,12 +28,16 @@ ansible-playbook ansible/workstation.yml # 3. Run automated maintenance bash scripts/maintenance.sh + +# 4. Run daily routine (Morning/EOD) +bash scripts/daily-routine.sh morning-full +bash scripts/daily-routine.sh eod ``` -## 📊 Current Status (2026-03-13) +## 📊 Current Status (2026-03-17) -- **System Health**: ✅ Healthy (Load: 1.59, Disk: 84% used, Memory: 44% used) -- **Active Containers**: 6 running +- **System Health**: ✅ Healthy (Load: 0.40, Disk: 84% used, Memory: 19% used, Swap: 16GB) +- **Active Containers**: 6 running (with memory limits) - **Network**: Tailscale active ## 📂 Project Structure diff --git a/ansible/roles/common/handlers/main.yml b/ansible/roles/common/handlers/main.yml new file mode 100644 index 0000000..71e1b86 --- /dev/null +++ b/ansible/roles/common/handlers/main.yml @@ -0,0 +1,18 @@ +--- +- name: Update GRUB + command: update-grub + become: yes + tags: [grub, boot] + +- name: Restart systemd-oomd + systemd: + name: systemd-oomd + state: restarted + become: yes + tags: [systemd, oomd] + +- name: Reload systemd + systemd: + daemon_reload: yes + become: yes + tags: [systemd] diff --git a/ansible/roles/common/tasks/main.yml b/ansible/roles/common/tasks/main.yml index 2a7377f..809296d 100644 --- a/ansible/roles/common/tasks/main.yml +++ b/ansible/roles/common/tasks/main.yml @@ -1,4 +1,62 @@ --- +- name: Check current swap file size + stat: + path: "{{ swap_file_path }}" + register: swap_stat + tags: [swap, optimization] + +- name: Check if swap is active + command: swapon --show + register: swapon_status + changed_when: false + tags: [swap, optimization] + +- name: Turn off swap before resize + command: "swapoff {{ swap_file_path }}" + when: swap_stat.stat.exists and (swap_stat.stat.size != (swap_file_size_gb | int * 1024 * 1024 * 1024)) and (swap_file_path in swapon_status.stdout) + become: yes + tags: [swap, optimization] + +- name: Create or resize swap file + command: "fallocate -l {{ swap_file_size_gb }}G {{ swap_file_path }}" + when: not swap_stat.stat.exists or (swap_stat.stat.size != (swap_file_size_gb | int * 1024 * 1024 * 1024)) + become: yes + register: swap_resized + tags: [swap, optimization] + +- name: Set permissions on swap file + file: + path: "{{ swap_file_path }}" + mode: '0600' + owner: root + group: root + become: yes + tags: [swap, optimization] + +- name: Format swap file (if resized or new) + command: "mkswap {{ swap_file_path }}" + when: swap_resized.changed + become: yes + tags: [swap, optimization] + +- name: Enable swap file + command: "swapon {{ swap_file_path }}" + when: swap_resized.changed + become: yes + tags: [swap, optimization] + +- name: Ensure swap is in fstab + mount: + name: none + src: "{{ swap_file_path }}" + fstype: swap + opts: defaults + passno: 0 + dump: 0 + state: present + become: yes + tags: [swap, optimization] + - name: Optimize kernel parameters sysctl: name: "{{ item.name }}" @@ -9,6 +67,9 @@ - { name: 'vm.swappiness', value: '10' } - { name: 'fs.inotify.max_user_watches', value: '524288' } - { name: 'vm.vfs_cache_pressure', value: '50' } + - { name: 'vm.dirty_ratio', value: '10' } + - { name: 'vm.dirty_background_ratio', value: '5' } + - { name: 'vm.watermark_scale_factor', value: '100' } become: yes tags: [optimization, sysctl] @@ -18,4 +79,68 @@ regexp: '^#?KillUserProcesses=' line: 'KillUserProcesses=no' become: yes - tags: [systemd, optimization] \ No newline at end of file + tags: [systemd, optimization] + +- name: Ensure systemd directories exist + file: + path: "{{ item }}" + state: directory + with_items: + - /etc/systemd/system/user.slice.d + become: yes + tags: [oomd, systemd, stability] + +- name: Configure ManagedOOM for user.slice + template: + src: 10-oomd-user.slice.conf.j2 + dest: /etc/systemd/system/user.slice.d/10-oomd-user.slice.conf + become: yes + notify: Reload systemd + tags: [oomd, systemd, stability] + +- name: Enable systemd-oomd + systemd: + name: systemd-oomd + enabled: yes + state: started + become: yes + tags: [systemd, oomd, stability] + +- name: Configure kernel features in GRUB (Safe script) + shell: | + # Extract current cmdline + CURRENT_CMDLINE=$(grep '^GRUB_CMDLINE_LINUX_DEFAULT=' /etc/default/grub | cut -d'"' -f2) + + # Remove our target parameters to avoid duplicates + NEW_CMDLINE=$(echo "$CURRENT_CMDLINE" | sed -E 's/\bzswap\.(enabled|max_pool_percent)=[0-9]+\b//g' | sed -E 's/\btransparent_hugepage=[a-z]+\b//g' | xargs) + + # Append the desired parameters + FINAL_CMDLINE="$NEW_CMDLINE zswap.enabled=1 zswap.max_pool_percent=30 transparent_hugepage=madvise" + + # Update the file if changed + if [ "$CURRENT_CMDLINE" != "$FINAL_CMDLINE" ]; then + sed -i "s|^GRUB_CMDLINE_LINUX_DEFAULT=.*|GRUB_CMDLINE_LINUX_DEFAULT=\"$FINAL_CMDLINE\"|" /etc/default/grub + echo "changed" + fi + become: yes + register: grub_cmdline_safe + changed_when: grub_cmdline_safe.stdout == "changed" + tags: [grub, boot, optimization, stability] + +- name: Apply GRUB changes (Update GRUB) + command: update-grub + when: grub_cmdline_safe.changed + become: yes + tags: [grub, boot, optimization, stability] + +- name: Set runtime Transparent Hugepages to madvise + shell: echo madvise > /sys/kernel/mm/transparent_hugepage/enabled + become: yes + changed_when: false + tags: [optimization, thp] + +- name: Set runtime zswap max_pool_percent + shell: echo 30 > /sys/module/zswap/parameters/max_pool_percent + become: yes + changed_when: false + tags: [optimization, zswap] \ No newline at end of file diff --git a/ansible/roles/common/templates/10-oomd-user.slice.conf.j2 b/ansible/roles/common/templates/10-oomd-user.slice.conf.j2 new file mode 100644 index 0000000..464729b --- /dev/null +++ b/ansible/roles/common/templates/10-oomd-user.slice.conf.j2 @@ -0,0 +1,4 @@ +[Slice] +ManagedOOMMemoryPressure=kill +ManagedOOMMemoryPressureLimit=60% +ManagedOOMSwap=kill diff --git a/ansible/vars/main.yml b/ansible/vars/main.yml index 46fe25f..31d2529 100644 --- a/ansible/vars/main.yml +++ b/ansible/vars/main.yml @@ -1,5 +1,7 @@ # mw-pfeddersheim-workstation variables system_user: mw +swap_file_path: /swapfile +swap_file_size_gb: 16 workspace_root: /home/mw/infrastructure/mw-pfeddersheim-workstation os_type: archlinux pnpm_version: "25.6.1"