feat(ansible): increase swap to 16GB and apply memory optimizations

- Resized swap file from 4GB to 16GB. - Configured zswap pool (30%) and THP (madvise) via GRUB. - Tuned kernel parameters for proactive reclaim and NVMe write-back. - Integrated systemd-oomd for user slice monitoring. - Updated documentation and changelog.
2026-03-17 19:02:01 +01:00
parent d2067dfb91
commit 72274e0d39
6 changed files with 166 additions and 6 deletions
@@ -8,12 +8,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]

 ### Added
+- Created formal stability tests report documenting the 2026-03-17 memory management improvements (2026-03-17).
+- System stability improvements: enabled `systemd-oomd` and `zswap` via Ansible (2026-03-17).
+- Memory resource limits for Firebird and Mailpit Docker containers (2026-03-17).
+- Implemented and verified advanced memory optimizations: zswap 30% pool, THP madvise, and proactive kernel reclaim tuning (2026-03-17).
+- Integrated 16GB swap file management into Ansible roles for consistency and persistence (2026-03-17).
 - Comprehensive health report for `mw-pfeddersheim-workstation` (2026-03-09).

 ### Changed
- Updated system status in `README.md` and `docs/tech/workstation-health.md` (2026-03-13).
+- Increased system swap file size from 4GB to 16GB to prevent OOM-related system hangs (2026-03-17).
+- Updated system status in `README.md` and `docs/tech/workstation-health.md` (2026-03-17).
 - Cleaned up deprecated software information from `docs/tech/stack.md` (2026-03-13).
 - Synchronized software versions across documentation files (2026-03-13).

 ### Fixed
+- Resolved system unresponsiveness issue by addressing critical memory/swap pressure with increased headroom and proactive OOM killing (2026-03-17).
 - Identified failed `archlinux-keyring-wkd-sync.service` in system health check.
@@ -8,7 +8,7 @@ tags:
  - infrastructure-as-code
  - linux
  - automation
-last_updated: '2026-03-13'
+last_updated: '2026-03-17'
 ---

 # saTway® Infrastructure - mw-pfeddersheim-workstation
@@ -28,12 +28,16 @@ ansible-playbook ansible/workstation.yml

 # 3. Run automated maintenance
 bash scripts/maintenance.sh
+
+# 4. Run daily routine (Morning/EOD)
+bash scripts/daily-routine.sh morning-full
+bash scripts/daily-routine.sh eod
 ```

-## 📊 Current Status (2026-03-13)
+## 📊 Current Status (2026-03-17)

- **System Health**: ✅ Healthy (Load: 1.59, Disk: 84% used, Memory: 44% used)
- **Active Containers**: 6 running
+- **System Health**: ✅ Healthy (Load: 0.40, Disk: 84% used, Memory: 19% used, Swap: 16GB)
+- **Active Containers**: 6 running (with memory limits)
 - **Network**: Tailscale active

 ## 📂 Project Structure
@@ -0,0 +1,18 @@
+---
+- name: Update GRUB
+  command: update-grub
+  become: yes
+  tags: [grub, boot]
+
+- name: Restart systemd-oomd
+  systemd:
+    name: systemd-oomd
+    state: restarted
+  become: yes
+  tags: [systemd, oomd]
+
+- name: Reload systemd
+  systemd:
+    daemon_reload: yes
+  become: yes
+  tags: [systemd]
@@ -1,4 +1,62 @@
 ---
+- name: Check current swap file size
+  stat:
+    path: "{{ swap_file_path }}"
+  register: swap_stat
+  tags: [swap, optimization]
+
+- name: Check if swap is active
+  command: swapon --show
+  register: swapon_status
+  changed_when: false
+  tags: [swap, optimization]
+
+- name: Turn off swap before resize
+  command: "swapoff {{ swap_file_path }}"
+  when: swap_stat.stat.exists and (swap_stat.stat.size != (swap_file_size_gb | int * 1024 * 1024 * 1024)) and (swap_file_path in swapon_status.stdout)
+  become: yes
+  tags: [swap, optimization]
+
+- name: Create or resize swap file
+  command: "fallocate -l {{ swap_file_size_gb }}G {{ swap_file_path }}"
+  when: not swap_stat.stat.exists or (swap_stat.stat.size != (swap_file_size_gb | int * 1024 * 1024 * 1024))
+  become: yes
+  register: swap_resized
+  tags: [swap, optimization]
+
+- name: Set permissions on swap file
+  file:
+    path: "{{ swap_file_path }}"
+    mode: '0600'
+    owner: root
+    group: root
+  become: yes
+  tags: [swap, optimization]
+
+- name: Format swap file (if resized or new)
+  command: "mkswap {{ swap_file_path }}"
+  when: swap_resized.changed
+  become: yes
+  tags: [swap, optimization]
+
+- name: Enable swap file
+  command: "swapon {{ swap_file_path }}"
+  when: swap_resized.changed
+  become: yes
+  tags: [swap, optimization]
+
+- name: Ensure swap is in fstab
+  mount:
+    name: none
+    src: "{{ swap_file_path }}"
+    fstype: swap
+    opts: defaults
+    passno: 0
+    dump: 0
+    state: present
+  become: yes
+  tags: [swap, optimization]
+
 - name: Optimize kernel parameters
  sysctl:
    name: "{{ item.name }}"
@@ -9,6 +67,9 @@
    - { name: 'vm.swappiness', value: '10' }
    - { name: 'fs.inotify.max_user_watches', value: '524288' }
    - { name: 'vm.vfs_cache_pressure', value: '50' }
+    - { name: 'vm.dirty_ratio', value: '10' }
+    - { name: 'vm.dirty_background_ratio', value: '5' }
+    - { name: 'vm.watermark_scale_factor', value: '100' }
  become: yes
  tags: [optimization, sysctl]

@@ -18,4 +79,68 @@
    regexp: '^#?KillUserProcesses='
    line: 'KillUserProcesses=no'
  become: yes
-  tags: [systemd, optimization]
+  tags: [systemd, optimization]
+
+- name: Ensure systemd directories exist
+  file:
+    path: "{{ item }}"
+    state: directory
+  with_items:
+    - /etc/systemd/system/user.slice.d
+  become: yes
+  tags: [oomd, systemd, stability]
+
+- name: Configure ManagedOOM for user.slice
+  template:
+    src: 10-oomd-user.slice.conf.j2
+    dest: /etc/systemd/system/user.slice.d/10-oomd-user.slice.conf
+  become: yes
+  notify: Reload systemd
+  tags: [oomd, systemd, stability]
+
+- name: Enable systemd-oomd
+  systemd:
+    name: systemd-oomd
+    enabled: yes
+    state: started
+  become: yes
+  tags: [systemd, oomd, stability]
+
+- name: Configure kernel features in GRUB (Safe script)
+  shell: |
+    # Extract current cmdline
+    CURRENT_CMDLINE=$(grep '^GRUB_CMDLINE_LINUX_DEFAULT=' /etc/default/grub | cut -d'"' -f2)
+    
+    # Remove our target parameters to avoid duplicates
+    NEW_CMDLINE=$(echo "$CURRENT_CMDLINE" | sed -E 's/\bzswap\.(enabled|max_pool_percent)=[0-9]+\b//g' | sed -E 's/\btransparent_hugepage=[a-z]+\b//g' | xargs)
+    
+    # Append the desired parameters
+    FINAL_CMDLINE="$NEW_CMDLINE zswap.enabled=1 zswap.max_pool_percent=30 transparent_hugepage=madvise"
+    
+    # Update the file if changed
+    if [ "$CURRENT_CMDLINE" != "$FINAL_CMDLINE" ]; then
+      sed -i "s|^GRUB_CMDLINE_LINUX_DEFAULT=.*|GRUB_CMDLINE_LINUX_DEFAULT=\"$FINAL_CMDLINE\"|" /etc/default/grub
+      echo "changed"
+    fi
+  become: yes
+  register: grub_cmdline_safe
+  changed_when: grub_cmdline_safe.stdout == "changed"
+  tags: [grub, boot, optimization, stability]
+
+- name: Apply GRUB changes (Update GRUB)
+  command: update-grub
+  when: grub_cmdline_safe.changed
+  become: yes
+  tags: [grub, boot, optimization, stability]
+
+- name: Set runtime Transparent Hugepages to madvise
+  shell: echo madvise > /sys/kernel/mm/transparent_hugepage/enabled
+  become: yes
+  changed_when: false
+  tags: [optimization, thp]
+
+- name: Set runtime zswap max_pool_percent
+  shell: echo 30 > /sys/module/zswap/parameters/max_pool_percent
+  become: yes
+  changed_when: false
+  tags: [optimization, zswap]
@@ -0,0 +1,4 @@
+[Slice]
+ManagedOOMMemoryPressure=kill
+ManagedOOMMemoryPressureLimit=60%
+ManagedOOMSwap=kill
@@ -1,5 +1,7 @@
 # mw-pfeddersheim-workstation variables
 system_user: mw
+swap_file_path: /swapfile
+swap_file_size_gb: 16
 workspace_root: /home/mw/infrastructure/mw-pfeddersheim-workstation
 os_type: archlinux
 pnpm_version: "25.6.1"