From 172e8075e5dcdf53676db8bf9a9a27938889d637 Mon Sep 17 00:00:00 2001 From: Assis Ngolo Date: Sat, 3 Oct 2026 18:57:39 +0000 Subject: [PATCH] ops: record host recovery and scheduled maintenance automation --- .ai/todo.md | 22 +++++++++---- docs/operations/host-maintenance.md | 32 +++++++++++++++++++ docs/operations/host-maintenance.sh | 20 ++++++++++++ docs/operations/host-postboot.sh | 26 +++++++++++++++ docs/operations/jean-host-maintenance.service | 8 +++++ docs/operations/jean-host-maintenance.timer | 8 +++++ docs/operations/jean-host-postboot.service | 11 +++++++ 7 files changed, 121 insertions(+), 6 deletions(-) create mode 100644 docs/operations/host-maintenance.md create mode 100755 docs/operations/host-maintenance.sh create mode 100755 docs/operations/host-postboot.sh create mode 100644 docs/operations/jean-host-maintenance.service create mode 100644 docs/operations/jean-host-maintenance.timer create mode 100644 docs/operations/jean-host-postboot.service diff --git a/.ai/todo.md b/.ai/todo.md index 960883acf..3c7eb1506 100644 --- a/.ai/todo.md +++ b/.ai/todo.md @@ -1,7 +1,17 @@ -# PR #16 main conflict resolution +# Host maintenance +- [x] Reclaim unused Docker volumes, stopped containers, old images and build cache +- [x] Install updates with service restart suppression; restore new container starts via containerd restart +- [x] Enable Compose init and arm 2026-10-04 02:00 UTC host maintenance timer +- [x] Install postboot verification, validate units/scripts, smoke-test idempotency guards +- [x] Finish required jean:dev build +- [x] Create audit PR #19 +- [ ] Verify actual reboot/kernel/zombie cleanup after scheduled window -- [x] Merge main 2e23c71d, retain managed Claude installer and upstream settings behavior. -- [x] Resolve icon migration and removed smoke-test import. -- [x] Full check:all: 2,876 frontend tests, 1,339 Rust tests passed (one ignored); jean:dev rebuilt. -- [x] Agent-browser smoke check: Settings → Claude shows Install latest with PATH selected. -- [x] Pushed merge 342c6083 to open PR #16; GitHub reports MERGEABLE. +## Review +- Root usage: 99% initially, 60% after the required rebuild, with 167 GB available. +- Volume prune reclaimed 66.16 GB; cache prune reclaimed 11.56 GB. +- Apt update/upgrade/autoremove completed, dpkg audit clean, no pending upgrades. +- Timer verified active; Compose valid and persistent Jean services have init=true configured. +- Actual kernel switch and zombie removal intentionally pending scheduled reboot. +- New-container smoke test passed after containerd-only restart; Jean service uptime unchanged. +- Two temporary kinaxixi-eng717 application containers vanished during concurrent activity; cause unverified. diff --git a/docs/operations/host-maintenance.md b/docs/operations/host-maintenance.md new file mode 100644 index 000000000..fb9102d0f --- /dev/null +++ b/docs/operations/host-maintenance.md @@ -0,0 +1,32 @@ +# Host maintenance — 2026-10-04 + +Host timer: `jean-host-maintenance.timer`, scheduled for 02:00 UTC (04:00 CEST). +Cleanup removes unreferenced volumes and stopped containers, prunes container-unused +images older than seven days, and prunes build cache unused for 24 hours. Recent +images are retained because other development sessions are building concurrently. + +Persistent Jean Compose services have `init: true`. They are recreated at the +maintenance window using the image IDs saved before maintenance, not mutable tags. + +Installed scripts are root-owned under `/usr/local/sbin/`. These copies are an +audit record, not a general-purpose installer. State and package logs live in +`/var/lib/jean-host-maintenance/` on the host. The Compose backup contains secrets; +keep that directory restricted and do not commit or print its contents. + +## Verify + +```sh +systemctl list-timers jean-host-maintenance.timer +journalctl -u jean-host-maintenance.service +journalctl -u jean-host-postboot.service +cat /var/lib/jean-host-maintenance/postboot-report.txt +docker inspect dev-personal-1 dev-squire-1 --format '{{.Name}} Init={{.HostConfig.Init}}' +df -h / +``` + +The reboot marker prevents repeated maintenance reboots. Postboot verification +runs after Docker starts and records results; inspect the report for failures. +Containers without restart policies may remain stopped after the reboot. + +Timer semantics: https://github.com/systemd/systemd/blob/main/man/systemd.timer.xml +Docker volume cleanup: https://docs.docker.com/engine/storage/volumes/ diff --git a/docs/operations/host-maintenance.sh b/docs/operations/host-maintenance.sh new file mode 100755 index 000000000..f8bb9984e --- /dev/null +++ b/docs/operations/host-maintenance.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail +exec 9>/var/lib/jean-host-maintenance/lock +flock -n 9 || exit 0 +[ ! -f /var/lib/jean-host-maintenance/reboot-requested ] || exit 0 +export DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=l +apt-get update --error-on=any +apt-get -y --with-new-pkgs upgrade +cd /home/ubuntu/dev +docker compose config --quiet +# Preserve the exact deployed images rather than switching to a newer mutable tag. +python3 - <<'PY' +import json +base='/var/lib/jean-host-maintenance/' +services={n:{'image':open(base+n+'-image.txt').read().strip()} for n in ['personal','squire']} +open(base+'images.json','w').write(json.dumps({'services':services})) +PY +docker compose -f docker-compose.yml -f /var/lib/jean-host-maintenance/images.json up -d --no-deps --force-recreate personal squire +date -u > /var/lib/jean-host-maintenance/reboot-requested +systemctl reboot diff --git a/docs/operations/host-postboot.sh b/docs/operations/host-postboot.sh new file mode 100755 index 000000000..5c6a45038 --- /dev/null +++ b/docs/operations/host-postboot.sh @@ -0,0 +1,26 @@ +#!/bin/bash +set -euo pipefail +base=/var/lib/jean-host-maintenance +[ -f "$base/reboot-requested" ] || exit 0 +{ +date -u +uname -r +df -h / +systemctl is-active docker +[ ! -f /var/run/reboot-required ] || echo REBOOT_STILL_REQUIRED +docker ps --format '{{.Names}} {{.Status}}' +docker inspect dev-personal-1 dev-squire-1 --format '{{.Name}} Init={{.HostConfig.Init}} Status={{.State.Status}}' +ps -eo stat= | awk '$1 ~ /^Z/{n++} END{print "Zombies:", n+0}' +dpkg --audit +python3 - <<'PYTHON' +import subprocess +base='/var/lib/jean-host-maintenance/' +expected=set(x.strip().lstrip('/') for x in open(base+'expected-persistent.txt') if x.strip()) +running=set(subprocess.check_output(['docker','ps','--format','{{.Names}}'],text=True).splitlines()) +missing=sorted(expected-running) +print('Missing persistent containers:',missing) +if missing: raise SystemExit(1) +PYTHON +} > "$base/postboot-report.txt" 2>&1 +cat "$base/postboot-report.txt" +date -u > "$base/postboot-checked" diff --git a/docs/operations/jean-host-maintenance.service b/docs/operations/jean-host-maintenance.service new file mode 100644 index 000000000..ff6355c69 --- /dev/null +++ b/docs/operations/jean-host-maintenance.service @@ -0,0 +1,8 @@ +[Unit] +Description=Jean host scheduled maintenance +After=network-online.target docker.service +Wants=network-online.target +[Service] +Type=oneshot +ExecStart=/usr/local/sbin/jean-host-maintenance +TimeoutStartSec=2h diff --git a/docs/operations/jean-host-maintenance.timer b/docs/operations/jean-host-maintenance.timer new file mode 100644 index 000000000..ed3d20b67 --- /dev/null +++ b/docs/operations/jean-host-maintenance.timer @@ -0,0 +1,8 @@ +[Unit] +Description=Jean host maintenance at 02:00 UTC +[Timer] +OnCalendar=2026-10-04 02:00:00 UTC +AccuracySec=1s +Persistent=true +[Install] +WantedBy=timers.target diff --git a/docs/operations/jean-host-postboot.service b/docs/operations/jean-host-postboot.service new file mode 100644 index 000000000..8d07cf0fc --- /dev/null +++ b/docs/operations/jean-host-postboot.service @@ -0,0 +1,11 @@ +[Unit] +Description=Jean host post-reboot verification +After=docker.service network-online.target +Wants=docker.service network-online.target +ConditionPathExists=/var/lib/jean-host-maintenance/reboot-requested +[Service] +Type=oneshot +ExecStartPre=/bin/sleep 60 +ExecStart=/usr/local/sbin/jean-host-postboot +[Install] +WantedBy=multi-user.target