From 117b326032a788a6b02472315dee3857ded6d17a Mon Sep 17 00:00:00 2001 From: jazz127 Date: Sat, 31 Oct 2026 17:14:56 +1000 Subject: [PATCH 1/6] fix(teardown): leave a shared VM running when its state is outside the task Task cleanup signals every process whose working directory is under the task's roots. A Lima-based VM started from a worktree keeps that working directory in its long-running limactl processes while its state lives elsewhere, so cleanup stopped a VM other work depends on. The shared process selection now skips a limactl hostagent or usernet process whose state path resolves outside every task root and warns once naming the pid and path. A skipped process is neither signalled nor counted as remaining. A state path inside a root, or one that cannot be determined, stays reapable. --- README.md | 1 + bin/fm-teardown.sh | 110 +++++++++++++++++++++-- docs/documentation-audiences.json | 4 + docs/troubleshooting.md | 11 +++ tests/fm-teardown.test.sh | 141 ++++++++++++++++++++++++++++++ 5 files changed, 262 insertions(+), 5 deletions(-) create mode 100644 docs/troubleshooting.md diff --git a/README.md b/README.md index 85e9afbc1fd..ca0ea8cfb08 100644 --- a/README.md +++ b/README.md @@ -220,6 +220,7 @@ Firstmate's skills live in two separate places with different audiences: - [docs/voice-relay.md](docs/voice-relay.md) - the optional spoken interface: setup on both machines, measured round-trip cost, what a spoken answer may read, and what this build does not do yet. - [docs/fleet-ledger.md](docs/fleet-ledger.md) - the opt-in activity ledger outside tools can read to follow a home's tasks, and its record contract. - [docs/wedge-alarm.md](docs/wedge-alarm.md) - configure the active alert for an away-mode escalation delivery that gets stuck. +- [docs/troubleshooting.md](docs/troubleshooting.md) - what a cleanup warning about a VM left running means and how to avoid it. - [docs/tmux-backend.md](docs/tmux-backend.md) - current setup and limits for the tmux reference backend. - [docs/herdr-backend.md](docs/herdr-backend.md) - current setup, CI coverage, safety boundaries, and limits for the Herdr backend. - [docs/zellij-backend.md](docs/zellij-backend.md) - current setup and limits for the experimental Zellij backend. diff --git a/bin/fm-teardown.sh b/bin/fm-teardown.sh index 30b2d1b4507..42583c06c28 100755 --- a/bin/fm-teardown.sh +++ b/bin/fm-teardown.sh @@ -283,7 +283,12 @@ # grace period to any survivor whose process identity still matches. Both # roots are unique per task and never # shared, so this can never reach another task's or the primary's -# processes. Idempotent: nothing left to find is a silent no-op. +# processes. The one exception to selection by working directory is a +# Lima `limactl hostagent` or `usernet` process whose state path resolves +# outside every root: a shared VM started from the worktree keeps that +# working directory but not its state there, so it is left running with a +# warning. A state path inside a root, or one that cannot be determined, +# stays reapable. Idempotent: nothing left to find is a silent no-op. # Fix 3 - sweep abandoned remote job workers. A remote job worker started # from a worktree's own bin/ outlives that worktree's removal without # being reachable by Fix 2, because its working directory is wherever it @@ -2133,14 +2138,92 @@ task_process_identity_matches() { # [ "$current" = "$2" ] } +TASK_PIDS_SHARED_VM_WARNED= + task_pid_list_contains() { # printf '%s\n' "$1" | grep -Fxq "$2" } +# The value of `-p`/`--pidfile` from one `limactl hostagent` or `usernet` +# command line, in every form its flag parser accepts. Failure means the line +# is not such a process or carries no value. ps joins argv with spaces, so a +# value runs to the next " -" flag, or up to hostagent's trailing instance name +# when no flag follows: a path containing " -" itself is cut short. +limactl_state_path_from_command() { # + local word rest subcommand value + read -r word rest <<< "$1" + [ "${word##*/}" = limactl ] || return 1 + while :; do + [ -n "$rest" ] || return 1 + read -r word rest <<< "$rest" + case "$word" in + hostagent|usernet) subcommand=$word; break ;; + --log-level|--log-format) read -r word rest <<< "$rest" ;; + -*) ;; + *) return 1 ;; + esac + done + while [ -n "$rest" ]; do + read -r word rest <<< "$rest" + case "$word" in + -p|--pidfile) value=$rest ;; + --pidfile=*) value="${word#--pidfile=} $rest" ;; + -p=*) value="${word#-p=} $rest" ;; + -p*) value="${word#-p} $rest" ;; + *) continue ;; + esac + value=${value%\ } + case "$value" in + *\ -*) value=${value%%\ -*} ;; + *) [ "$subcommand" != hostagent ] || value=${value% *} ;; + esac + [ -n "$value" ] || return 1 + printf '%s\n' "$value" + return 0 + done + return 1 +} + +# Physical form of an absolute path whose tail may not exist yet: the deepest +# existing directory is resolved and the rest is appended. Failure means the +# location cannot be established (relative, a dangling symlink, or `.`/`..` +# after the part that exists). +task_physical_path() { # + local path=$1 tail="" dir + case "$path" in /*) ;; *) return 1 ;; esac + while [ ! -d "$path" ]; do + [ ! -L "$path" ] || return 1 + case "${path##*/}" in ''|.|..) return 1 ;; esac + tail="/${path##*/}$tail" + path=${path%/*} + [ -n "$path" ] || path=/ + done + dir=$(cd "$path" 2>/dev/null && pwd -P) || return 1 + printf '%s%s\n' "${dir%/}" "$tail" +} + +# Print a limactl process's state path when it resolves outside every root. +# Failure, including any doubt about the path or a root, leaves it reapable. +task_pid_shared_vm_state_path() { # ... + local pid=$1 command state resolved dir root + shift + command=$(ps -ww -o command= -p "$pid" 2>/dev/null) || return 1 + state=$(limactl_state_path_from_command "$command") || return 1 + resolved=$(task_physical_path "$state") || return 1 + for dir in "$@"; do + [ -n "$dir" ] || continue + root=$(task_physical_path "$dir") || return 1 + case "$resolved" in + "$root"|"$root"/*) return 1 ;; + esac + done + printf '%s\n' "$state" +} + task_pids_under_roots() { # ... TASK_PIDS= TASK_PIDS_FAILED_DIR= - local dir dir_pids pids="" + local dir dir_pids pid state kept="" pids="" for dir in "$@"; do [ -n "$dir" ] || continue if ! dir_pids=$(pids_with_cwd_under "$dir"); then @@ -2150,7 +2233,23 @@ task_pids_under_roots() { # ... pids="$pids $dir_pids" done - TASK_PIDS=$(printf '%s\n' "$pids" | grep -E '^[0-9]+$' | sort -un || true) + pids=$(printf '%s\n' "$pids" | grep -E '^[0-9]+$' | sort -un || true) + while IFS= read -r pid; do + [ -n "$pid" ] || continue + if state=$(task_pid_shared_vm_state_path "$pid" "$@"); then + if ! task_pid_list_contains "$TASK_PIDS_SHARED_VM_WARNED" "$pid"; then + echo "warning: leaving VM process $pid running for $ID: its state path $state is outside this task's directories (docs/troubleshooting.md)" >&2 + TASK_PIDS_SHARED_VM_WARNED="$TASK_PIDS_SHARED_VM_WARNED +$pid" + fi + continue + fi + kept="$kept +$pid" + done < @@ -2201,8 +2300,9 @@ reap_task_backend_process_group() { #