diff --git a/README.md b/README.md index 85e9afbc1fd..92adaa616c0 100644 --- a/README.md +++ b/README.md @@ -220,6 +220,7 @@ Firstmate's skills live in two separate places with different audiences: - [docs/voice-relay.md](docs/voice-relay.md) - the optional spoken interface: setup on both machines, measured round-trip cost, what a spoken answer may read, and what this build does not do yet. - [docs/fleet-ledger.md](docs/fleet-ledger.md) - the opt-in activity ledger outside tools can read to follow a home's tasks, and its record contract. - [docs/wedge-alarm.md](docs/wedge-alarm.md) - configure the active alert for an away-mode escalation delivery that gets stuck. +- [docs/troubleshooting.md](docs/troubleshooting.md) - what a cleanup refusal about a shared VM means and how to resolve it. - [docs/tmux-backend.md](docs/tmux-backend.md) - current setup and limits for the tmux reference backend. - [docs/herdr-backend.md](docs/herdr-backend.md) - current setup, CI coverage, safety boundaries, and limits for the Herdr backend. - [docs/zellij-backend.md](docs/zellij-backend.md) - current setup and limits for the experimental Zellij backend. diff --git a/bin/fm-teardown.sh b/bin/fm-teardown.sh index 30b2d1b4507..f7e9db87fab 100755 --- a/bin/fm-teardown.sh +++ b/bin/fm-teardown.sh @@ -95,7 +95,7 @@ # from it. # Worktree-slot ownership (teardown-slot-collision): a treehouse pool slot is # reused across tasks, so a stale, duplicated, or drifted worktree= record can -# name a slot a DIFFERENT live task now holds. Cleanup kills every process under +# name a slot a DIFFERENT live task now holds. Cleanup reaps processes under # that path and hard-resets it before returning it, so releasing a slot that is # not genuinely this task's destroys another worker's live work. Before the first # cleanup step, teardown verifies record exclusivity: no OTHER task record in @@ -276,14 +276,13 @@ # process group, so it survives reparented to init (observed 2026-08-03: # two `go test` binaries, deadlines blown past by ~100x, pinning CPU for # hours with no live task meta to attribute them to once teardown had -# already removed it). reap_task_worktree_processes finds every process +# already removed it). reap_task_worktree_processes finds processes # whose CURRENT WORKING DIRECTORY is this task's own worktree or tasktmp # root via `lsof -a -d cwd` (cheap: bounded by process count, not by -# walking the worktree's file tree) and sends TERM, then KILL after a short -# grace period to any survivor whose process identity still matches. Both -# roots are unique per task and never -# shared, so this can never reach another task's or the primary's -# processes. Idempotent: nothing left to find is a silent no-op. +# walking the worktree's file tree) and sends reapable processes TERM, then +# KILL after a short grace period to any survivor whose process identity +# still matches. Shared-VM refusals and detection limits are documented in +# docs/troubleshooting.md. Idempotent: nothing left to find is a silent no-op. # Fix 3 - sweep abandoned remote job workers. A remote job worker started # from a worktree's own bin/ outlives that worktree's removal without # being reachable by Fix 2, because its working directory is wherever it @@ -2133,14 +2132,85 @@ task_process_identity_matches() { # [ "$current" = "$2" ] } +TASK_PIDS_SHARED_VM_REFUSED= + task_pid_list_contains() { # printf '%s\n' "$1" | grep -Fxq "$2" } +limactl_state_path_from_command() { # + local executable subcommand rest state_flag next_flag word value="" i state_count=0 next_count=0 + local -a words + case "$1" in *$'\n'*|*$'\r'*) return 1 ;; esac + read -r executable rest <<< "$1" + [ "${executable##*/}" = limactl ] || return 1 + while [ -n "$rest" ]; do + read -r subcommand rest <<< "$rest" + case "$subcommand" in + hostagent) state_flag=--pidfile; next_flag=--socket; break ;; + usernet) state_flag=-p; next_flag=-e; break ;; + -*) ;; + *) return 1 ;; + esac + done + [ -n "${state_flag:-}" ] || return 1 + read -r -a words <<< "$1" + for i in "${!words[@]}"; do + word=${words[$i]} + case "$word" in + --pidfile|-p) + state_count=$((state_count + 1)) + if [ "$word" = "$state_flag" ] && [ "${words[$((i + 2))]:-}" = "$next_flag" ]; then + value=${words[$((i + 1))]:-} + fi + ;; + esac + [ "$word" != "$next_flag" ] || next_count=$((next_count + 1)) + done + [ "$state_count" -eq 1 ] && [ "$next_count" -eq 1 ] && [ -n "$value" ] || return 1 + printf '%s\n' "$value" +} + +# Physical form of an absolute path whose tail may not exist yet: the deepest +# existing directory is resolved and the rest is appended. Failure means the +# location cannot be established (relative, a dangling symlink, or `.`/`..` +# after the part that exists). +task_physical_path() { # + local path=$1 tail="" dir + case "$path" in /*) ;; *) return 1 ;; esac + while [ ! -d "$path" ]; do + [ ! -L "$path" ] || return 1 + case "${path##*/}" in ''|.|..) return 1 ;; esac + tail="/${path##*/}$tail" + path=${path%/*} + [ -n "$path" ] || path=/ + done + dir=$(cd "$path" 2>/dev/null && pwd -P) || return 1 + printf '%s%s\n' "${dir%/}" "$tail" +} + +# Print a limactl process's state path when it resolves outside every root. +# Failure, including any doubt about the path or a root, leaves it reapable. +task_pid_shared_vm_state_path() { # ... + local pid=$1 command state resolved dir root + shift + command=$(ps -ww -o command= -p "$pid" 2>/dev/null) || return 1 + state=$(limactl_state_path_from_command "$command") || return 1 + resolved=$(task_physical_path "$state") || return 1 + for dir in "$@"; do + [ -n "$dir" ] || continue + root=$(task_physical_path "$dir") || return 1 + case "$resolved" in + "$root"|"$root"/*) return 1 ;; + esac + done + printf '%s\n' "$state" +} + task_pids_under_roots() { # ... TASK_PIDS= TASK_PIDS_FAILED_DIR= - local dir dir_pids pids="" + local dir dir_pids pid parent state shared="" kept="" pids="" for dir in "$@"; do [ -n "$dir" ] || continue if ! dir_pids=$(pids_with_cwd_under "$dir"); then @@ -2150,7 +2220,47 @@ task_pids_under_roots() { # ... pids="$pids $dir_pids" done - TASK_PIDS=$(printf '%s\n' "$pids" | grep -E '^[0-9]+$' | sort -un || true) + pids=$(printf '%s\n' "$pids" | grep -E '^[0-9]+$' | sort -un || true) + while IFS= read -r pid; do + [ -n "$pid" ] || continue + if state=$(task_pid_shared_vm_state_path "$pid" "$@"); then + shared="$shared +$pid" + if ! task_pid_list_contains "$TASK_PIDS_SHARED_VM_REFUSED" "$pid"; then + echo "REFUSED: leaving VM process $pid running for $ID: its state path $state is outside this task's directories; preserving the worktree/tasktmp and task records. Restart that VM through its service manager, then rerun cleanup (docs/troubleshooting.md)." >&2 + TASK_PIDS_SHARED_VM_REFUSED="$TASK_PIDS_SHARED_VM_REFUSED +$pid" + fi + fi + done </dev/null | tr -d '[:space:]') || break + case "$parent" in ''|*[!0-9]*) break ;; esac + done + [ -n "$parent" ] && task_pid_list_contains "$shared" "$parent" && continue + kept="$kept +$pid" + done < ... + local ID=$1 TASK_PIDS_SHARED_VM_REFUSED='' TASK_PIDS TASK_PIDS_FAILED_DIR + shift + command -v lsof >/dev/null 2>&1 || return 0 + if ! task_pids_under_roots "$@"; then + echo "REFUSED: cannot determine processes under ${TASK_PIDS_FAILED_DIR:-} for $ID (lsof failed); preserving task records and directories." >&2 + return 1 + fi + [ -z "$TASK_PIDS_SHARED_VM_REFUSED" ] } reap_task_backend_process_group() { #