From 407c6334df380e4958fded476bf3e3a26b50a3f4 Mon Sep 17 00:00:00 2001
From: Ben Davis <45952064+bmdavis419@users.noreply.github.com>
Date: Wed, 22 Jul 2026 13:25:43 -0700
Subject: [PATCH 1/7] feat: supervise custom box services

---
 Makefile                         |   2 +-
 README.md                        |  50 ++++
 docker/entrypoint.sh             |   7 +-
 docs/tx9-cli-design.md           |  16 +-
 guest/hb                         |  59 ++++-
 guest/hb-workload                |  32 ++-
 guest/tx9-logs                   |  28 ++-
 guest/tx9-services               | 392 +++++++++++++++++++++++++++++++
 internal/assets/assets.go        |   2 +-
 internal/assets/assets_test.go   |   5 +-
 internal/cli/cmd_logs.go         |   4 +-
 internal/cli/dispatch.go         |   2 +-
 internal/cli/dispatch_test.go    |   8 +
 provision/provision.sh           |   3 +-
 tests/regressions-hb-workload.sh | 107 +++++++++
 tests/regressions-services.sh    | 304 ++++++++++++++++++++++++
 tests/static.sh                  |  11 +-
 17 files changed, 1008 insertions(+), 24 deletions(-)
 create mode 100755 guest/tx9-services
 create mode 100755 tests/regressions-services.sh

diff --git a/Makefile b/Makefile
index d4cd933..8e82806 100644
--- a/Makefile
+++ b/Makefile
@@ -2,7 +2,7 @@
 
 # tests/regressions-*.sh is globbed so new split-out regression files don't
 # require touching this Makefile.
-SHELL_FILES := guest/hb guest/hb-workload guest/lib-mcp.sh guest/profile.sh guest/agent-bash-profile.sh provision/provision.sh docker/entrypoint.sh docker/executor-entrypoint.sh tests/lib.sh tests/static.sh tests/hermes-state.sh $(wildcard tests/regressions-*.sh)
+SHELL_FILES := guest/hb guest/hb-workload guest/tx9-services guest/lib-mcp.sh guest/profile.sh guest/agent-bash-profile.sh provision/provision.sh docker/entrypoint.sh docker/executor-entrypoint.sh tests/lib.sh tests/static.sh tests/hermes-state.sh $(wildcard tests/regressions-*.sh)
 
 syntax:
 	bash -n $(SHELL_FILES)
diff --git a/README.md b/README.md
index c570e20..8c5e02c 100644
--- a/README.md
+++ b/README.md
@@ -72,6 +72,8 @@ Inside the agent container:
 hb status / hb doctor / hb versions
 hb down            # durable pause; the reconcile loop won't restart services
 hb up
+hb services        # status for portable custom service drop-ins
+hb services-reload # reconcile custom services immediately
 hb gateway-enable --confirm-single-writer I_CONFIRM_NO_OTHER_GATEWAY_USES_THIS_IDENTITY
 hb gateway-disable
 hb verify-state
@@ -87,6 +89,54 @@ tx9 gateway enable <box> --confirm-single-writer
 tx9 gateway disable <box>
 ```
 
+### Portable custom services
+
+An executable regular file placed directly in
+`~/.config/hermes-box/services.d/` defines a custom service. Names must match
+`[a-z0-9][a-z0-9_-]{0,62}`; directories, symlinks, non-executable files, and
+unsafe names are ignored and reported by `hb services`. Definitions live on
+the portable `/data` volume, so they survive backup, import, and upgrade.
+
+Each file is executed directly by absolute path, without shell sourcing,
+evaluation, arguments, or interpolation. Scripts must remain in the
+foreground and should `exec` their daemon. For example:
+
+```bash
+install -d -m 700 ~/.config/hermes-box/services.d
+cat >~/.config/hermes-box/services.d/signal <<'EOF'
+#!/usr/bin/env bash
+exec signal-cli daemon
+EOF
+chmod 700 ~/.config/hermes-box/services.d/signal
+hb services-reload
+```
+
+The existing 20-second workload loop also reconciles definitions
+automatically. Each service is supervised independently with a bounded
+restart delay; one crashing service cannot block the others. `hb pause` and
+`hb down` synchronously stop all custom services and prevent restart, while
+`hb up`/`hb resume` start them again. Disabling only the Hermes gateway does
+not affect custom services. Container termination is forwarded through the
+log supervisor to each foreground process.
+
+`hb services` distinguishes a stable foreground child (`running`) from a
+live capture wrapper waiting to restart it (`restarting`). This is process
+state, not an application health check; `hb doctor` cannot infer a generic
+health contract for arbitrary service code.
+
+Service output uses the same rotating, redacted durable log pipeline as other
+tx9-owned processes. Sources are distinct and collision-free:
+
+```bash
+tx9 logs media-bot --source service-signal
+tx9 logs media-bot --source all
+```
+
+Drop-ins have the same permissions and network/filesystem access as the
+`agent` user. Treat installing one as installing executable code in the box;
+the filename and regular non-symlink checks prevent accidental shell
+evaluation or symlink execution, but do not sandbox trusted service code.
+
 ## Logs and resource allocation
 
 Runtime output from the agent supervisor, Hermes gateway, and Executor is
diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh
index 49d3b7c..842ad06 100755
--- a/docker/entrypoint.sh
+++ b/docker/entrypoint.sh
@@ -1,8 +1,9 @@
 #!/usr/bin/env bash
 # Container PID-1 workload (run under docker --init so signals behave).
-# Reuses guest/hb-workload verbatim: it reconciles Executor, the Hermes
-# gateway, and the 0.0.0.0 socat bridges every 20s. tx9-logs supervises the
-# loop process itself; Docker's restart policy supervises the container.
+# Reuses guest/hb-workload verbatim: it reconciles custom services, Executor,
+# the Hermes gateway, and the 0.0.0.0 socat bridges every 20s. tx9-logs
+# supervises the loop process itself; Docker's restart policy supervises the
+# container.
 set -uo pipefail
 
 trap 'exit 143' TERM
diff --git a/docs/tx9-cli-design.md b/docs/tx9-cli-design.md
index 4d57158..e625f73 100644
--- a/docs/tx9-cli-design.md
+++ b/docs/tx9-cli-design.md
@@ -52,7 +52,7 @@ Docker overview can be read. `tx9 help` remains Docker-independent.
 | `tx9 backup <box>` (aliases `export`, `save`) | Flags: `--path` (default `~/Downloads`), `--password`/env/prompt, `--no-encrypt`. Quiesce → archive agent /data → validate → (encrypt) → verify → `<box>-<timestamp>.tx9`. |
 | `tx9 import <file.tx9>` (aliases `load`, `restore`) | Flags: `--name`, `--password`/env/prompt. Validate before creating anything; restore staged; arrive quiesced + gateway-disabled + fresh token; fail on name collision. |
 | `tx9 mount <add\|list\|remove> ...` | Persist host-directory bind mounts for an agent and recreate only its disposable container. Targets must be below `/mnt`, outside the portable `/data` volume. `add` supports `--read-only` and `--require-mountpoint`. |
-| `tx9 logs <box>` | Query durable agent, Executor, Hermes, Codex, and Claude events. Filters include source, age, text, severity (`--level`, this level and above; unleveled events count as info), count, and normalized JSONL. `tx9 logs export <box>` creates a mode-0600 portable log bundle from both isolated volumes. |
+| `tx9 logs <box>` | Query durable agent, Executor, Hermes, Codex, Claude, and custom-service events. Filters include source, age, text, severity (`--level`, this level and above; unleveled events count as info), count, and normalized JSONL. Custom sources use `service-<name>`. `tx9 logs export <box>` creates a mode-0600 portable log bundle from both isolated volumes. |
 | `tx9 resources <box>` | Show actual container CPU/RAM limits and volume usage versus advisory budgets. `resources set` updates limits live and persists them; `resources reset` restores 4 CPU/8 GiB (agent) and 2 CPU/2 GiB (Executor). |
 | `tx9 gateway <status\|enable\|disable> <box>` | Inspect or control the container-supervised Hermes gateway. Enable requires `--confirm-single-writer`. |
 | `tx9 open <box>` | Print (or open) the authenticated dashboard URL (`?_token=`). |
@@ -108,6 +108,20 @@ the port:
   SQLite histories. Executor data is never mounted into the agent container.
   This provides complete tx9-owned runtime output, not an independent audit
   trail for operations that Executor does not emit to any durable sink.
+- **Portable custom services**: direct executable regular-file children of
+  `${XDG_CONFIG_HOME:-$HOME/.config}/hermes-box/services.d` are supervised as
+  the agent. Names are limited to `[a-z0-9][a-z0-9_-]{0,62}`. The absolute
+  file path is passed as argv directly, never sourced or evaluated; scripts
+  stay foreground and should `exec` their daemon. Each service has an
+  independent `tx9-logs` capture and bounded restart loop under source
+  `service-<name>`. Runtime PID/lock state stays outside `/data`, while the
+  definitions and logs remain on `/data` and therefore travel in backups.
+  `hb services` reports status and ignored entries; `hb services-reload`
+  reconciles immediately in addition to the normal 20-second loop. Quiesce
+  synchronously stops every custom service and blocks restart until
+  `hb up`/`hb resume`; Hermes gateway enable/disable is independent. Drop-ins
+  run with the agent's ordinary access and are trusted executable code, not a
+  sandbox boundary.
 - **Gateway single-writer**: restored boxes never auto-enable the Hermes
   gateway. `tx9 gateway enable <box> --confirm-single-writer` delegates to
   `hb gateway-enable` and stays the only host-side path.
diff --git a/guest/hb b/guest/hb
index 577175c..d6213d5 100755
--- a/guest/hb
+++ b/guest/hb
@@ -4,8 +4,9 @@ set -uo pipefail
 # shellcheck disable=SC1091
 [ -f /etc/profile.d/hermes-box.sh ] && . /etc/profile.d/hermes-box.sh
 export TX9_LOG_MAX_BYTES TX9_LOG_MAX_FILES
+HB_BIN_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 # shellcheck source=guest/lib-mcp.sh
-. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/lib-mcp.sh"
+. "$HB_BIN_DIR/lib-mcp.sh"
 
 DATA="${HB_DATA:-/data}"
 AGENT_HOME="$DATA/home/agent"
@@ -20,6 +21,7 @@ GATEWAY_LOCK="$STATE_DIR/gateway.lock"
 GATEWAY_RELOAD_REQUESTS="$STATE_DIR/gateway-reload-requests"
 GATEWAY_RELOAD_LOCK="$STATE_DIR/gateway-reload.lock"
 EXECUTOR_LOCK="$STATE_DIR/executor.lock"
+SERVICES_DIR="$STATE_DIR/services.d"
 WIRE_VERSION=http-v3
 WIRED="$AGENT_HOME/.hermes-box-wired"
 TOKEN_FILE="$AGENT_HOME/.executor/server-control/auth.json"
@@ -64,9 +66,11 @@ init() {
   local managed=(
     "$AGENT_HOME" "$AGENT_HOME/.claude" "$AGENT_HOME/.codex" "$AGENT_HOME/.hermes"
     "$AGENT_HOME/workspace" "$AGENT_HOME/.config" "$AGENT_HOME/.config/hermes-box"
-    "$AGENT_HOME/.local" "$AGENT_HOME/.local/share" "$AGENT_HOME/.local/state" "$LOGS"
+    "$SERVICES_DIR" "$AGENT_HOME/.local" "$AGENT_HOME/.local/share"
+    "$AGENT_HOME/.local/state" "$LOGS"
   )
   mkdir -p "${managed[@]}"
+  chmod 0700 "$SERVICES_DIR" 2>/dev/null || true
   id agent >/dev/null 2>&1 && chown agent:agent "${managed[@]}" 2>/dev/null || true
   if [[ -e "$LEGACY_QUIESCE_FILE" ]]; then
     touch "$QUIESCE_FILE" || return 1
@@ -81,6 +85,20 @@ init() {
   id agent >/dev/null 2>&1 && chown agent:agent "$GATEWAY_DISABLED" "$GATEWAY_POLICY" 2>/dev/null || true
 }
 
+_services_helper() {
+  local helper
+  helper="$(command -v tx9-services 2>/dev/null || true)"
+  [[ -n "$helper" ]] || helper="$HB_BIN_DIR/tx9-services"
+  [[ -x "$helper" ]] || {
+    echo "tx9-services is unavailable" >&2
+    return 1
+  }
+  TX9_SERVICES_CONFIG_ROOT="$STATE_DIR" TX9_SERVICES_LOG_DIR="$LOGS" "$helper" "$@"
+}
+
+_services_reconcile() { _services_helper reconcile; }
+_services_stop() { _services_helper stop-all; }
+
 _port_open() {
   (exec 3<>"/dev/tcp/$EXECUTOR_HOST/${EXECUTOR_PORT:-4788}") 2>/dev/null && { exec 3>&-; return 0; }
   return 1
@@ -333,15 +351,29 @@ _start_gateway() {
 }
 
 up() {
+  local services_failed=0
+  local gateway_status
+
   init
   rm -f "$QUIESCE_FILE"
+  _services_reconcile || {
+    services_failed=1
+    echo "warning: custom service reconciliation failed; run 'hb services-reload' for details" >&2
+  }
   _executor_up || return 1
   _start_gateway
+  gateway_status=$?
+  [[ "$gateway_status" == 0 ]] || return "$gateway_status"
+  [[ "$services_failed" == 0 ]]
 }
 
 reconcile() {
   init
-  [[ ! -e "$QUIESCE_FILE" ]] || return 0
+  if [[ -e "$QUIESCE_FILE" ]]; then
+    _services_stop
+    return
+  fi
+  _services_reconcile || echo "warning: custom service reconciliation failed; continuing core reconciliation" >&2
   HB_STEADY_QUIET=1 _executor_up || return 1
   _start_gateway
 }
@@ -372,13 +404,14 @@ pause() {
   local failed=0
   init
   touch "$QUIESCE_FILE"
+  _services_stop || failed=1
   _stop_gateway || failed=1
   _stop_executor || failed=1
   if command -v hermes-state >/dev/null 2>&1; then
     hermes-state verify --checkpoint >/dev/null || failed=1
   fi
   [[ "$failed" == 0 ]] || return 1
-  echo "Hermes gateway and Executor: quiesced"
+  echo "Custom services, Hermes gateway, and Executor: quiesced"
 }
 
 resume() {
@@ -729,7 +762,7 @@ write_manifest() {
 }
 
 status() {
-  local gateway_status
+  local gateway_status services_count
   if [[ -e "$GATEWAY_DISABLED" ]]; then
     gateway_status=disabled
   elif _gateway_running; then
@@ -747,9 +780,20 @@ status() {
     printf 'platform: no runtime state\n'
   fi
   printf 'mcp:      %s\n' "$([[ -f "$WIRED" ]] && cat "$WIRED" || echo unwired)"
+  services_count="$(find "$SERVICES_DIR" -mindepth 1 -maxdepth 1 -printf x 2>/dev/null | wc -c | tr -d ' ')"
+  printf "services: %s drop-in entries (run 'hb services')\n" "$services_count"
   printf 'data:     %s\n' "$(du -sh "$DATA" 2>/dev/null | cut -f1)"
 }
 
+services() { _services_helper status; }
+
+services_reload() {
+  local failed=0
+  _services_reconcile || failed=1
+  services || failed=1
+  return "$failed"
+}
+
 versions() {
   echo "node:     $(node --version 2>/dev/null || echo -)"
   # sed -n 1p, not head -1: vp prints more after the version line, and head
@@ -822,6 +866,7 @@ doctor() {
     _check "Codex HTTP MCP config" grep -qF "url = \"$(_executor_url)\"" "$CODEX_HOME/config.toml" || failed=1
     _check "Hermes HTTP MCP config" _hermes_http_config || failed=1
   fi
+  echo "info custom services have no generic health contract; inspect process state with 'hb services'"
   [[ "$failed" == 0 ]]
 }
 
@@ -861,9 +906,11 @@ main() {
     write-manifest) write_manifest ;;
     doctor) doctor ;;
     status) status ;;
+    services) services ;;
+    services-reload) services_reload ;;
     versions) versions ;;
     logs) shift; logs "${1:-executor}" ;;
-    *) echo "usage: hb {init|up|reconcile|down|pause|resume|gateway-enable|gateway-disable|gateway-reload-if-requested|verify-state|acknowledge-active-paths|cutover-ready|web|wire-mcp|doctor|status|versions|logs}"; return 1 ;;
+    *) echo "usage: hb {init|up|reconcile|down|pause|resume|services|services-reload|gateway-enable|gateway-disable|gateway-reload-if-requested|verify-state|acknowledge-active-paths|cutover-ready|web|wire-mcp|doctor|status|versions|logs}"; return 1 ;;
   esac
 }
 
diff --git a/guest/hb-workload b/guest/hb-workload
index b19fd79..4019a97 100755
--- a/guest/hb-workload
+++ b/guest/hb-workload
@@ -17,6 +17,8 @@ EXECUTOR_TARGET_PORT="${EXECUTOR_PORT:-4788}"
 HERMES_TARGET_PORT="${HERMES_API_PORT:-8642}"
 LOGS="${HB_WORKLOAD_LOGS:-/data/logs}"
 BOX_ENV="${HB_BOX_ENV:-/etc/hermes-box.env}"
+SERVICES_HELPER="${HB_SERVICES_HELPER:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/tx9-services}"
+WORKLOAD_SLEEP_PID=''
 
 _exec_up() { (exec 3<>"/dev/tcp/127.0.0.1/$EXECUTOR_TARGET_PORT") 2>/dev/null && { exec 3>&-; return 0; }; return 1; }
 _api_up() { (exec 3<>"/dev/tcp/127.0.0.1/$HERMES_TARGET_PORT") 2>/dev/null && { exec 3>&-; return 0; }; return 1; }
@@ -54,6 +56,22 @@ _spawn_logged() {
   fi
 }
 
+_stop_custom_services() {
+  [[ -x "$SERVICES_HELPER" ]] || return 0
+  TX9_SERVICES_CONFIG_ROOT="$(dirname "$QUIESCE_FILE")" \
+    TX9_SERVICES_LOG_DIR="$LOGS" "$SERVICES_HELPER" stop-all
+}
+
+_shutdown() {
+  local exit_status="$1"
+  trap - TERM INT
+  [[ -z "$WORKLOAD_SLEEP_PID" ]] || kill -TERM "$WORKLOAD_SLEEP_PID" 2>/dev/null || true
+  _stop_custom_services || echo "custom service shutdown failed during workload termination" >&2
+  _stop_executor_bridge
+  _stop_api_bridge
+  exit "$exit_status"
+}
+
 _gateway_reload_pending() {
   [ -d "$GATEWAY_RELOAD_REQUESTS" ] &&
     [ -n "$(find "$GATEWAY_RELOAD_REQUESTS" -maxdepth 1 -type f -name 'request.*' -print -quit)" ]
@@ -89,6 +107,10 @@ reconcile_once() {
     return
   fi
   if [ -e "$QUIESCE_FILE" ] || [ -e "$LEGACY_QUIESCE_FILE" ]; then
+    # Reconciliation while quiesced is a stop-only operation for portable
+    # custom services. Explicit hb pause performs the synchronous gate; this
+    # also covers a durable marker restored or created between loop cycles.
+    _stop_custom_services || echo "custom service shutdown reconciliation failed" >&2
     _stop_executor_bridge
     _stop_api_bridge
     return
@@ -121,11 +143,19 @@ reconcile_once() {
 }
 
 main() {
+  # Install lifecycle traps only for the executed workload. Regression tests
+  # source this file to exercise reconcile_once and must retain their own
+  # process-level trap policy.
+  trap '_shutdown 143' TERM
+  trap '_shutdown 130' INT
   mkdir -p "$LOGS"
   while true; do
     reconcile_once
     [[ "${HB_WORKLOAD_ONCE:-0}" != 1 ]] || break
-    sleep 20
+    sleep 20 &
+    WORKLOAD_SLEEP_PID=$!
+    wait "$WORKLOAD_SLEEP_PID" 2>/dev/null || true
+    WORKLOAD_SLEEP_PID=''
   done
 }
 
diff --git a/guest/tx9-logs b/guest/tx9-logs
index c3507a4..f512f8e 100755
--- a/guest/tx9-logs
+++ b/guest/tx9-logs
@@ -74,6 +74,7 @@ SOURCE_ALIASES = {
     "socat": "agent",
     "hermes-api-socat": "hermes",
 }
+SERVICE_SOURCE_RE = re.compile(r"service-[a-z0-9][a-z0-9_-]{0,62}")
 
 BEARER_RE = re.compile(r"(?i)(\bbearer[ \t]+)([^\s,;\"']+)")
 AUTHORIZATION_RE = re.compile(r"(?i)(\b(?:proxy-)?authorization\b[ \t]*[=:][ \t]*)([^\r\n]+)")
@@ -1146,7 +1147,12 @@ def selected_sources(value: str) -> set[str] | None:
     if not values or "all" in values:
         return None
     normalized = {normalize_source(item) for item in values}
-    unknown = normalized - {"agent", "executor", "hermes", "codex", "claude"}
+    unknown = {
+        source
+        for source in normalized
+        if source not in {"agent", "executor", "hermes", "codex", "claude"}
+        and SERVICE_SOURCE_RE.fullmatch(source) is None
+    }
     if unknown:
         raise ValueError("unknown source(s): " + ", ".join(sorted(unknown)))
     return normalized
@@ -1616,7 +1622,10 @@ def runtime_source(stem: str, scope: str) -> str:
         return "executor"
     stem = stem.partition(".legacy-")[0]
     source = normalize_source(stem)
-    if source not in {"agent", "hermes", "codex", "claude"}:
+    if (
+        source not in {"agent", "hermes", "codex", "claude"}
+        and SERVICE_SOURCE_RE.fullmatch(source) is None
+    ):
         return "agent"
     return source
 
@@ -1641,7 +1650,10 @@ def read_runtime_logs(root: Path, *, scope: str, box: str) -> Iterator[dict[str,
         if replacement and replacement in names:
             continue
         source = runtime_source(active.stem, scope)
-        if source not in {"agent", "executor", "hermes", "codex", "claude"}:
+        if (
+            source not in {"agent", "executor", "hermes", "codex", "claude"}
+            and SERVICE_SOURCE_RE.fullmatch(source) is None
+        ):
             source = "agent" if scope == "agent" else "executor"
         yield from read_raw(active, root=root, box=box, source=source)
 
@@ -1842,7 +1854,9 @@ def filtered_events(args: argparse.Namespace) -> Iterator[tuple[dt.datetime, int
     agent_home = home_for_root(agent_root)
 
     streams: list[Iterable[dict[str, Any]]] = []
-    if selected is None or selected & {"agent", "hermes", "codex", "claude"}:
+    if selected is None or selected & {"agent", "hermes", "codex", "claude"} or any(
+        SERVICE_SOURCE_RE.fullmatch(source) is not None for source in selected
+    ):
         streams.append(read_runtime_logs(agent_root, scope="agent", box=args.box))
     if source_wanted("executor", selected):
         streams.append(read_runtime_logs(executor_root, scope="executor", box=args.box))
@@ -2012,7 +2026,11 @@ def add_query_options(parser: argparse.ArgumentParser, *, default_tail: int | No
     parser.add_argument("--agent-root", required=True, help="read-only agent volume root")
     parser.add_argument("--executor-root", required=True, help="read-only Executor volume root")
     parser.add_argument("--box", required=True, help="box name used in normalized events")
-    parser.add_argument("--source", default="all", help="comma-separated agent,executor,hermes,codex,claude")
+    parser.add_argument(
+        "--source",
+        default="all",
+        help="comma-separated agent,executor,hermes,codex,claude,service-<name>",
+    )
     parser.add_argument("--tail", type=lambda value: positive_int(value), default=default_tail)
     parser.add_argument("--since", default="", help="duration such as 24h or RFC3339 timestamp")
     parser.add_argument("--grep", default="", help="case-insensitive text filter")
diff --git a/guest/tx9-services b/guest/tx9-services
new file mode 100755
index 0000000..cbd8afe
--- /dev/null
+++ b/guest/tx9-services
@@ -0,0 +1,392 @@
+#!/usr/bin/env bash
+# Portable guest-side service supervisor for executable drop-ins.
+set -uo pipefail
+
+CONFIG_ROOT="${TX9_SERVICES_CONFIG_ROOT:-${XDG_CONFIG_HOME:-$HOME/.config}/hermes-box}"
+DEFINITIONS="$CONFIG_ROOT/services.d"
+STATE_DIR="${TX9_SERVICES_STATE_DIR:-${XDG_RUNTIME_DIR:-/tmp}/hermes-box-services-$(id -u)}"
+LOG_DIR="${TX9_SERVICES_LOG_DIR:-/data/logs}"
+QUIESCE_FILE="${TX9_SERVICES_QUIESCE_FILE:-$CONFIG_ROOT/quiesced}"
+RESTART_DELAY="${TX9_SERVICES_RESTART_DELAY:-5}"
+STOP_TIMEOUT="${TX9_SERVICES_STOP_TIMEOUT:-5}"
+LOCK_WAIT="${TX9_SERVICES_LOCK_WAIT:-10}"
+LOCK_FILE="$STATE_DIR/reconcile.lock"
+LOCK_FD=''
+
+_safe_name() { [[ "$1" =~ ^[a-z0-9][a-z0-9_-]{0,62}$ ]]; }
+
+_ensure_dirs() {
+  mkdir -p "$DEFINITIONS" "$STATE_DIR" "$LOG_DIR" || return 1
+  chmod 0700 "$DEFINITIONS" 2>/dev/null || true
+  chmod 0700 "$STATE_DIR" 2>/dev/null || true
+}
+
+_log_helper() {
+  local helper="${TX9_SERVICES_LOG_HELPER:-}"
+  if [[ -z "$helper" ]]; then
+    helper="$(command -v tx9-logs 2>/dev/null || true)"
+  fi
+  [[ -n "$helper" && "$helper" == /* && -x "$helper" ]] || {
+    echo "tx9-services: tx9-logs is unavailable" >&2
+    return 1
+  }
+  printf '%s\n' "$helper"
+}
+
+_acquire_lock() {
+  _ensure_dirs || return 1
+  exec {LOCK_FD}>"$LOCK_FILE"
+  if ! flock -w "$LOCK_WAIT" "$LOCK_FD"; then
+    echo "tx9-services: could not acquire reconciliation lock" >&2
+    eval "exec ${LOCK_FD}>&-"
+    LOCK_FD=''
+    return 1
+  fi
+}
+
+_release_lock() {
+  [[ -n "$LOCK_FD" ]] || return 0
+  flock -u "$LOCK_FD" 2>/dev/null || true
+  eval "exec ${LOCK_FD}>&-" 2>/dev/null || true
+  LOCK_FD=''
+}
+
+_definition_fingerprint() {
+  local path="$1" metadata digest
+  metadata="$(stat -c '%d:%i:%s:%a' -- "$path" 2>/dev/null)" || return 1
+  digest="$(sha256sum -- "$path" 2>/dev/null)" || return 1
+  digest="${digest%% *}"
+  digest="${digest#\\}"
+  [[ "$digest" =~ ^[a-f0-9]{64}$ ]] || return 1
+  printf '%s:%s\n' "$metadata" "$digest"
+}
+
+_valid_definition() {
+  local path="$1"
+  [[ ! -L "$path" && -f "$path" && -x "$path" ]]
+}
+
+_process_start_time() {
+  local pid="$1" line rest
+  local -a fields
+  [[ "$pid" =~ ^[1-9][0-9]*$ ]] || return 1
+  IFS= read -r line 2>/dev/null <"/proc/$pid/stat" || return 1
+  rest="${line##*) }"
+  read -r -a fields <<<"$rest"
+  ((${#fields[@]} >= 20)) || return 1
+  printf '%s\n' "${fields[19]}"
+}
+
+_read_state() {
+  local state_file="$1"
+  IFS=$'\t' read -r STATE_PID STATE_START STATE_FINGERPRINT STATE_HELPER STATE_DELAY 2>/dev/null <"$state_file" || return 1
+  [[ "$STATE_PID" =~ ^[1-9][0-9]*$ && "$STATE_START" =~ ^[0-9]+$ && -n "$STATE_FINGERPRINT" && "$STATE_HELPER" == /* && -n "$STATE_DELAY" ]]
+}
+
+_capture_argv_matches() {
+  local pid="$1" name="$2" definition="$3" helper="$4" delay="$5" index
+  local -a argv expected
+  mapfile -d '' -t argv <"/proc/$pid/cmdline" 2>/dev/null || return 1
+  ((${#argv[@]} > 0)) || return 1
+  if [[ "${argv[0]}" == "$helper" ]]; then
+    index=0
+  elif ((${#argv[@]} > 1)) && [[ "${argv[1]}" == "$helper" ]]; then
+    index=1
+  else
+    return 1
+  fi
+  expected=(
+    "$helper" capture --source "service-$name" --log-dir "$LOG_DIR"
+    --restart-delay "$delay" -- "$definition"
+  )
+  ((${#argv[@]} - index == ${#expected[@]})) || return 1
+  local offset
+  for ((offset = 0; offset < ${#expected[@]}; offset++)); do
+    [[ "${argv[index + offset]}" == "${expected[offset]}" ]] || return 1
+  done
+}
+
+_state_process_matches() {
+  local state_file="$1" name="$2" definition="$3" current_start current_uid
+  _read_state "$state_file" || return 1
+  current_start="$(_process_start_time "$STATE_PID")" || return 1
+  [[ "$current_start" == "$STATE_START" ]] || return 1
+  current_uid="$(stat -c '%u' -- "/proc/$STATE_PID" 2>/dev/null)" || return 1
+  [[ "$current_uid" == "$(id -u)" ]] || return 1
+  _capture_argv_matches "$STATE_PID" "$name" "$definition" "$STATE_HELPER" "$STATE_DELAY"
+}
+
+_write_state() {
+  local state_file="$1" pid="$2" start="$3" fingerprint="$4" helper="$5" delay="$6" tmp
+  tmp="$state_file.tmp.$$"
+  umask 077
+  printf '%s\t%s\t%s\t%s\t%s\n' "$pid" "$start" "$fingerprint" "$helper" "$delay" >"$tmp" || return 1
+  mv "$tmp" "$state_file"
+}
+
+_start_one() {
+  local name="$1" definition="$2" fingerprint="$3" helper pid start attempt state_file
+  state_file="$STATE_DIR/$name.state"
+  helper="$(_log_helper)" || return 1
+  (
+    # flock locks are tied to open file descriptions and survive fork. The
+    # long-lived capture must never inherit the reconciliation lock.
+    [[ -z "$LOCK_FD" ]] || eval "exec ${LOCK_FD}>&-"
+    exec "$helper" capture --source "service-$name" --log-dir "$LOG_DIR" \
+      --restart-delay "$RESTART_DELAY" -- "$definition"
+  ) </dev/null >/dev/null 2>&1 &
+  pid=$!
+  start="$(_process_start_time "$pid")" || {
+    wait "$pid" 2>/dev/null || true
+    echo "tx9-services: $name failed before supervision started" >&2
+    return 1
+  }
+  _write_state "$state_file" "$pid" "$start" "$fingerprint" "$helper" "$RESTART_DELAY" || {
+    kill -TERM "$pid" 2>/dev/null || true
+    wait "$pid" 2>/dev/null || true
+    return 1
+  }
+  for ((attempt = 0; attempt < 100; attempt++)); do
+    if _state_process_matches "$state_file" "$name" "$definition"; then
+      echo "tx9-services: started $name"
+      return 0
+    fi
+    kill -0 "$pid" 2>/dev/null || break
+    sleep 0.02
+  done
+  kill -TERM "$pid" 2>/dev/null || true
+  wait "$pid" 2>/dev/null || true
+  rm -f "$state_file"
+  echo "tx9-services: $name capture failed to start" >&2
+  return 1
+}
+
+_direct_children() {
+  local pid="$1" child current_uid
+  [[ -r "/proc/$pid/task/$pid/children" ]] || return 0
+  for child in $(<"/proc/$pid/task/$pid/children"); do
+    [[ "$child" =~ ^[1-9][0-9]*$ ]] || continue
+    current_uid="$(stat -c '%u' -- "/proc/$child" 2>/dev/null)" || continue
+    [[ "$current_uid" == "$(id -u)" ]] || continue
+    printf '%s\n' "$child"
+  done
+}
+
+_pid_alive() {
+  local state
+  kill -0 "$1" 2>/dev/null || return 1
+  state="$(ps -o stat= -p "$1" 2>/dev/null)" || return 1
+  [[ "$state" != Z* ]]
+}
+
+_supervised_child_running() {
+  local capture_pid="$1" child start current_start
+  local -a children
+  mapfile -t children < <(_direct_children "$capture_pid")
+  ((${#children[@]} == 1)) || return 1
+  child="${children[0]}"
+  _pid_alive "$child" || return 1
+  start="$(_process_start_time "$child")" || return 1
+  # A child that merely flashes into existence during a crash loop is not a
+  # running service. Require the same direct child to remain present across a
+  # short observation window; this is process-state reporting, not a generic
+  # application health check.
+  sleep 0.05
+  current_start="$(_process_start_time "$child")" || return 1
+  [[ "$current_start" == "$start" ]] || return 1
+  _pid_alive "$child" || return 1
+  [[ "$(sed -n 's/^PPid:[[:space:]]*//p' "/proc/$child/status" 2>/dev/null)" == "$capture_pid" ]]
+}
+
+_stop_one() {
+  local name="$1" definition="$2" attempt failed=0
+  local state_file="$STATE_DIR/$name.state"
+  local -a child_groups=()
+  if ! _state_process_matches "$state_file" "$name" "$definition"; then
+    rm -f "$state_file"
+    return 0
+  fi
+  mapfile -t child_groups < <(_direct_children "$STATE_PID")
+  kill -TERM "$STATE_PID" 2>/dev/null || true
+  for ((attempt = 0; attempt < STOP_TIMEOUT * 10; attempt++)); do
+    _pid_alive "$STATE_PID" || break
+    sleep 0.1
+  done
+  if _pid_alive "$STATE_PID"; then
+    local group
+    for group in "${child_groups[@]}"; do
+      kill -KILL -- "-$group" 2>/dev/null || true
+    done
+    kill -KILL "$STATE_PID" 2>/dev/null || true
+    for ((attempt = 0; attempt < 20; attempt++)); do
+      _pid_alive "$STATE_PID" || break
+      sleep 0.05
+    done
+  fi
+  if _pid_alive "$STATE_PID"; then
+    echo "tx9-services: $name capture survived stop" >&2
+    failed=1
+  fi
+  local group
+  for group in "${child_groups[@]}"; do
+    if kill -0 -- "-$group" 2>/dev/null; then
+      echo "tx9-services: $name process group survived stop" >&2
+      failed=1
+    fi
+  done
+  if [[ "$failed" == 0 ]]; then
+    rm -f "$state_file"
+    echo "tx9-services: stopped $name"
+  fi
+  return "$failed"
+}
+
+_definition_reason() {
+  local path="$1" name="$2"
+  if ! _safe_name "$name"; then printf 'unsafe name'; return; fi
+  if [[ -L "$path" ]]; then printf 'symlink'; return; fi
+  if [[ ! -f "$path" ]]; then printf 'not a regular file'; return; fi
+  if [[ ! -x "$path" ]]; then printf 'not executable'; return; fi
+  printf 'valid'
+}
+
+_reconcile_locked() {
+  local path name fingerprint state_file failed=0
+  local -A desired=()
+  while IFS= read -r -d '' path; do
+    name="${path##*/}"
+    if ! _valid_definition "$path" || ! _safe_name "$name"; then
+      echo "tx9-services: ignored $name ($(_definition_reason "$path" "$name"))" >&2
+      continue
+    fi
+    fingerprint="$(_definition_fingerprint "$path")" || {
+      echo "tx9-services: ignored $name (could not inspect definition)" >&2
+      continue
+    }
+    desired["$name"]="$fingerprint"
+  done < <(find "$DEFINITIONS" -mindepth 1 -maxdepth 1 -print0 2>/dev/null)
+
+  shopt -s nullglob
+  for state_file in "$STATE_DIR"/*.state; do
+    name="${state_file##*/}"
+    name="${name%.state}"
+    path="$DEFINITIONS/$name"
+    if [[ -z "${desired[$name]+yes}" ]]; then
+      _stop_one "$name" "$path" || failed=1
+      continue
+    fi
+    if _state_process_matches "$state_file" "$name" "$path" &&
+      _read_state "$state_file" && [[ "$STATE_FINGERPRINT" == "${desired[$name]}" ]]; then
+      continue
+    fi
+    _stop_one "$name" "$path" || failed=1
+  done
+  shopt -u nullglob
+
+  [[ ! -e "$QUIESCE_FILE" ]] || return "$failed"
+  for name in "${!desired[@]}"; do
+    state_file="$STATE_DIR/$name.state"
+    path="$DEFINITIONS/$name"
+    if _state_process_matches "$state_file" "$name" "$path" &&
+      _read_state "$state_file" && [[ "$STATE_FINGERPRINT" == "${desired[$name]}" ]]; then
+      continue
+    fi
+    _start_one "$name" "$path" "${desired[$name]}" || failed=1
+  done
+  return "$failed"
+}
+
+reconcile() {
+  _acquire_lock || return 1
+  _reconcile_locked
+  local status=$?
+  _release_lock
+  return "$status"
+}
+
+_stop_all_locked() {
+  local state_file name definition failed=0
+  shopt -s nullglob
+  for state_file in "$STATE_DIR"/*.state; do
+    name="${state_file##*/}"
+    name="${name%.state}"
+    definition="$DEFINITIONS/$name"
+    _stop_one "$name" "$definition" || failed=1
+  done
+  shopt -u nullglob
+  return "$failed"
+}
+
+stop_all() {
+  _acquire_lock || return 1
+  _stop_all_locked
+  local status=$?
+  _release_lock
+  return "$status"
+}
+
+_status_locked() {
+  local path name reason fingerprint state_file rows=0
+  local -A seen=()
+  printf 'NAME\tSTATUS\tDEFINITION\n'
+  while IFS= read -r -d '' path; do
+    name="${path##*/}"
+    seen["$name"]=1
+    rows=$((rows + 1))
+    reason="$(_definition_reason "$path" "$name")"
+    if [[ "$reason" != valid ]]; then
+      printf '%s\tignored (%s)\t%s\n' "$name" "$reason" "$path"
+      continue
+    fi
+    state_file="$STATE_DIR/$name.state"
+    fingerprint="$(_definition_fingerprint "$path" 2>/dev/null || true)"
+    if _state_process_matches "$state_file" "$name" "$path" && _read_state "$state_file"; then
+      if [[ "$STATE_FINGERPRINT" == "$fingerprint" ]]; then
+        if _supervised_child_running "$STATE_PID"; then
+          printf '%s\trunning\t%s\n' "$name" "$path"
+        else
+          printf '%s\trestarting\t%s\n' "$name" "$path"
+        fi
+      else
+        printf '%s\treload pending\t%s\n' "$name" "$path"
+      fi
+    elif [[ -e "$QUIESCE_FILE" ]]; then
+      printf '%s\tstopped (quiesced)\t%s\n' "$name" "$path"
+    else
+      printf '%s\tstopped\t%s\n' "$name" "$path"
+    fi
+  done < <(find "$DEFINITIONS" -mindepth 1 -maxdepth 1 -print0 2>/dev/null)
+
+  shopt -s nullglob
+  for state_file in "$STATE_DIR"/*.state; do
+    name="${state_file##*/}"
+    name="${name%.state}"
+    if [[ -z "${seen[$name]+yes}" ]]; then
+      rows=$((rows + 1))
+      printf '%s\tstale state\t%s\n' "$name" "$DEFINITIONS/$name"
+    fi
+  done
+  shopt -u nullglob
+  if [[ "$rows" == 0 ]]; then
+    printf 'No custom services configured. Add executable files to %s\n' "$DEFINITIONS"
+  fi
+}
+
+status() {
+  _acquire_lock || return 1
+  _status_locked
+  local result=$?
+  _release_lock
+  return "$result"
+}
+
+main() {
+  case "${1:-status}" in
+    reconcile | reload) reconcile ;;
+    stop-all) stop_all ;;
+    status) status ;;
+    *) echo 'usage: tx9-services {status|reconcile|reload|stop-all}' >&2; return 2 ;;
+  esac
+}
+
+if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then main "$@"; fi
diff --git a/internal/assets/assets.go b/internal/assets/assets.go
index ae42468..c66ee93 100644
--- a/internal/assets/assets.go
+++ b/internal/assets/assets.go
@@ -102,7 +102,7 @@ func fileMode(p string) int64 {
 	switch {
 	case strings.HasSuffix(p, ".sh"):
 		return 0o755
-	case p == "guest/hb" || strings.HasPrefix(p, "guest/hb-"):
+	case p == "guest/hb" || strings.HasPrefix(p, "guest/hb-") || p == "guest/tx9-services":
 		return 0o755
 	case p == "guest/hermes-state":
 		return 0o755
diff --git a/internal/assets/assets_test.go b/internal/assets/assets_test.go
index 4eca9ad..b18d953 100644
--- a/internal/assets/assets_test.go
+++ b/internal/assets/assets_test.go
@@ -6,6 +6,7 @@ import (
 	"io/fs"
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"
 )
 
@@ -51,9 +52,11 @@ func TestBuildContextTar(t *testing.T) {
 	if mode != 0o755 {
 		t.Errorf("guest/hb mode = %o, want 0755", mode)
 	}
-	for _, want := range []string{"box.env", "provision/provision.sh", "docker/entrypoint.sh", "docker/executor-entrypoint.sh", "guest/hb-workload", "guest/hermes-state"} {
+	for _, want := range []string{"box.env", "provision/provision.sh", "docker/entrypoint.sh", "docker/executor-entrypoint.sh", "guest/hb-workload", "guest/tx9-services", "guest/hermes-state"} {
 		if _, ok := modes[want]; !ok {
 			t.Errorf("%s missing from build context tar", want)
+		} else if strings.HasPrefix(want, "guest/") && modes[want] != 0o755 {
+			t.Errorf("%s mode = %o, want 0755", want, modes[want])
 		}
 	}
 	for name := range modes {
diff --git a/internal/cli/cmd_logs.go b/internal/cli/cmd_logs.go
index 7129260..6cbe2dc 100644
--- a/internal/cli/cmd_logs.go
+++ b/internal/cli/cmd_logs.go
@@ -42,7 +42,7 @@ func cmdLogs(args []string) error {
 
 	fs := flag.NewFlagSet("logs", flag.ContinueOnError)
 	opts := logsQueryOptions{}
-	fs.StringVar(&opts.Source, "source", "all", "comma-separated sources: agent,executor,hermes,codex,claude")
+	fs.StringVar(&opts.Source, "source", "all", "comma-separated sources: agent,executor,hermes,codex,claude,service-<name>")
 	fs.IntVar(&opts.Tail, "tail", 200, "show the newest N matching events")
 	fs.StringVar(&opts.Since, "since", "", "only events after a duration or RFC3339 timestamp (for example 24h)")
 	fs.StringVar(&opts.Contains, "grep", "", "only events containing text")
@@ -89,7 +89,7 @@ func cmdLogsExport(args []string) error {
 	fs := flag.NewFlagSet("logs export", flag.ContinueOnError)
 	opts := logsQueryOptions{}
 	output := fs.String("output", "", "output .tar.gz path (default: <box>-logs-<timestamp>.tar.gz)")
-	fs.StringVar(&opts.Source, "source", "all", "comma-separated sources: agent,executor,hermes,codex,claude")
+	fs.StringVar(&opts.Source, "source", "all", "comma-separated sources: agent,executor,hermes,codex,claude,service-<name>")
 	fs.StringVar(&opts.Since, "since", "", "only events after a duration or RFC3339 timestamp (for example 24h)")
 	fs.StringVar(&opts.Contains, "grep", "", "only events containing text")
 	fs.StringVar(&opts.Level, "level", "", "only events at this level or above: debug|info|warn|error (unleveled events count as info)")
diff --git a/internal/cli/dispatch.go b/internal/cli/dispatch.go
index ac2dce3..b4d25e1 100644
--- a/internal/cli/dispatch.go
+++ b/internal/cli/dispatch.go
@@ -41,7 +41,7 @@ var commandSpecs = []commandSpec{
 	{name: "enter", help: "exec into a box's agent container (--executor for the executor container)", aliases: []string{"ssh", "shell"}, run: cmdEnter},
 	{name: "start", help: "start a stopped box", run: cmdStart},
 	{name: "stop", help: "stop a running box", run: cmdStop},
-	{name: "logs", help: "query/export durable agent, Executor, Hermes, Codex, and Claude events", run: cmdLogs},
+	{name: "logs", help: "query/export durable agent, Executor, Hermes, Codex, Claude, and custom-service events", run: cmdLogs},
 	{name: "resources", help: "show/update container limits and volume budgets", run: cmdResources},
 	{name: "backup", help: "archive a box to a .tx9 file", aliases: []string{"export", "save"}, run: cmdBackup},
 	{name: "import", help: "restore a box from a .tx9 file", aliases: []string{"load", "restore"}, run: cmdImport},
diff --git a/internal/cli/dispatch_test.go b/internal/cli/dispatch_test.go
index 0bae2a1..66fcd96 100644
--- a/internal/cli/dispatch_test.go
+++ b/internal/cli/dispatch_test.go
@@ -59,6 +59,14 @@ func TestUsageShowsAliasesFromCommandSpecs(t *testing.T) {
 	}
 }
 
+func TestUsageMentionsCustomServiceLogs(t *testing.T) {
+	var usage bytes.Buffer
+	printUsage(&usage)
+	if !strings.Contains(usage.String(), "custom-service events") {
+		t.Fatalf("logs help does not mention custom services:\n%s", usage.String())
+	}
+}
+
 func TestNoArgumentsShowsOverviewAndCommands(t *testing.T) {
 	var stdout, stderr bytes.Buffer
 	status := runWithOverview([]string{"tx9"}, nil, func(w io.Writer) error {
diff --git a/provision/provision.sh b/provision/provision.sh
index 7838c93..bf067ad 100755
--- a/provision/provision.sh
+++ b/provision/provision.sh
@@ -220,12 +220,13 @@ install_config() {
 }
 
 place_assets() {
-  log "profile, tmux config, hb and log helpers"
+  log "profile, tmux config, hb, service, and log helpers"
   install -m 0644 "$CTX/guest/profile.sh"  /etc/profile.d/hermes-box.sh
   install -m 0644 "$CTX/guest/tmux.conf"    "$OPT/tmux.conf"
   install -m 0644 "$CTX/guest/lib-mcp.sh"   "$OPT/bin/lib-mcp.sh"
   install -m 0755 "$CTX/guest/hb"           "$OPT/bin/hb"
   install -m 0755 "$CTX/guest/hb-workload"  "$OPT/bin/hb-workload"
+  install -m 0755 "$CTX/guest/tx9-services" "$OPT/bin/tx9-services"
   install -m 0755 "$CTX/guest/hermes-state" "$OPT/bin/hermes-state"
   install -m 0755 "$CTX/guest/tx9-logs"      "$OPT/bin/tx9-logs"
   # agent login shell auto-attaches tmux; see guest/agent-bash-profile.sh
diff --git a/tests/regressions-hb-workload.sh b/tests/regressions-hb-workload.sh
index edcc446..a851232 100755
--- a/tests/regressions-hb-workload.sh
+++ b/tests/regressions-hb-workload.sh
@@ -359,6 +359,92 @@ HB_DATA="$gateway_data" "$PROJECT_ROOT/guest/hb" gateway-enable \
   [[ ! -e "$tmp/up-gateway-started" ]]
 )
 
+# Interactive startup reports a custom-service failure after still starting
+# Executor and Hermes. Resume preserves the same contract via hb up.
+(
+  HB_DATA="$tmp/up-service-failure-data"
+  HOME="$HB_DATA/home/agent"
+  mkdir -p "$HOME"
+  # shellcheck disable=SC1090
+  source "$PROJECT_ROOT/guest/hb"
+
+  calls="$tmp/up-service-failure.calls"
+  init() { mkdir -p "$STATE_DIR"; }
+  _services_reconcile() {
+    echo services >>"$calls"
+    return 1
+  }
+  _executor_up() {
+    echo executor >>"$calls"
+    return 0
+  }
+  _start_gateway() {
+    echo gateway >>"$calls"
+    return 0
+  }
+
+  if up >"$tmp/up-service-failure.stdout" 2>"$tmp/up-service-failure.stderr"; then
+    echo "hb up masked custom-service reconciliation failure" >&2
+    exit 1
+  fi
+  [[ "$(cat "$calls")" == $'services\nexecutor\ngateway' ]]
+  grep -q 'custom service reconciliation failed' "$tmp/up-service-failure.stderr"
+
+  : >"$calls"
+  touch "$QUIESCE_FILE"
+  if resume >"$tmp/resume-service-failure.stdout" 2>"$tmp/resume-service-failure.stderr"; then
+    echo "hb resume masked custom-service reconciliation failure" >&2
+    exit 1
+  fi
+  [[ ! -e "$QUIESCE_FILE" ]]
+  [[ "$(cat "$calls")" == $'services\nexecutor\ngateway' ]]
+  grep -q 'custom service reconciliation failed' "$tmp/resume-service-failure.stderr"
+)
+
+# Custom-service failures are isolated from the first-party lifecycle. Both
+# core reconciliation steps still run, hb reconcile returns their success,
+# and hb-workload is allowed to bring up the Hermes API bridge.
+(
+  HB_DATA="$tmp/service-isolation-data"
+  HOME="$HB_DATA/home/agent"
+  mkdir -p "$HOME/.config/hermes-box" "$tmp/service-isolation-logs"
+  # shellcheck disable=SC1090
+  source "$PROJECT_ROOT/guest/hb"
+  init() { :; }
+  _services_reconcile() { touch "$tmp/service-isolation-services"; return 1; }
+  _executor_up() { touch "$tmp/service-isolation-executor"; return 0; }
+  _start_gateway() { touch "$tmp/service-isolation-gateway"; return 0; }
+  reconcile >"$tmp/service-isolation.stdout" 2>"$tmp/service-isolation.stderr"
+  [[ -e "$tmp/service-isolation-services" ]]
+  [[ -e "$tmp/service-isolation-executor" ]]
+  [[ -e "$tmp/service-isolation-gateway" ]]
+  grep -q 'continuing core reconciliation' "$tmp/service-isolation.stderr"
+  services() { touch "$tmp/service-isolation-status"; return 0; }
+  if services_reload; then
+    echo "services-reload masked custom reconciliation failure" >&2
+    exit 1
+  fi
+  [[ -e "$tmp/service-isolation-status" ]]
+
+  # shellcheck disable=SC1090
+  source "$PROJECT_ROOT/guest/hb-workload"
+  PORT=4788
+  HERMES_BRIDGE_PORT=8659
+  LOGS="$tmp/service-isolation-logs"
+  BOX_ENV="$tmp/service-isolation-box.env"
+  LEGACY_QUIESCE_FILE="$tmp/service-isolation-legacy"
+  printf 'EXECUTOR_PORT=4788\nHERMES_API_PORT=8642\n' >"$BOX_ENV"
+  rm -f "$GATEWAY_DISABLED" "$QUIESCE_FILE"
+  hb() { [[ "$1" == reconcile ]] && reconcile; }
+  pgrep() { return 1; }
+  pkill() { :; }
+  _wait_api_ready() { return 0; }
+  socat() { printf '%s\n' "$*" >>"$tmp/service-isolation-bridge"; }
+  reconcile_once
+  wait
+  grep -q 'TCP-LISTEN:8659,' "$tmp/service-isolation-bridge"
+)
+
 # The reconcile loop re-checks Executor every cycle; unchanged reachability
 # must log only on state changes so the durable agent stream is not flooded
 # with one identical line per cycle. Interactive paths always report.
@@ -415,6 +501,27 @@ HB_DATA="$gateway_data" "$PROJECT_ROOT/guest/hb" gateway-enable \
   [[ ! -e "$tmp/hermes-state-called" ]]
 )
 
+# Top-level status links to custom-service detail, while doctor explicitly
+# avoids presenting arbitrary processes as generically health-checkable.
+(
+  HB_DATA="$tmp/services-discoverability-data"
+  HERMES_HOME="$HB_DATA/home/agent/.hermes"
+  # shellcheck disable=SC1090
+  source "$PROJECT_ROOT/guest/hb"
+  init
+  _port_open() { return 1; }
+  _gateway_running() { return 1; }
+  status >"$tmp/services-discoverability.status"
+  grep -Fq "services: 0 drop-in entries (run 'hb services')" \
+    "$tmp/services-discoverability.status"
+  _check() { return 0; }
+  INSTALL_HERMES=0 INSTALL_EXECUTOR=0 WIRE_EXECUTOR_MCP=0 \
+    doctor >"$tmp/services-discoverability.doctor"
+  grep -Fq "custom services have no generic health contract" \
+    "$tmp/services-discoverability.doctor"
+  grep -Fq "'hb services'" "$tmp/services-discoverability.doctor"
+)
+
 # Doctor validates Codex MCP wiring against the configured Executor port.
 (
   HB_DATA="$tmp/doctor-port-data"
diff --git a/tests/regressions-services.sh b/tests/regressions-services.sh
new file mode 100755
index 0000000..74bc123
--- /dev/null
+++ b/tests/regressions-services.sh
@@ -0,0 +1,304 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC2016,SC2030,SC2031,SC2329
+set -euo pipefail
+
+# shellcheck source=tests/lib.sh
+source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/lib.sh"
+
+data="$tmp/data"
+home="$data/home/agent"
+config_root="$home/.config/hermes-box"
+definitions="$config_root/services.d"
+runtime="$tmp/runtime"
+logs="$data/logs"
+events="$tmp/events"
+mkdir -p "$definitions" "$runtime" "$logs" "$events"
+
+export TX9_SERVICES_CONFIG_ROOT="$config_root"
+export TX9_SERVICES_STATE_DIR="$runtime"
+export TX9_SERVICES_LOG_DIR="$logs"
+export TX9_SERVICES_LOG_HELPER="$PROJECT_ROOT/guest/tx9-logs"
+export TX9_SERVICES_RESTART_DELAY=0.2
+export TX9_SERVICES_STOP_TIMEOUT=2
+export SERVICE_TEST_EVENTS="$events"
+services="$PROJECT_ROOT/guest/tx9-services"
+
+wait_for_count() {
+  local expected="$1" file="$2" attempt count
+  for ((attempt = 0; attempt < 150; attempt++)); do
+    if [[ -f "$file" ]]; then count="$(wc -l <"$file")"; else count=0; fi
+    [[ "$count" -ge "$expected" ]] && return 0
+    sleep 0.02
+  done
+  echo "timed out waiting for $expected records in $file" >&2
+  return 1
+}
+
+wait_for_absent() {
+  local path="$1" attempt
+  for ((attempt = 0; attempt < 150; attempt++)); do
+    [[ ! -e "$path" ]] && return 0
+    sleep 0.02
+  done
+  echo "timed out waiting for $path to disappear" >&2
+  return 1
+}
+
+cat >"$definitions/healthy" <<'EOF'
+#!/usr/bin/env bash
+printf 'start %s\n' "$$" >>"$SERVICE_TEST_EVENTS/healthy.starts"
+printf 'healthy-log\n'
+trap 'printf "term %s\n" "$$" >>"$SERVICE_TEST_EVENTS/healthy.terms"; exit 0' TERM INT
+while :; do sleep 1; done
+EOF
+chmod 0700 "$definitions/healthy"
+
+cat >"$definitions/broken" <<'EOF'
+#!/usr/bin/env bash
+printf 'start %s\n' "$$" >>"$SERVICE_TEST_EVENTS/broken.starts"
+printf 'broken-log\n'
+exit 17
+EOF
+chmod 0700 "$definitions/broken"
+
+# Invalid direct children are ignored and reported: unsafe name, non-executable
+# regular file, directory, and symlink.
+printf '#!/usr/bin/env bash\nexit 0\n' >"$definitions/BadName"
+chmod 0700 "$definitions/BadName"
+printf '#!/usr/bin/env bash\nexit 0\n' >"$definitions/disabled"
+mkdir "$definitions/not-a-file"
+ln -s "$definitions/healthy" "$definitions/linked"
+
+"$services" reconcile
+wait_for_count 1 "$events/healthy.starts"
+wait_for_count 2 "$events/broken.starts"
+[[ -s "$runtime/healthy.state" ]]
+[[ -s "$runtime/broken.state" ]]
+if ! flock -n "$runtime/reconcile.lock" -c true; then
+  echo "service capture inherited and pinned the reconciliation lock" >&2
+  exit 1
+fi
+
+# A stale or forged PID record is never authority to kill an arbitrary
+# same-user process. PID identity and the exact capture argv must both match.
+sleep 30 &
+decoy_pid=$!
+pids+=("$decoy_pid")
+decoy_start="$(awk '{print $22}' "/proc/$decoy_pid/stat")"
+printf '%s\t%s\tstale\t%s\t%s\n' "$decoy_pid" "$decoy_start" \
+  "$TX9_SERVICES_LOG_HELPER" "$TX9_SERVICES_RESTART_DELAY" \
+  >"$runtime/ghost.state"
+"$services" reconcile >/dev/null 2>&1
+kill -0 "$decoy_pid"
+[[ ! -e "$runtime/ghost.state" ]]
+kill "$decoy_pid"
+wait "$decoy_pid" 2>/dev/null || true
+
+"$services" status >"$tmp/services.status"
+grep -Fq $'healthy\trunning' "$tmp/services.status"
+grep -Fq $'broken\trestarting' "$tmp/services.status"
+if grep -Fq $'broken\trunning' "$tmp/services.status"; then
+  echo "crash-looping service was reported as running" >&2
+  exit 1
+fi
+grep -Fq $'BadName\tignored (unsafe name)' "$tmp/services.status"
+grep -Fq $'disabled\tignored (not executable)' "$tmp/services.status"
+grep -Fq $'not-a-file\tignored (not a regular file)' "$tmp/services.status"
+grep -Fq $'linked\tignored (symlink)' "$tmp/services.status"
+
+# The flock and runtime identity record serialize concurrent reconciliations;
+# the already-running foreground service is not launched twice.
+for _ in 1 2 3 4 5 6 7 8; do
+  "$services" reconcile >"$tmp/concurrent.$_.out" 2>"$tmp/concurrent.$_.err" &
+done
+wait
+[[ "$(wc -l <"$events/healthy.starts" | tr -d ' ')" == 1 ]]
+
+# Definition identity includes the content digest, not just whole-second stat
+# metadata. Rewrite a same-size script in place, restore its original mtime,
+# and require immediate replacement of the old foreground child.
+cat >"$definitions/reloadable" <<'EOF'
+#!/usr/bin/env bash
+printf 'old\n' >>"$SERVICE_TEST_EVENTS/reloadable.starts"
+trap 'printf "old\n" >>"$SERVICE_TEST_EVENTS/reloadable.terms"; exit 0' TERM INT
+while :; do sleep 1; done
+EOF
+chmod 0700 "$definitions/reloadable"
+"$services" reconcile >/dev/null
+wait_for_pattern old "$events/reloadable.starts"
+reload_metadata="$(stat -c '%d:%i:%Y:%s:%a' "$definitions/reloadable")"
+reload_mtime="$(stat -c '%Y' "$definitions/reloadable")"
+cat >"$definitions/reloadable" <<'EOF'
+#!/usr/bin/env bash
+printf 'new\n' >>"$SERVICE_TEST_EVENTS/reloadable.starts"
+trap 'printf "new\n" >>"$SERVICE_TEST_EVENTS/reloadable.terms"; exit 0' TERM INT
+while :; do sleep 1; done
+EOF
+touch -d "@$reload_mtime" "$definitions/reloadable"
+[[ "$(stat -c '%d:%i:%Y:%s:%a' "$definitions/reloadable")" == "$reload_metadata" ]]
+"$services" reconcile >/dev/null
+wait_for_pattern old "$events/reloadable.terms"
+wait_for_pattern new "$events/reloadable.starts"
+rm -f "$definitions/reloadable"
+"$services" reconcile >/dev/null
+
+# A crashing service restarts with the configured bounded delay while the
+# healthy service stays up and reconciliation remains responsive.
+before="$(wc -l <"$events/broken.starts" | tr -d ' ')"
+sleep 0.1
+after_early="$(wc -l <"$events/broken.starts" | tr -d ' ')"
+[[ "$after_early" -le $((before + 1)) ]]
+wait_for_count $((before + 1)) "$events/broken.starts"
+[[ "$(wc -l <"$events/healthy.starts" | tr -d ' ')" == 1 ]]
+
+# Each custom source is independently queryable, and `all` includes it.
+wait_for_pattern healthy-log "$logs/service-healthy.jsonl"
+"$PROJECT_ROOT/guest/tx9-logs" query --agent-root "$data" \
+  --executor-root "$tmp/empty-executor" --box fixture --source service-healthy \
+  --tail 100 --json >"$tmp/service-query.jsonl"
+jq -e 'select(.source == "service-healthy" and .message == "healthy-log")' \
+  "$tmp/service-query.jsonl" >/dev/null
+"$PROJECT_ROOT/guest/tx9-logs" query --agent-root "$data" \
+  --executor-root "$tmp/empty-executor" --box fixture --source all \
+  --tail 100 --json >"$tmp/all-query.jsonl"
+jq -e 'select(.source == "service-healthy" and .message == "healthy-log")' \
+  "$tmp/all-query.jsonl" >/dev/null
+
+# Removing execute permission removes the desired service and synchronously
+# stops its capture and foreground child. Restoring it starts one fresh copy.
+chmod 0600 "$definitions/healthy"
+"$services" reconcile
+wait_for_absent "$runtime/healthy.state"
+wait_for_count 1 "$events/healthy.terms"
+chmod 0700 "$definitions/healthy"
+"$services" reconcile
+wait_for_count 2 "$events/healthy.starts"
+
+# Direct capture termination reaches the service process group, including its
+# foreground shell trap. Reconciliation replaces the now-stale runtime record.
+capture_pid="$(cut -f1 "$runtime/healthy.state")"
+kill -TERM "$capture_pid"
+wait_for_count 2 "$events/healthy.terms"
+for _ in {1..100}; do
+  kill -0 "$capture_pid" 2>/dev/null || break
+  [[ "$(ps -o stat= -p "$capture_pid" 2>/dev/null)" == Z* ]] && break
+  sleep 0.02
+done
+"$services" reconcile
+wait_for_count 3 "$events/healthy.starts"
+
+# hb owns the durable lifecycle gate: pause waits for every service, a
+# quiesced reload cannot restart them, resume starts them, and gateway-disable
+# remains independent.
+(
+  export HB_DATA="$data" HOME="$home"
+  # shellcheck disable=SC1090
+  source "$PROJECT_ROOT/guest/hb"
+  _stop_gateway() { return 0; }
+  _stop_executor() { return 0; }
+  _executor_up() { return 0; }
+  _start_gateway() { return 0; }
+  pause >/dev/null
+)
+wait_for_absent "$runtime/healthy.state"
+wait_for_absent "$runtime/broken.state"
+healthy_before_resume="$(wc -l <"$events/healthy.starts" | tr -d ' ')"
+"$services" reconcile
+sleep 0.1
+[[ "$(wc -l <"$events/healthy.starts" | tr -d ' ')" == "$healthy_before_resume" ]]
+(
+  export HB_DATA="$data" HOME="$home"
+  # shellcheck disable=SC1090
+  source "$PROJECT_ROOT/guest/hb"
+  _executor_up() { return 0; }
+  _start_gateway() { return 0; }
+  resume >/dev/null
+)
+wait_for_count $((healthy_before_resume + 1)) "$events/healthy.starts"
+healthy_capture="$(cut -f1 "$runtime/healthy.state")"
+(
+  export HB_DATA="$data" HOME="$home"
+  # shellcheck disable=SC1090
+  source "$PROJECT_ROOT/guest/hb"
+  _stop_gateway() { return 0; }
+  gateway_disable >/dev/null
+)
+kill -0 "$healthy_capture"
+
+# Deletion stops a service, while the other definition remains independently
+# supervised. Finish by proving stop-all leaves no managed capture alive.
+rm -f "$definitions/broken"
+"$services" reconcile
+wait_for_absent "$runtime/broken.state"
+"$services" stop-all
+wait_for_absent "$runtime/healthy.state"
+
+# An empty status includes the exact installation path instead of presenting
+# a bare table with no next step.
+empty_config="$tmp/empty-config"
+TX9_SERVICES_CONFIG_ROOT="$empty_config" \
+  TX9_SERVICES_STATE_DIR="$tmp/empty-runtime" \
+  TX9_SERVICES_LOG_DIR="$tmp/empty-logs" \
+  "$services" status >"$tmp/empty.status"
+grep -Fq "No custom services configured. Add executable files to $empty_config/services.d" \
+  "$tmp/empty.status"
+
+# A service started by a separate reload/session is outside the workload's
+# process group. TERM on the workload must still synchronously stop it via the
+# runtime identity record and preserve signal-derived exit status.
+term_data="$tmp/term-data"
+term_home="$term_data/home/agent"
+term_config="$term_home/.config/hermes-box"
+term_definitions="$term_config/services.d"
+term_runtime="$tmp/term-runtime"
+term_logs="$term_data/logs"
+term_events="$tmp/term-events"
+mkdir -p "$term_definitions" "$term_runtime" "$term_logs" "$term_events" "$tmp/term-bin"
+cat >"$term_definitions/separate" <<'EOF'
+#!/usr/bin/env bash
+printf 'started\n' >>"$SERVICE_TERM_EVENTS/starts"
+trap 'printf "terminated\n" >>"$SERVICE_TERM_EVENTS/terms"; exit 0' TERM INT
+while :; do sleep 1; done
+EOF
+chmod 0700 "$term_definitions/separate"
+cat >"$tmp/term-bin/hb" <<'EOF'
+#!/usr/bin/env bash
+exit 0
+EOF
+chmod 0700 "$tmp/term-bin/hb"
+setsid env HOME="$term_home" SERVICE_TERM_EVENTS="$term_events" \
+  TX9_SERVICES_CONFIG_ROOT="$term_config" \
+  TX9_SERVICES_STATE_DIR="$term_runtime" \
+  TX9_SERVICES_LOG_DIR="$term_logs" \
+  TX9_SERVICES_LOG_HELPER="$PROJECT_ROOT/guest/tx9-logs" \
+  TX9_SERVICES_RESTART_DELAY=0.2 TX9_SERVICES_STOP_TIMEOUT=2 \
+  "$services" reconcile >/dev/null
+wait_for_pattern started "$term_events/starts"
+separate_capture="$(cut -f1 "$term_runtime/separate.state")"
+pids+=("$separate_capture")
+setsid env HOME="$term_home" PATH="$tmp/term-bin:$PATH" \
+  TX9_SERVICES_STATE_DIR="$term_runtime" \
+  TX9_SERVICES_LOG_HELPER="$PROJECT_ROOT/guest/tx9-logs" \
+  TX9_SERVICES_RESTART_DELAY=0.2 TX9_SERVICES_STOP_TIMEOUT=2 \
+  HB_WORKLOAD_LOGS="$term_logs" HB_BOX_ENV="$tmp/missing-box.env" \
+  "$PROJECT_ROOT/guest/hb-workload" 4788 0 \
+  >"$tmp/term-workload.stdout" 2>"$tmp/term-workload.stderr" &
+workload_pid=$!
+pids+=("$workload_pid")
+for _ in {1..100}; do
+  kill -0 "$workload_pid" 2>/dev/null && break
+  sleep 0.02
+done
+separate_pgid="$(ps -o pgid= -p "$separate_capture" | tr -d ' ')"
+workload_pgid="$(ps -o pgid= -p "$workload_pid" | tr -d ' ')"
+[[ -n "$separate_pgid" && -n "$workload_pgid" && "$separate_pgid" != "$workload_pgid" ]]
+kill -TERM "$workload_pid"
+set +e
+wait "$workload_pid"
+workload_status=$?
+set -e
+[[ "$workload_status" == 143 ]]
+wait_for_pattern terminated "$term_events/terms"
+wait_for_absent "$term_runtime/separate.state"
+
+echo "custom service regression checks passed"
diff --git a/tests/static.sh b/tests/static.sh
index 6d8f1be..fa1c112 100755
--- a/tests/static.sh
+++ b/tests/static.sh
@@ -12,6 +12,7 @@ regression_files=(tests/regressions-*.sh)
 files=(
   guest/hb
   guest/hb-workload
+  guest/tx9-services
   guest/lib-mcp.sh
   guest/profile.sh
   guest/agent-bash-profile.sh
@@ -26,7 +27,7 @@ files=(
 bash -n "${files[@]}"
 python3 -c 'compile(open("guest/hermes-state", encoding="utf-8").read(), "guest/hermes-state", "exec")'
 python3 -c 'compile(open("guest/tx9-logs", encoding="utf-8").read(), "guest/tx9-logs", "exec")'
-for file in guest/hb guest/hb-workload guest/hermes-state guest/tx9-logs provision/provision.sh tests/hermes-state.sh "${regression_files[@]}"; do
+for file in guest/hb guest/hb-workload guest/tx9-services guest/hermes-state guest/tx9-logs provision/provision.sh tests/hermes-state.sh "${regression_files[@]}"; do
   [[ -x "$file" ]] || { echo "not executable: $file" >&2; exit 1; }
 done
 
@@ -66,6 +67,13 @@ grep -q '_refresh_targets' guest/hb-workload
 grep -q '\. "$BOX_ENV"' guest/hb-workload
 grep -q '_stop_executor_bridge' guest/hb-workload
 
+# --- guest/tx9-services invariants ---------------------------------------
+grep -q 'services.d' guest/tx9-services
+grep -q -- '--source "service-\$name"' guest/tx9-services
+grep -q -- '--restart-delay' guest/tx9-services
+grep -q '_services_reconcile' guest/hb
+grep -q '_services_stop' guest/hb
+
 # --- provisioning / config invariants ------------------------------------
 grep -q 'tools) tools_only ;;' provision/provision.sh
 grep -q 'assets) assets_only ;;' provision/provision.sh
@@ -92,6 +100,7 @@ grep -q 'tx9-logs capture' docker/entrypoint.sh
 grep -q -- '--source agent' docker/entrypoint.sh
 grep -q 'tx9-logs capture --source executor' docker/executor-entrypoint.sh
 grep -q 'tx9-logs.*OPT/bin/tx9-logs' provision/provision.sh
+grep -q 'tx9-services.*OPT/bin/tx9-services' provision/provision.sh
 grep -Fq '[ -t 0 ] && [ -t 1 ]' guest/agent-bash-profile.sh
 
 grep -qi 'networking is always enabled' README.md

From 824bd1531e0e7300d7cbd33e1a6e26b5fb333cbc Mon Sep 17 00:00:00 2001
From: Ben Davis <45952064+bmdavis419@users.noreply.github.com>
Date: Wed, 22 Jul 2026 13:33:56 -0700
Subject: [PATCH 2/7] fix: preserve custom service runtime identity

---
 guest/tx9-services            | 47 +++++++++++------
 tests/regressions-services.sh | 98 ++++++++++++++++++++++++++++++++++-
 2 files changed, 127 insertions(+), 18 deletions(-)

diff --git a/guest/tx9-services b/guest/tx9-services
index cbd8afe..3a5e947 100755
--- a/guest/tx9-services
+++ b/guest/tx9-services
@@ -17,6 +17,7 @@ _safe_name() { [[ "$1" =~ ^[a-z0-9][a-z0-9_-]{0,62}$ ]]; }
 
 _ensure_dirs() {
   mkdir -p "$DEFINITIONS" "$STATE_DIR" "$LOG_DIR" || return 1
+  LOG_DIR="$(cd -- "$LOG_DIR" && pwd -P)" || return 1
   chmod 0700 "$DEFINITIONS" 2>/dev/null || true
   chmod 0700 "$STATE_DIR" 2>/dev/null || true
 }
@@ -79,12 +80,12 @@ _process_start_time() {
 
 _read_state() {
   local state_file="$1"
-  IFS=$'\t' read -r STATE_PID STATE_START STATE_FINGERPRINT STATE_HELPER STATE_DELAY 2>/dev/null <"$state_file" || return 1
-  [[ "$STATE_PID" =~ ^[1-9][0-9]*$ && "$STATE_START" =~ ^[0-9]+$ && -n "$STATE_FINGERPRINT" && "$STATE_HELPER" == /* && -n "$STATE_DELAY" ]]
+  IFS=$'\t' read -r STATE_PID STATE_START STATE_FINGERPRINT STATE_HELPER STATE_LOG_DIR STATE_DELAY 2>/dev/null <"$state_file" || return 1
+  [[ "$STATE_PID" =~ ^[1-9][0-9]*$ && "$STATE_START" =~ ^[0-9]+$ && -n "$STATE_FINGERPRINT" && "$STATE_HELPER" == /* && "$STATE_LOG_DIR" == /* && -n "$STATE_DELAY" ]]
 }
 
 _capture_argv_matches() {
-  local pid="$1" name="$2" definition="$3" helper="$4" delay="$5" index
+  local pid="$1" name="$2" definition="$3" helper="$4" log_dir="$5" delay="$6" index
   local -a argv expected
   mapfile -d '' -t argv <"/proc/$pid/cmdline" 2>/dev/null || return 1
   ((${#argv[@]} > 0)) || return 1
@@ -96,7 +97,7 @@ _capture_argv_matches() {
     return 1
   fi
   expected=(
-    "$helper" capture --source "service-$name" --log-dir "$LOG_DIR"
+    "$helper" capture --source "service-$name" --log-dir "$log_dir"
     --restart-delay "$delay" -- "$definition"
   )
   ((${#argv[@]} - index == ${#expected[@]})) || return 1
@@ -113,21 +114,23 @@ _state_process_matches() {
   [[ "$current_start" == "$STATE_START" ]] || return 1
   current_uid="$(stat -c '%u' -- "/proc/$STATE_PID" 2>/dev/null)" || return 1
   [[ "$current_uid" == "$(id -u)" ]] || return 1
-  _capture_argv_matches "$STATE_PID" "$name" "$definition" "$STATE_HELPER" "$STATE_DELAY"
+  _capture_argv_matches "$STATE_PID" "$name" "$definition" \
+    "$STATE_HELPER" "$STATE_LOG_DIR" "$STATE_DELAY"
 }
 
 _write_state() {
-  local state_file="$1" pid="$2" start="$3" fingerprint="$4" helper="$5" delay="$6" tmp
+  local state_file="$1" pid="$2" start="$3" fingerprint="$4" helper="$5" log_dir="$6" delay="$7" tmp
   tmp="$state_file.tmp.$$"
   umask 077
-  printf '%s\t%s\t%s\t%s\t%s\n' "$pid" "$start" "$fingerprint" "$helper" "$delay" >"$tmp" || return 1
+  printf '%s\t%s\t%s\t%s\t%s\t%s\n' \
+    "$pid" "$start" "$fingerprint" "$helper" "$log_dir" "$delay" >"$tmp" || return 1
   mv "$tmp" "$state_file"
 }
 
 _start_one() {
-  local name="$1" definition="$2" fingerprint="$3" helper pid start attempt state_file
+  local name="$1" definition="$2" fingerprint="$3" helper="$4" pid start attempt state_file
   state_file="$STATE_DIR/$name.state"
-  helper="$(_log_helper)" || return 1
+  [[ -n "$helper" ]] || return 1
   (
     # flock locks are tied to open file descriptions and survive fork. The
     # long-lived capture must never inherit the reconciliation lock.
@@ -141,7 +144,8 @@ _start_one() {
     echo "tx9-services: $name failed before supervision started" >&2
     return 1
   }
-  _write_state "$state_file" "$pid" "$start" "$fingerprint" "$helper" "$RESTART_DELAY" || {
+  _write_state "$state_file" "$pid" "$start" "$fingerprint" \
+    "$helper" "$LOG_DIR" "$RESTART_DELAY" || {
     kill -TERM "$pid" 2>/dev/null || true
     wait "$pid" 2>/dev/null || true
     return 1
@@ -251,7 +255,7 @@ _definition_reason() {
 }
 
 _reconcile_locked() {
-  local path name fingerprint state_file failed=0
+  local path name fingerprint state_file current_helper='' failed=0
   local -A desired=()
   while IFS= read -r -d '' path; do
     name="${path##*/}"
@@ -266,6 +270,10 @@ _reconcile_locked() {
     desired["$name"]="$fingerprint"
   done < <(find "$DEFINITIONS" -mindepth 1 -maxdepth 1 -print0 2>/dev/null)
 
+  if ((${#desired[@]} > 0)); then
+    current_helper="$(_log_helper)" || failed=1
+  fi
+
   shopt -s nullglob
   for state_file in "$STATE_DIR"/*.state; do
     name="${state_file##*/}"
@@ -276,7 +284,9 @@ _reconcile_locked() {
       continue
     fi
     if _state_process_matches "$state_file" "$name" "$path" &&
-      _read_state "$state_file" && [[ "$STATE_FINGERPRINT" == "${desired[$name]}" ]]; then
+      [[ "$STATE_FINGERPRINT" == "${desired[$name]}" &&
+        "$STATE_HELPER" == "$current_helper" && "$STATE_LOG_DIR" == "$LOG_DIR" &&
+        "$STATE_DELAY" == "$RESTART_DELAY" ]]; then
       continue
     fi
     _stop_one "$name" "$path" || failed=1
@@ -288,10 +298,12 @@ _reconcile_locked() {
     state_file="$STATE_DIR/$name.state"
     path="$DEFINITIONS/$name"
     if _state_process_matches "$state_file" "$name" "$path" &&
-      _read_state "$state_file" && [[ "$STATE_FINGERPRINT" == "${desired[$name]}" ]]; then
+      [[ "$STATE_FINGERPRINT" == "${desired[$name]}" &&
+        "$STATE_HELPER" == "$current_helper" && "$STATE_LOG_DIR" == "$LOG_DIR" &&
+        "$STATE_DELAY" == "$RESTART_DELAY" ]]; then
       continue
     fi
-    _start_one "$name" "$path" "${desired[$name]}" || failed=1
+    _start_one "$name" "$path" "${desired[$name]}" "$current_helper" || failed=1
   done
   return "$failed"
 }
@@ -326,8 +338,9 @@ stop_all() {
 }
 
 _status_locked() {
-  local path name reason fingerprint state_file rows=0
+  local path name reason fingerprint state_file current_helper rows=0
   local -A seen=()
+  current_helper="$(_log_helper 2>/dev/null || true)"
   printf 'NAME\tSTATUS\tDEFINITION\n'
   while IFS= read -r -d '' path; do
     name="${path##*/}"
@@ -341,7 +354,9 @@ _status_locked() {
     state_file="$STATE_DIR/$name.state"
     fingerprint="$(_definition_fingerprint "$path" 2>/dev/null || true)"
     if _state_process_matches "$state_file" "$name" "$path" && _read_state "$state_file"; then
-      if [[ "$STATE_FINGERPRINT" == "$fingerprint" ]]; then
+      if [[ "$STATE_FINGERPRINT" == "$fingerprint" &&
+        "$STATE_HELPER" == "$current_helper" && "$STATE_LOG_DIR" == "$LOG_DIR" &&
+        "$STATE_DELAY" == "$RESTART_DELAY" ]]; then
         if _supervised_child_running "$STATE_PID"; then
           printf '%s\trunning\t%s\n' "$name" "$path"
         else
diff --git a/tests/regressions-services.sh b/tests/regressions-services.sh
index 74bc123..d462d47 100755
--- a/tests/regressions-services.sh
+++ b/tests/regressions-services.sh
@@ -85,8 +85,9 @@ sleep 30 &
 decoy_pid=$!
 pids+=("$decoy_pid")
 decoy_start="$(awk '{print $22}' "/proc/$decoy_pid/stat")"
-printf '%s\t%s\tstale\t%s\t%s\n' "$decoy_pid" "$decoy_start" \
-  "$TX9_SERVICES_LOG_HELPER" "$TX9_SERVICES_RESTART_DELAY" \
+printf '%s\t%s\tstale\t%s\t%s\t%s\n' "$decoy_pid" "$decoy_start" \
+  "$TX9_SERVICES_LOG_HELPER" "$TX9_SERVICES_LOG_DIR" \
+  "$TX9_SERVICES_RESTART_DELAY" \
   >"$runtime/ghost.state"
 "$services" reconcile >/dev/null 2>&1
 kill -0 "$decoy_pid"
@@ -114,6 +115,99 @@ done
 wait
 [[ "$(wc -l <"$events/healthy.starts" | tr -d ' ')" == 1 ]]
 
+# Runtime logging configuration is part of supervisor identity. Status reports
+# drift in each setting, and reconcile uses the stored settings to stop the old
+# process before starting exactly one replacement with the current settings.
+drift_config="$tmp/drift-config"
+drift_definitions="$drift_config/services.d"
+drift_runtime="$tmp/drift-runtime"
+drift_logs_a="$tmp/drift-logs-a"
+drift_logs_b="$tmp/drift-logs-b"
+drift_events="$tmp/drift-events"
+drift_helper_b="$tmp/tx9-logs-b"
+mkdir -p "$drift_definitions" "$drift_runtime" "$drift_logs_a" \
+  "$drift_logs_b" "$drift_events"
+cp "$PROJECT_ROOT/guest/tx9-logs" "$drift_helper_b"
+chmod 0700 "$drift_helper_b"
+cat >"$drift_definitions/drift" <<'EOF'
+#!/usr/bin/env bash
+printf 'start %s\n' "$$" >>"$SERVICE_DRIFT_EVENTS/starts"
+printf 'drift-log %s\n' "$$"
+trap 'printf "term %s\n" "$$" >>"$SERVICE_DRIFT_EVENTS/terms"; exit 0' TERM INT
+while :; do sleep 1; done
+EOF
+chmod 0700 "$drift_definitions/drift"
+
+drift_services() {
+  local log_dir="$1" helper="$2" delay="$3"
+  shift 3
+  env SERVICE_DRIFT_EVENTS="$drift_events" \
+    TX9_SERVICES_CONFIG_ROOT="$drift_config" \
+    TX9_SERVICES_STATE_DIR="$drift_runtime" \
+    TX9_SERVICES_LOG_DIR="$log_dir" \
+    TX9_SERVICES_LOG_HELPER="$helper" \
+    TX9_SERVICES_RESTART_DELAY="$delay" \
+    TX9_SERVICES_STOP_TIMEOUT=2 \
+    "$services" "$@"
+}
+
+drift_services "$drift_logs_a" "$PROJECT_ROOT/guest/tx9-logs" 0.2 reconcile >/dev/null
+wait_for_count 1 "$drift_events/starts"
+IFS=$'\t' read -r old_capture _old_start _old_fingerprint old_helper \
+  old_log_dir old_delay <"$drift_runtime/drift.state"
+pids+=("$old_capture")
+[[ "$old_helper" == "$PROJECT_ROOT/guest/tx9-logs" ]]
+[[ "$old_log_dir" == "$drift_logs_a" ]]
+[[ "$old_delay" == 0.2 ]]
+
+drift_services "$drift_logs_b" "$PROJECT_ROOT/guest/tx9-logs" 0.2 status \
+  >"$tmp/drift-log-dir.status"
+grep -Fq $'drift\treload pending' "$tmp/drift-log-dir.status"
+drift_services "$drift_logs_a" "$drift_helper_b" 0.2 status \
+  >"$tmp/drift-helper.status"
+grep -Fq $'drift\treload pending' "$tmp/drift-helper.status"
+drift_services "$drift_logs_a" "$PROJECT_ROOT/guest/tx9-logs" 0.7 status \
+  >"$tmp/drift-delay.status"
+grep -Fq $'drift\treload pending' "$tmp/drift-delay.status"
+
+drift_services "$drift_logs_b" "$drift_helper_b" 0.7 reconcile >/dev/null
+wait_for_count 1 "$drift_events/terms"
+wait_for_count 2 "$drift_events/starts"
+wait_for_pattern drift-log "$drift_logs_b/service-drift.jsonl"
+IFS=$'\t' read -r new_capture _new_start _new_fingerprint new_helper \
+  new_log_dir new_delay <"$drift_runtime/drift.state"
+pids+=("$new_capture")
+[[ "$new_capture" != "$old_capture" ]]
+[[ "$new_helper" == "$drift_helper_b" ]]
+[[ "$new_log_dir" == "$drift_logs_b" ]]
+[[ "$new_delay" == 0.7 ]]
+old_service="$(awk 'NR == 1 { print $2 }' "$drift_events/starts")"
+new_service="$(awk 'NR == 2 { print $2 }' "$drift_events/starts")"
+for _ in {1..100}; do
+  if ! kill -0 "$old_capture" 2>/dev/null ||
+    [[ "$(ps -o stat= -p "$old_capture" 2>/dev/null)" == Z* ]]; then
+    break
+  fi
+  sleep 0.02
+done
+if kill -0 "$old_capture" 2>/dev/null &&
+  [[ "$(ps -o stat= -p "$old_capture" 2>/dev/null)" != Z* ]]; then
+  echo "old service supervisor survived runtime-config reconciliation" >&2
+  exit 1
+fi
+if kill -0 "$old_service" 2>/dev/null &&
+  [[ "$(ps -o stat= -p "$old_service" 2>/dev/null)" != Z* ]]; then
+  echo "old custom service survived runtime-config reconciliation" >&2
+  exit 1
+fi
+kill -0 "$new_capture"
+kill -0 "$new_service"
+[[ "$(wc -l <"$drift_events/starts" | tr -d ' ')" == 2 ]]
+[[ "$(wc -l <"$drift_events/terms" | tr -d ' ')" == 1 ]]
+drift_services "$drift_logs_b" "$drift_helper_b" 0.7 stop-all >/dev/null
+wait_for_count 2 "$drift_events/terms"
+wait_for_absent "$drift_runtime/drift.state"
+
 # Definition identity includes the content digest, not just whole-second stat
 # metadata. Rewrite a same-size script in place, restore its original mtime,
 # and require immediate replacement of the old foreground child.

From 8d9971e5e283ead4221fb8209bac33e9094d802d Mon Sep 17 00:00:00 2001
From: Ben Davis <45952064+bmdavis419@users.noreply.github.com>
Date: Wed, 22 Jul 2026 13:42:29 -0700
Subject: [PATCH 3/7] fix: bind service stops to process identity

---
 guest/tx9-services            | 79 +++++++++++++++++++++++++-------
 tests/regressions-services.sh | 86 +++++++++++++++++++++++++++++++++++
 2 files changed, 149 insertions(+), 16 deletions(-)

diff --git a/guest/tx9-services b/guest/tx9-services
index 3a5e947..fce3bc6 100755
--- a/guest/tx9-services
+++ b/guest/tx9-services
@@ -183,6 +183,21 @@ _pid_alive() {
   [[ "$state" != Z* ]]
 }
 
+_pid_identity_alive() {
+  local pid="$1" expected_start="$2" current_start
+  _pid_alive "$pid" || return 1
+  current_start="$(_process_start_time "$pid")" || return 1
+  [[ "$current_start" == "$expected_start" ]]
+}
+
+_process_group_identity_alive() {
+  local leader="$1" expected_start="$2" expected_group="$3" current_group
+  _pid_identity_alive "$leader" "$expected_start" || return 1
+  current_group="$(ps -o pgid= -p "$leader" 2>/dev/null)" || return 1
+  current_group="${current_group//[[:space:]]/}"
+  [[ "$current_group" == "$expected_group" ]]
+}
+
 _supervised_child_running() {
   local capture_pid="$1" child start current_start
   local -a children
@@ -203,37 +218,69 @@ _supervised_child_running() {
 }
 
 _stop_one() {
-  local name="$1" definition="$2" attempt failed=0
+  local name="$1" definition="$2" attempt child start group index alive failed=0
   local state_file="$STATE_DIR/$name.state"
   local -a child_groups=()
+  local -a child_starts=()
   if ! _state_process_matches "$state_file" "$name" "$definition"; then
     rm -f "$state_file"
     return 0
   fi
-  mapfile -t child_groups < <(_direct_children "$STATE_PID")
+
+  while IFS= read -r child; do
+    start="$(_process_start_time "$child")" || continue
+    group="$(ps -o pgid= -p "$child" 2>/dev/null)" || continue
+    group="${group//[[:space:]]/}"
+    [[ "$group" == "$child" ]] || continue
+    _process_group_identity_alive "$child" "$start" "$group" || continue
+    child_groups+=("$group")
+    child_starts+=("$start")
+  done < <(_direct_children "$STATE_PID")
+
+  if ! _pid_identity_alive "$STATE_PID" "$STATE_START"; then
+    rm -f "$state_file"
+    return 0
+  fi
   kill -TERM "$STATE_PID" 2>/dev/null || true
   for ((attempt = 0; attempt < STOP_TIMEOUT * 10; attempt++)); do
-    _pid_alive "$STATE_PID" || break
+    alive=0
+    _pid_identity_alive "$STATE_PID" "$STATE_START" && alive=1
+    for ((index = 0; index < ${#child_groups[@]}; index++)); do
+      _process_group_identity_alive \
+        "${child_groups[index]}" "${child_starts[index]}" "${child_groups[index]}" && alive=1
+    done
+    [[ "$alive" == 1 ]] || break
     sleep 0.1
   done
-  if _pid_alive "$STATE_PID"; then
-    local group
-    for group in "${child_groups[@]}"; do
-      kill -KILL -- "-$group" 2>/dev/null || true
-    done
+
+  for ((index = 0; index < ${#child_groups[@]}; index++)); do
+    if _process_group_identity_alive \
+      "${child_groups[index]}" "${child_starts[index]}" "${child_groups[index]}"; then
+      kill -KILL -- "-${child_groups[index]}" 2>/dev/null || true
+    fi
+  done
+  if _pid_identity_alive "$STATE_PID" "$STATE_START"; then
     kill -KILL "$STATE_PID" 2>/dev/null || true
-    for ((attempt = 0; attempt < 20; attempt++)); do
-      _pid_alive "$STATE_PID" || break
-      sleep 0.05
-    done
   fi
-  if _pid_alive "$STATE_PID"; then
+
+  for ((attempt = 0; attempt < 20; attempt++)); do
+    alive=0
+    _pid_identity_alive "$STATE_PID" "$STATE_START" && alive=1
+    for ((index = 0; index < ${#child_groups[@]}; index++)); do
+      _process_group_identity_alive \
+        "${child_groups[index]}" "${child_starts[index]}" "${child_groups[index]}" && alive=1
+    done
+    [[ "$alive" == 1 ]] || break
+    sleep 0.05
+  done
+
+  if _pid_identity_alive "$STATE_PID" "$STATE_START"; then
     echo "tx9-services: $name capture survived stop" >&2
     failed=1
   fi
-  local group
-  for group in "${child_groups[@]}"; do
-    if kill -0 -- "-$group" 2>/dev/null; then
+  for ((index = 0; index < ${#child_groups[@]}; index++)); do
+    if _process_group_identity_alive \
+      "${child_groups[index]}" "${child_starts[index]}" "${child_groups[index]}"; then
       echo "tx9-services: $name process group survived stop" >&2
       failed=1
     fi
diff --git a/tests/regressions-services.sh b/tests/regressions-services.sh
index d462d47..e1d1828 100755
--- a/tests/regressions-services.sh
+++ b/tests/regressions-services.sh
@@ -95,6 +95,92 @@ kill -0 "$decoy_pid"
 kill "$decoy_pid"
 wait "$decoy_pid" 2>/dev/null || true
 
+# Stop escalation remains bound to the exact capture and service identities
+# observed before TERM. Reused numeric PIDs/PGIDs are never sent SIGKILL.
+(
+  # shellcheck disable=SC1090
+  source "$services"
+  STATE_DIR="$tmp/reused-stop-runtime"
+  STOP_TIMEOUT=1
+  mkdir -p "$STATE_DIR"
+  touch "$STATE_DIR/reused.state"
+  signal_log="$tmp/reused-stop.signals"
+  capture_reads="$tmp/reused-capture.reads"
+  child_reads="$tmp/reused-child.reads"
+
+  _state_process_matches() {
+    STATE_PID=4242
+    STATE_START=100
+    return 0
+  }
+  _direct_children() { printf '6262\n'; }
+  _pid_alive() { return 0; }
+  _process_start_time() {
+    local pid="$1" reads_file count=0
+    if [[ "$pid" == 4242 ]]; then
+      reads_file="$capture_reads"
+      [[ -f "$reads_file" ]] && count="$(wc -l <"$reads_file")"
+      printf 'read\n' >>"$reads_file"
+      if [[ "$count" == 0 ]]; then printf '100\n'; else printf '200\n'; fi
+      return
+    fi
+    reads_file="$child_reads"
+    [[ -f "$reads_file" ]] && count="$(wc -l <"$reads_file")"
+    printf 'read\n' >>"$reads_file"
+    if ((count < 2)); then printf '300\n'; else printf '400\n'; fi
+  }
+  ps() { printf '6262\n'; }
+  kill() { printf '%s\n' "$*" >>"$signal_log"; }
+  sleep() { :; }
+
+  _stop_one reused "$definitions/reused" >/dev/null
+  grep -Fxq -- '-TERM 4242' "$signal_log"
+  if grep -Fq -- '-KILL' "$signal_log"; then
+    echo "stop sent SIGKILL after capture or service PID identity changed" >&2
+    exit 1
+  fi
+  [[ ! -e "$STATE_DIR/reused.state" ]]
+)
+
+# A genuinely stuck capture and same-identity service group are both
+# escalated. If they survive SIGKILL, stop fails and retains state for retry.
+(
+  # shellcheck disable=SC1090
+  source "$services"
+  STATE_DIR="$tmp/stuck-stop-runtime"
+  STOP_TIMEOUT=0
+  mkdir -p "$STATE_DIR"
+  touch "$STATE_DIR/stuck.state"
+  signal_log="$tmp/stuck-stop.signals"
+
+  _state_process_matches() {
+    STATE_PID=5252
+    STATE_START=500
+    return 0
+  }
+  _direct_children() { printf '7272\n'; }
+  _pid_alive() { return 0; }
+  _process_start_time() {
+    if [[ "$1" == 5252 ]]; then printf '500\n'; else printf '700\n'; fi
+  }
+  ps() { printf '7272\n'; }
+  kill() { printf '%s\n' "$*" >>"$signal_log"; }
+  sleep() { :; }
+
+  if _stop_one stuck "$definitions/stuck" \
+    >"$tmp/stuck-stop.stdout" 2>"$tmp/stuck-stop.stderr"; then
+    echo "stop succeeded after same-identity processes survived SIGKILL" >&2
+    exit 1
+  fi
+  : "$STOP_TIMEOUT" "$STATE_PID" "$STATE_START"
+  grep -Fxq -- '-TERM 5252' "$signal_log"
+  grep -Fxq -- '-KILL -- -7272' "$signal_log"
+  grep -Fxq -- '-KILL 5252' "$signal_log"
+  grep -Fq 'capture survived stop' "$tmp/stuck-stop.stderr"
+  grep -Fq 'process group survived stop' "$tmp/stuck-stop.stderr"
+  [[ -e "$STATE_DIR/stuck.state" ]]
+)
+
 "$services" status >"$tmp/services.status"
 grep -Fq $'healthy\trunning' "$tmp/services.status"
 grep -Fq $'broken\trestarting' "$tmp/services.status"

From bdec5f5d40f471e70093d4a9321543b0c3716552 Mon Sep 17 00:00:00 2001
From: Ben Davis <45952064+bmdavis419@users.noreply.github.com>
Date: Wed, 22 Jul 2026 13:48:49 -0700
Subject: [PATCH 4/7] fix: preserve services when logging is unavailable

---
 guest/tx9-services            | 12 +++++-
 tests/regressions-services.sh | 78 +++++++++++++++++++++++++++++++++++
 2 files changed, 88 insertions(+), 2 deletions(-)

diff --git a/guest/tx9-services b/guest/tx9-services
index fce3bc6..74e753f 100755
--- a/guest/tx9-services
+++ b/guest/tx9-services
@@ -302,7 +302,7 @@ _definition_reason() {
 }
 
 _reconcile_locked() {
-  local path name fingerprint state_file current_helper='' failed=0
+  local path name fingerprint state_file current_helper='' helper_available=1 failed=0
   local -A desired=()
   while IFS= read -r -d '' path; do
     name="${path##*/}"
@@ -318,7 +318,10 @@ _reconcile_locked() {
   done < <(find "$DEFINITIONS" -mindepth 1 -maxdepth 1 -print0 2>/dev/null)
 
   if ((${#desired[@]} > 0)); then
-    current_helper="$(_log_helper)" || failed=1
+    if ! current_helper="$(_log_helper)"; then
+      helper_available=0
+      failed=1
+    fi
   fi
 
   shopt -s nullglob
@@ -330,6 +333,10 @@ _reconcile_locked() {
       _stop_one "$name" "$path" || failed=1
       continue
     fi
+    # A helper is required to replace a configured supervisor. Preserve the
+    # existing launch when helper resolution is transiently unavailable;
+    # otherwise stopping it here would turn a recoverable error into downtime.
+    [[ "$helper_available" == 1 ]] || continue
     if _state_process_matches "$state_file" "$name" "$path" &&
       [[ "$STATE_FINGERPRINT" == "${desired[$name]}" &&
         "$STATE_HELPER" == "$current_helper" && "$STATE_LOG_DIR" == "$LOG_DIR" &&
@@ -344,6 +351,7 @@ _reconcile_locked() {
   for name in "${!desired[@]}"; do
     state_file="$STATE_DIR/$name.state"
     path="$DEFINITIONS/$name"
+    [[ "$helper_available" == 1 ]] || continue
     if _state_process_matches "$state_file" "$name" "$path" &&
       [[ "$STATE_FINGERPRINT" == "${desired[$name]}" &&
         "$STATE_HELPER" == "$current_helper" && "$STATE_LOG_DIR" == "$LOG_DIR" &&
diff --git a/tests/regressions-services.sh b/tests/regressions-services.sh
index e1d1828..5cf9382 100755
--- a/tests/regressions-services.sh
+++ b/tests/regressions-services.sh
@@ -201,6 +201,84 @@ done
 wait
 [[ "$(wc -l <"$events/healthy.starts" | tr -d ' ')" == 1 ]]
 
+# A transiently missing current helper cannot replace configured services, so
+# reconcile fails without stopping or duplicating their existing supervisors.
+# Removed definitions and explicit stop-all still use stored identity to stop.
+helper_failure_config="$tmp/helper-failure-config"
+helper_failure_definitions="$helper_failure_config/services.d"
+helper_failure_runtime="$tmp/helper-failure-runtime"
+helper_failure_logs="$tmp/helper-failure-logs"
+helper_failure_events="$tmp/helper-failure-events"
+missing_helper="$tmp/missing-tx9-logs"
+mkdir -p "$helper_failure_definitions" "$helper_failure_runtime" \
+  "$helper_failure_logs" "$helper_failure_events"
+cat >"$helper_failure_definitions/keep" <<'EOF'
+#!/usr/bin/env bash
+name="${0##*/}"
+printf 'start %s\n' "$$" >>"$HELPER_FAILURE_EVENTS/$name.starts"
+trap 'printf "term %s\n" "$$" >>"$HELPER_FAILURE_EVENTS/$name.terms"; exit 0' TERM INT
+while :; do sleep 1; done
+EOF
+cp "$helper_failure_definitions/keep" "$helper_failure_definitions/remove"
+chmod 0700 "$helper_failure_definitions/keep" "$helper_failure_definitions/remove"
+
+helper_failure_services() {
+  local helper="$1"
+  shift
+  env HELPER_FAILURE_EVENTS="$helper_failure_events" \
+    TX9_SERVICES_CONFIG_ROOT="$helper_failure_config" \
+    TX9_SERVICES_STATE_DIR="$helper_failure_runtime" \
+    TX9_SERVICES_LOG_DIR="$helper_failure_logs" \
+    TX9_SERVICES_LOG_HELPER="$helper" \
+    TX9_SERVICES_RESTART_DELAY=0.2 \
+    TX9_SERVICES_STOP_TIMEOUT=2 \
+    "$services" "$@"
+}
+
+helper_failure_services "$PROJECT_ROOT/guest/tx9-logs" reconcile >/dev/null
+wait_for_count 1 "$helper_failure_events/keep.starts"
+wait_for_count 1 "$helper_failure_events/remove.starts"
+keep_state="$(cat "$helper_failure_runtime/keep.state")"
+remove_state="$(cat "$helper_failure_runtime/remove.state")"
+keep_capture="${keep_state%%$'\t'*}"
+remove_capture="${remove_state%%$'\t'*}"
+keep_child="$(awk 'NR == 1 { print $2 }' "$helper_failure_events/keep.starts")"
+remove_child="$(awk 'NR == 1 { print $2 }' "$helper_failure_events/remove.starts")"
+pids+=("$keep_capture" "$remove_capture")
+
+if helper_failure_services "$missing_helper" reconcile \
+  >"$tmp/helper-failure.stdout" 2>"$tmp/helper-failure.stderr"; then
+  echo "reconcile succeeded without an available log helper" >&2
+  exit 1
+fi
+grep -Fq 'tx9-logs is unavailable' "$tmp/helper-failure.stderr"
+[[ "$(cat "$helper_failure_runtime/keep.state")" == "$keep_state" ]]
+[[ "$(cat "$helper_failure_runtime/remove.state")" == "$remove_state" ]]
+kill -0 "$keep_capture"
+kill -0 "$remove_capture"
+kill -0 "$keep_child"
+kill -0 "$remove_child"
+[[ "$(wc -l <"$helper_failure_events/keep.starts" | tr -d ' ')" == 1 ]]
+[[ "$(wc -l <"$helper_failure_events/remove.starts" | tr -d ' ')" == 1 ]]
+[[ ! -e "$helper_failure_events/keep.terms" ]]
+[[ ! -e "$helper_failure_events/remove.terms" ]]
+
+rm -f "$helper_failure_definitions/remove"
+if helper_failure_services "$missing_helper" reconcile >/dev/null 2>&1; then
+  echo "reconcile masked helper failure after removing a definition" >&2
+  exit 1
+fi
+wait_for_absent "$helper_failure_runtime/remove.state"
+wait_for_count 1 "$helper_failure_events/remove.terms"
+[[ "$(cat "$helper_failure_runtime/keep.state")" == "$keep_state" ]]
+kill -0 "$keep_capture"
+kill -0 "$keep_child"
+[[ "$(wc -l <"$helper_failure_events/keep.starts" | tr -d ' ')" == 1 ]]
+
+helper_failure_services "$missing_helper" stop-all >/dev/null
+wait_for_absent "$helper_failure_runtime/keep.state"
+wait_for_count 1 "$helper_failure_events/keep.terms"
+
 # Runtime logging configuration is part of supervisor identity. Status reports
 # drift in each setting, and reconcile uses the stored settings to stop the old
 # process before starting exactly one replacement with the current settings.

From c53b2084a49bd62aaefd810ca6b0af56f37dd75f Mon Sep 17 00:00:00 2001
From: Ben Davis <45952064+bmdavis419@users.noreply.github.com>
Date: Wed, 22 Jul 2026 13:58:27 -0700
Subject: [PATCH 5/7] fix: prevent orphaned captured services

---
 guest/tx9-logs                | 71 ++++++++++++++++++++++++++++++++++-
 tests/regressions-logs.sh     | 32 ++++++++++++++++
 tests/regressions-services.sh | 64 +++++++++++++++++++++++++++++++
 3 files changed, 166 insertions(+), 1 deletion(-)

diff --git a/guest/tx9-logs b/guest/tx9-logs
index f512f8e..c25c9e1 100755
--- a/guest/tx9-logs
+++ b/guest/tx9-logs
@@ -10,6 +10,7 @@ from __future__ import annotations
 
 import argparse
 import base64
+import ctypes
 import datetime as dt
 import hashlib
 import heapq
@@ -51,6 +52,8 @@ CAPTURE_READER_DRAIN_SECONDS = 0.5
 CAPTURE_DEDUP_WINDOW_SECONDS = 60.0
 CAPTURE_DEDUP_MAX_REPEATS = 1000
 MAX_RAW_CONTINUATION_LINES = 1000
+PARENT_DEATH_EXEC_ACTION = "__parent-death-exec"
+PR_SET_PDEATHSIG = 1
 # Numeric values below this many epoch seconds (2001-09-09) cannot be event
 # times; loadavg fragments and counters otherwise parse as 1970 timestamps.
 MIN_PLAUSIBLE_EPOCH_SECONDS = 1_000_000_000
@@ -513,6 +516,58 @@ def nonnegative_float(value: str) -> float:
     return parsed
 
 
+def parent_death_exec(arguments: Sequence[str]) -> int:
+    """Arm Linux parent-death SIGKILL, close the setup race, then exec.
+
+    Capture's supported lifecycle contract is the direct foreground child. A
+    command that daemonizes descendants outside that child's process group is
+    outside this guarantee.
+    """
+
+    if len(arguments) < 3 or arguments[1] != "--":
+        print("tx9-logs: invalid internal parent-death invocation", file=sys.stderr)
+        return 126
+    try:
+        expected_parent = positive_int(arguments[0])
+    except argparse.ArgumentTypeError as exc:
+        print(f"tx9-logs: invalid internal parent pid: {exc}", file=sys.stderr)
+        return 126
+    command = list(arguments[2:])
+    if not command:
+        print("tx9-logs: internal parent-death exec requires a command", file=sys.stderr)
+        return 126
+
+    libc = ctypes.CDLL(None, use_errno=True)
+    prctl = libc.prctl
+    prctl.argtypes = [
+        ctypes.c_int,
+        ctypes.c_ulong,
+        ctypes.c_ulong,
+        ctypes.c_ulong,
+        ctypes.c_ulong,
+    ]
+    prctl.restype = ctypes.c_int
+    if prctl(PR_SET_PDEATHSIG, signal.SIGKILL, 0, 0, 0) != 0:
+        error = ctypes.get_errno()
+        print(
+            f"tx9-logs: could not arm parent-death signal: {os.strerror(error)}",
+            file=sys.stderr,
+        )
+        return 126
+
+    # PR_SET_PDEATHSIG is not retroactive. If capture died between Popen's
+    # fork and the prctl call, fail closed instead of execing an orphan.
+    if os.getppid() != expected_parent:
+        os.kill(os.getpid(), signal.SIGKILL)
+        os._exit(128 + signal.SIGKILL)
+
+    try:
+        os.execvpe(command[0], command, os.environ)
+    except OSError as exc:
+        print(f"tx9-logs: could not exec {Path(command[0]).name}: {exc}", file=sys.stderr)
+        return 127 if isinstance(exc, FileNotFoundError) else 126
+
+
 def path_parts_under(root: Path, path: Path) -> tuple[Path, tuple[str, ...]]:
     root = root.absolute()
     path = path.absolute()
@@ -1028,9 +1083,21 @@ class Capture:
         child_env = dict(os.environ)
         child_env["TX9_LOG_MIRROR_STDOUT_FD"] = str(self.mirror_out)
         child_env["TX9_LOG_MIRROR_STDERR_FD"] = str(self.mirror_err)
+        # Re-exec this module as a tiny setup shim instead of using
+        # subprocess.preexec_fn, whose Python callback is unsafe around the
+        # reader threads used by capture. Popen creates the new session first;
+        # the fresh interpreter then arms parent death and execs the command.
+        launch_command = [
+            sys.executable,
+            str(Path(__file__).resolve()),
+            PARENT_DEATH_EXEC_ACTION,
+            str(os.getpid()),
+            "--",
+            *command,
+        ]
         try:
             child = subprocess.Popen(
-                command,
+                launch_command,
                 stdout=subprocess.PIPE,
                 stderr=subprocess.PIPE,
                 env=child_env,
@@ -2076,6 +2143,8 @@ def build_parser() -> argparse.ArgumentParser:
 
 
 def main() -> int:
+    if sys.argv[1:2] == [PARENT_DEATH_EXEC_ACTION]:
+        return parent_death_exec(sys.argv[2:])
     parser = build_parser()
     args = parser.parse_args()
     return int(args.handler(args))
diff --git a/tests/regressions-logs.sh b/tests/regressions-logs.sh
index 42e7bd7..ab9be80 100755
--- a/tests/regressions-logs.sh
+++ b/tests/regressions-logs.sh
@@ -278,6 +278,38 @@ signal_status=$?
 set -e
 [[ "$signal_status" == 143 ]]
 
+# Capture arms its direct foreground child with a Linux parent-death SIGKILL.
+# Abrupt wrapper death therefore cannot leave the supported command orphaned.
+pdeath_dir="$tmp/parent-death"
+pdeath_pid_file="$tmp/parent-death.pid"
+PDEATH_PID_FILE="$pdeath_pid_file" \
+  "$helper" capture --source agent --log-dir "$pdeath_dir" -- \
+    python3 -c 'import os, pathlib, signal; pathlib.Path(os.environ["PDEATH_PID_FILE"]).write_text(str(os.getpid())); signal.pause()' \
+    >/dev/null 2>&1 &
+pdeath_capture_pid=$!
+pids+=("$pdeath_capture_pid")
+wait_for_file "$pdeath_pid_file"
+pdeath_child_pid="$(cat "$pdeath_pid_file")"
+kill -0 "$pdeath_child_pid"
+kill -KILL "$pdeath_capture_pid"
+set +e
+wait "$pdeath_capture_pid" 2>/dev/null
+pdeath_capture_status=$?
+set -e
+[[ "$pdeath_capture_status" == 137 ]]
+for _ in {1..150}; do
+  if ! kill -0 "$pdeath_child_pid" 2>/dev/null ||
+    [[ "$(ps -o stat= -p "$pdeath_child_pid" 2>/dev/null)" == Z* ]]; then
+    break
+  fi
+  sleep 0.02
+done
+if kill -0 "$pdeath_child_pid" 2>/dev/null &&
+  [[ "$(ps -o stat= -p "$pdeath_child_pid" 2>/dev/null)" != Z* ]]; then
+  echo "captured foreground child survived SIGKILL of its wrapper" >&2
+  exit 1
+fi
+
 # Internal cleanup gives a same-group descendant a bounded TERM grace before
 # cancellation/escalation, so normal shutdown handlers can flush state.
 graceful_dir="$tmp/graceful-descendant"
diff --git a/tests/regressions-services.sh b/tests/regressions-services.sh
index 5cf9382..34466b8 100755
--- a/tests/regressions-services.sh
+++ b/tests/regressions-services.sh
@@ -44,6 +44,18 @@ wait_for_absent() {
   return 1
 }
 
+wait_for_process_exit() {
+  local pid="$1" attempt state
+  for ((attempt = 0; attempt < 150; attempt++)); do
+    if ! kill -0 "$pid" 2>/dev/null; then return 0; fi
+    state="$(ps -o stat= -p "$pid" 2>/dev/null || true)"
+    [[ "$state" == Z* ]] && return 0
+    sleep 0.02
+  done
+  echo "timed out waiting for process $pid to exit" >&2
+  return 1
+}
+
 cat >"$definitions/healthy" <<'EOF'
 #!/usr/bin/env bash
 printf 'start %s\n' "$$" >>"$SERVICE_TEST_EVENTS/healthy.starts"
@@ -279,6 +291,58 @@ helper_failure_services "$missing_helper" stop-all >/dev/null
 wait_for_absent "$helper_failure_runtime/keep.state"
 wait_for_count 1 "$helper_failure_events/keep.terms"
 
+# Unexpected SIGKILL of a capture wrapper cannot orphan its direct foreground
+# service. Reconcile replaces the stale wrapper with one service, and stop-all
+# leaves no managed capture or foreground service behind.
+pdeath_config="$tmp/pdeath-config"
+pdeath_definitions="$pdeath_config/services.d"
+pdeath_runtime="$tmp/pdeath-runtime"
+pdeath_logs="$tmp/pdeath-logs"
+pdeath_events="$tmp/pdeath-events"
+mkdir -p "$pdeath_definitions" "$pdeath_runtime" "$pdeath_logs" "$pdeath_events"
+cat >"$pdeath_definitions/guarded" <<'EOF'
+#!/usr/bin/env bash
+printf 'start %s\n' "$$" >>"$PDEATH_SERVICE_EVENTS/starts"
+exec python3 -c 'import signal; signal.pause()' >/dev/null 2>&1
+EOF
+chmod 0700 "$pdeath_definitions/guarded"
+
+pdeath_services() {
+  env PDEATH_SERVICE_EVENTS="$pdeath_events" \
+    TX9_SERVICES_CONFIG_ROOT="$pdeath_config" \
+    TX9_SERVICES_STATE_DIR="$pdeath_runtime" \
+    TX9_SERVICES_LOG_DIR="$pdeath_logs" \
+    TX9_SERVICES_LOG_HELPER="$PROJECT_ROOT/guest/tx9-logs" \
+    TX9_SERVICES_RESTART_DELAY=0.2 \
+    TX9_SERVICES_STOP_TIMEOUT=2 \
+    "$services" "$@"
+}
+
+pdeath_services reconcile >/dev/null
+wait_for_count 1 "$pdeath_events/starts"
+old_pdeath_capture="$(cut -f1 "$pdeath_runtime/guarded.state")"
+old_pdeath_service="$(awk 'NR == 1 { print $2 }' "$pdeath_events/starts")"
+pids+=("$old_pdeath_capture")
+kill -KILL "$old_pdeath_capture"
+wait_for_process_exit "$old_pdeath_capture"
+wait_for_process_exit "$old_pdeath_service"
+
+pdeath_services reconcile >/dev/null
+wait_for_count 2 "$pdeath_events/starts"
+new_pdeath_capture="$(cut -f1 "$pdeath_runtime/guarded.state")"
+new_pdeath_service="$(awk 'NR == 2 { print $2 }' "$pdeath_events/starts")"
+pids+=("$new_pdeath_capture")
+[[ "$new_pdeath_capture" != "$old_pdeath_capture" ]]
+[[ "$new_pdeath_service" != "$old_pdeath_service" ]]
+kill -0 "$new_pdeath_capture"
+kill -0 "$new_pdeath_service"
+[[ "$(wc -l <"$pdeath_events/starts" | tr -d ' ')" == 2 ]]
+
+pdeath_services stop-all >/dev/null
+wait_for_absent "$pdeath_runtime/guarded.state"
+wait_for_process_exit "$new_pdeath_capture"
+wait_for_process_exit "$new_pdeath_service"
+
 # Runtime logging configuration is part of supervisor identity. Status reports
 # drift in each setting, and reconcile uses the stored settings to stop the old
 # process before starting exactly one replacement with the current settings.

From b8328659533b660aabdeb8fa4dc1e0f18d863b11 Mon Sep 17 00:00:00 2001
From: Ben Davis <45952064+bmdavis419@users.noreply.github.com>
Date: Wed, 22 Jul 2026 14:13:47 -0700
Subject: [PATCH 6/7] fix: report captured service exec failures

---
 guest/tx9-logs            | 170 +++++++++++++++++++++++++++-----------
 tests/regressions-logs.sh |  74 +++++++++++++++++
 2 files changed, 196 insertions(+), 48 deletions(-)

diff --git a/guest/tx9-logs b/guest/tx9-logs
index c25c9e1..6c6dad5 100755
--- a/guest/tx9-logs
+++ b/guest/tx9-logs
@@ -12,6 +12,7 @@ import argparse
 import base64
 import ctypes
 import datetime as dt
+import errno
 import hashlib
 import heapq
 import io
@@ -516,6 +517,14 @@ def nonnegative_float(value: str) -> float:
     return parsed
 
 
+def write_exec_failure(status_fd: int, stage: str, error: int) -> None:
+    payload = f"{stage}:{error}\n".encode("ascii")
+    try:
+        os.write(status_fd, payload)
+    except OSError:
+        pass
+
+
 def parent_death_exec(arguments: Sequence[str]) -> int:
     """Arm Linux parent-death SIGKILL, close the setup race, then exec.
 
@@ -524,35 +533,40 @@ def parent_death_exec(arguments: Sequence[str]) -> int:
     outside this guarantee.
     """
 
-    if len(arguments) < 3 or arguments[1] != "--":
+    if len(arguments) < 4 or arguments[2] != "--":
         print("tx9-logs: invalid internal parent-death invocation", file=sys.stderr)
         return 126
     try:
         expected_parent = positive_int(arguments[0])
+        status_fd = positive_int(arguments[1])
     except argparse.ArgumentTypeError as exc:
-        print(f"tx9-logs: invalid internal parent pid: {exc}", file=sys.stderr)
+        print(f"tx9-logs: invalid internal parent-death argument: {exc}", file=sys.stderr)
         return 126
-    command = list(arguments[2:])
+    command = list(arguments[3:])
     if not command:
         print("tx9-logs: internal parent-death exec requires a command", file=sys.stderr)
         return 126
 
-    libc = ctypes.CDLL(None, use_errno=True)
-    prctl = libc.prctl
-    prctl.argtypes = [
-        ctypes.c_int,
-        ctypes.c_ulong,
-        ctypes.c_ulong,
-        ctypes.c_ulong,
-        ctypes.c_ulong,
-    ]
-    prctl.restype = ctypes.c_int
-    if prctl(PR_SET_PDEATHSIG, signal.SIGKILL, 0, 0, 0) != 0:
-        error = ctypes.get_errno()
-        print(
-            f"tx9-logs: could not arm parent-death signal: {os.strerror(error)}",
-            file=sys.stderr,
-        )
+    try:
+        # Popen clears CLOEXEC for pass_fds so this fresh interpreter can
+        # report setup/exec errors. Restore it before the target exec: EOF on
+        # the parent end is then the unambiguous success handshake.
+        os.set_inheritable(status_fd, False)
+        libc = ctypes.CDLL(None, use_errno=True)
+        prctl = libc.prctl
+        prctl.argtypes = [
+            ctypes.c_int,
+            ctypes.c_ulong,
+            ctypes.c_ulong,
+            ctypes.c_ulong,
+            ctypes.c_ulong,
+        ]
+        prctl.restype = ctypes.c_int
+        if prctl(PR_SET_PDEATHSIG, signal.SIGKILL, 0, 0, 0) != 0:
+            error = ctypes.get_errno()
+            raise OSError(error, os.strerror(error))
+    except OSError as exc:
+        write_exec_failure(status_fd, "setup", exc.errno or 0)
         return 126
 
     # PR_SET_PDEATHSIG is not retroactive. If capture died between Popen's
@@ -564,7 +578,7 @@ def parent_death_exec(arguments: Sequence[str]) -> int:
     try:
         os.execvpe(command[0], command, os.environ)
     except OSError as exc:
-        print(f"tx9-logs: could not exec {Path(command[0]).name}: {exc}", file=sys.stderr)
+        write_exec_failure(status_fd, "exec", exc.errno or 0)
         return 127 if isinstance(exc, FileNotFoundError) else 126
 
 
@@ -1087,59 +1101,119 @@ class Capture:
         # subprocess.preexec_fn, whose Python callback is unsafe around the
         # reader threads used by capture. Popen creates the new session first;
         # the fresh interpreter then arms parent death and execs the command.
-        launch_command = [
-            sys.executable,
-            str(Path(__file__).resolve()),
-            PARENT_DEATH_EXEC_ACTION,
-            str(os.getpid()),
-            "--",
-            *command,
-        ]
+        status_read_fd = -1
+        status_write_fd = -1
+        reader_stop = threading.Event()
+        reader_cancelled = threading.Event()
+        readers: list[threading.Thread] = []
         try:
+            status_read_fd, status_write_fd = os.pipe()
+            launch_command = [
+                sys.executable,
+                str(Path(__file__).resolve()),
+                PARENT_DEATH_EXEC_ACTION,
+                str(os.getpid()),
+                str(status_write_fd),
+                "--",
+                *command,
+            ]
             child = subprocess.Popen(
                 launch_command,
                 stdout=subprocess.PIPE,
                 stderr=subprocess.PIPE,
                 env=child_env,
                 start_new_session=True,
-                pass_fds=(self.mirror_out, self.mirror_err),
+                pass_fds=(self.mirror_out, self.mirror_err, status_write_fd),
             )
             self.child = child
             self.active_pgid = child.pid
+            os.close(status_write_fd)
+            status_write_fd = -1
+            assert self.child.stdout is not None
+            assert self.child.stderr is not None
+            readers = [
+                threading.Thread(
+                    target=self._reader,
+                    args=("stdout", self.child.stdout, reader_stop, reader_cancelled),
+                    daemon=True,
+                ),
+                threading.Thread(
+                    target=self._reader,
+                    args=("stderr", self.child.stderr, reader_stop, reader_cancelled),
+                    daemon=True,
+                ),
+            ]
+            for reader in readers:
+                reader.start()
             # A signal can arrive after Popen creates the process group but
             # before the assignment above. Honor it before waiting so that
             # startup cannot lose Docker's shutdown request.
             if self.received_signal is not None:
                 self._forward_signal(self.received_signal, None)
+            read_fd = status_read_fd
+            status_read_fd = -1
+            with os.fdopen(read_fd, "rb", buffering=0) as status_pipe:
+                exec_status = status_pipe.read(128)
         except OSError as exc:
+            if status_read_fd >= 0:
+                os.close(status_read_fd)
+            if status_write_fd >= 0:
+                os.close(status_write_fd)
+            if self.child is not None:
+                self._signal_process_group(signal.SIGKILL)
+                self.child.wait()
+            reader_stop.set()
+            for reader in readers:
+                reader.join()
+            if reader_cancelled.is_set():
+                self._signal_process_group(signal.SIGKILL)
+            self.active_pgid = None
+            self.child = None
+            self.flush_repeats()
             message = f"could not start {Path(command[0]).name}: {exc}"
             self.output("stderr", (message + "\n").encode())
             self.structured("process_start_failed", message)
-            return 127 if isinstance(exc, FileNotFoundError) else 126
+            status = 127 if isinstance(exc, FileNotFoundError) else 126
+            if self.received_signal is not None:
+                return 128 + self.received_signal
+            return status
+
+        if exec_status:
+            try:
+                stage, error_text = exec_status.decode("ascii").strip().split(":", 1)
+                error = int(error_text)
+                if stage not in {"setup", "exec"} or error < 0:
+                    raise ValueError
+            except (UnicodeDecodeError, ValueError):
+                stage = "setup"
+                error = 0
+            self.child.wait()
+            reader_stop.set()
+            for reader in readers:
+                reader.join()
+            if reader_cancelled.is_set():
+                self._signal_process_group(signal.SIGKILL)
+            self.active_pgid = None
+            self.child = None
+            self.flush_repeats()
+            if stage == "exec":
+                detail = OSError(error, os.strerror(error), command[0])
+                status = 127 if error == errno.ENOENT else 126
+            else:
+                detail = OSError(error, os.strerror(error))
+                status = 126
+            message = f"could not start {Path(command[0]).name}: {detail}"
+            self.output("stderr", (message + "\n").encode())
+            self.structured("process_start_failed", message)
+            if self.received_signal is not None:
+                return 128 + self.received_signal
+            return status
 
         self.structured(
             "process_start",
             f"{Path(command[0]).name} started",
             {"pid": self.child.pid, "command": Path(command[0]).name},
         )
-        assert self.child.stdout is not None
-        assert self.child.stderr is not None
-        reader_stop = threading.Event()
-        reader_cancelled = threading.Event()
-        readers = [
-            threading.Thread(
-                target=self._reader,
-                args=("stdout", self.child.stdout, reader_stop, reader_cancelled),
-                daemon=True,
-            ),
-            threading.Thread(
-                target=self._reader,
-                args=("stderr", self.child.stderr, reader_stop, reader_cancelled),
-                daemon=True,
-            ),
-        ]
-        for reader in readers:
-            reader.start()
         return_code = self.child.wait()
         # The direct child owns the capture lifecycle. Stop background
         # descendants in its process group, then let readers drain buffered
diff --git a/tests/regressions-logs.sh b/tests/regressions-logs.sh
index ab9be80..c0054d8 100755
--- a/tests/regressions-logs.sh
+++ b/tests/regressions-logs.sh
@@ -51,6 +51,80 @@ args = parser.parse_args(["capture", "--source", "agent", "--", "true"])
 assert args.restart_delay is None
 PY
 
+# The parent-death re-exec shim preserves Popen's lifecycle contract: actual
+# exec failures are process_start_failed events, never successful starts.
+start_failure_root="$tmp/start-failures"
+mkdir -p "$start_failure_root"
+printf '#!/usr/bin/env bash\nexit 0\n' >"$start_failure_root/non-executable"
+chmod 0600 "$start_failure_root/non-executable"
+printf '#!/definitely/missing/tx9-interpreter\nexit 0\n' \
+  >"$start_failure_root/bad-shebang"
+chmod 0700 "$start_failure_root/bad-shebang"
+
+assert_start_failed() {
+  local label="$1" expected_status="$2" status log_dir
+  shift 2
+  log_dir="$start_failure_root/$label-logs"
+  set +e
+  "$helper" capture --source agent --log-dir "$log_dir" -- "$@" \
+    >"$start_failure_root/$label.stdout" 2>"$start_failure_root/$label.stderr"
+  status=$?
+  set -e
+  [[ "$status" == "$expected_status" ]]
+  jq -e 'select(.type == "process_start_failed")' "$log_dir/agent.jsonl" >/dev/null
+  if jq -e 'select(.type == "process_start")' "$log_dir/agent.jsonl" >/dev/null; then
+    echo "$label exec failure was reported as a successful process start" >&2
+    exit 1
+  fi
+}
+
+assert_start_failed missing 127 tx9-definitely-missing-command
+assert_start_failed non-executable 126 "$start_failure_root/non-executable"
+assert_start_failed bad-shebang 127 "$start_failure_root/bad-shebang"
+
+# The shim can emit more than a pipe buffer before exec (for example Python's
+# verbose import trace). Readers must drain concurrently with the handshake.
+verbose_start_dir="$start_failure_root/verbose-start-logs"
+if ! timeout 15 env PYTHONVERBOSE=2 \
+  "$helper" capture --source agent --log-dir "$verbose_start_dir" -- true \
+  >"$start_failure_root/verbose-start.stdout" \
+  2>"$start_failure_root/verbose-start.stderr"; then
+  echo "capture deadlocked while the exec shim filled stderr" >&2
+  exit 1
+fi
+jq -e 'select(.type == "process_start")' "$verbose_start_dir/agent.jsonl" >/dev/null
+jq -e 'select(.type == "process_exit" and .data.status == 0)' \
+  "$verbose_start_dir/agent.jsonl" >/dev/null
+
+# A signal received while startup is resolving overrides an exec failure's
+# 126/127 status just as it overrides a normal child exit.
+python3 - "$helper" "$start_failure_root/signal-override-logs" \
+  >"$start_failure_root/signal-override.stdout" \
+  2>"$start_failure_root/signal-override.stderr" <<'PY'
+import importlib.machinery
+import importlib.util
+import pathlib
+import signal
+import sys
+
+loader = importlib.machinery.SourceFileLoader("tx9_logs_signal_override", sys.argv[1])
+spec = importlib.util.spec_from_loader(loader.name, loader)
+assert spec is not None
+module = importlib.util.module_from_spec(spec)
+loader.exec_module(module)
+capture = module.Capture("agent", pathlib.Path(sys.argv[2]), 1024 * 1024, 2)
+capture.received_signal = signal.SIGTERM
+# Preserve the already-recorded signal without killing the short-lived shim,
+# making the exec-failure branch deterministic.
+capture._forward_signal = lambda _signum, _frame: None
+try:
+    assert capture.run_once(["tx9-definitely-missing-on-signal"]) == 143
+finally:
+    capture.close()
+PY
+jq -e 'select(.type == "process_start_failed")' \
+  "$start_failure_root/signal-override-logs/agent.jsonl" >/dev/null
+
 # Source reads stop at the first unterminated snapshot boundary instead of
 # reframing bytes appended during the query as a separate record. A record that
 # exactly fills the byte budget is retained; only a larger record is oversized.

From 3e7089be52d97a0b946bf8ba02a39c9ed493feca Mon Sep 17 00:00:00 2001
From: Ben Davis <45952064+bmdavis419@users.noreply.github.com>
Date: Wed, 22 Jul 2026 16:06:20 -0700
Subject: [PATCH 7/7] fix: make service log regression assertions deterministic

The all-source query asserted an early healthy-log event survived an
unfiltered --tail 100 while the intentionally broken service kept
crash-looping and appending records, so the check raced the flood.
Filter with --grep before the tail so eviction of unrelated events
cannot fail the assertion.

Both service log assertions also relied on jq -e select() over a
multi-record JSONL stream, whose exit status tracks the final record
rather than whether any record matched. Slurp and assert any() instead.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 tests/regressions-services.sh | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/tests/regressions-services.sh b/tests/regressions-services.sh
index 34466b8..67364db 100755
--- a/tests/regressions-services.sh
+++ b/tests/regressions-services.sh
@@ -473,17 +473,21 @@ after_early="$(wc -l <"$events/broken.starts" | tr -d ' ')"
 wait_for_count $((before + 1)) "$events/broken.starts"
 [[ "$(wc -l <"$events/healthy.starts" | tr -d ' ')" == 1 ]]
 
-# Each custom source is independently queryable, and `all` includes it.
+# Each custom source is independently queryable, and `all` includes it. The
+# crash-looping broken service floods the shared stream, so the all-source
+# check filters before the tail; an unfiltered tail legitimately evicts old
+# events. Assertions aggregate the stream: a matching record must exist even
+# when later records differ.
 wait_for_pattern healthy-log "$logs/service-healthy.jsonl"
 "$PROJECT_ROOT/guest/tx9-logs" query --agent-root "$data" \
   --executor-root "$tmp/empty-executor" --box fixture --source service-healthy \
   --tail 100 --json >"$tmp/service-query.jsonl"
-jq -e 'select(.source == "service-healthy" and .message == "healthy-log")' \
+jq -se 'any(.[]; .source == "service-healthy" and .message == "healthy-log")' \
   "$tmp/service-query.jsonl" >/dev/null
 "$PROJECT_ROOT/guest/tx9-logs" query --agent-root "$data" \
   --executor-root "$tmp/empty-executor" --box fixture --source all \
-  --tail 100 --json >"$tmp/all-query.jsonl"
-jq -e 'select(.source == "service-healthy" and .message == "healthy-log")' \
+  --grep healthy-log --tail 100 --json >"$tmp/all-query.jsonl"
+jq -se 'any(.[]; .source == "service-healthy" and .message == "healthy-log")' \
   "$tmp/all-query.jsonl" >/dev/null
 
 # Removing execute permission removes the desired service and synchronously