feat: supervise portable custom box services

Makefile

@@ -2,7 +2,7 @@
 
 # tests/regressions-*.sh is globbed so new split-out regression files don't
 # require touching this Makefile.
-SHELL_FILES := guest/hb guest/hb-workload guest/lib-mcp.sh guest/profile.sh guest/agent-bash-profile.sh provision/provision.sh docker/entrypoint.sh docker/executor-entrypoint.sh tests/lib.sh tests/static.sh tests/hermes-state.sh $(wildcard tests/regressions-*.sh)
+SHELL_FILES := guest/hb guest/hb-workload guest/tx9-services guest/lib-mcp.sh guest/profile.sh guest/agent-bash-profile.sh provision/provision.sh docker/entrypoint.sh docker/executor-entrypoint.sh tests/lib.sh tests/static.sh tests/hermes-state.sh $(wildcard tests/regressions-*.sh)
 
 syntax:
 	bash -n $(SHELL_FILES)

README.md

@@ -72,6 +72,8 @@ Inside the agent container:
 hb status / hb doctor / hb versions
 hb down            # durable pause; the reconcile loop won't restart services
 hb up
+hb services        # status for portable custom service drop-ins
+hb services-reload # reconcile custom services immediately
 hb gateway-enable --confirm-single-writer I_CONFIRM_NO_OTHER_GATEWAY_USES_THIS_IDENTITY
 hb gateway-disable
 hb verify-state
@@ -87,6 +89,54 @@ tx9 gateway enable <box> --confirm-single-writer
 tx9 gateway disable <box>
 ```
 
+### Portable custom services
+
+An executable regular file placed directly in
+`~/.config/hermes-box/services.d/` defines a custom service. Names must match
+`[a-z0-9][a-z0-9_-]{0,62}`; directories, symlinks, non-executable files, and
+unsafe names are ignored and reported by `hb services`. Definitions live on
+the portable `/data` volume, so they survive backup, import, and upgrade.
+
+Each file is executed directly by absolute path, without shell sourcing,
+evaluation, arguments, or interpolation. Scripts must remain in the
+foreground and should `exec` their daemon. For example:
+
+```bash
+install -d -m 700 ~/.config/hermes-box/services.d
+cat >~/.config/hermes-box/services.d/signal <<'EOF'
+#!/usr/bin/env bash
+exec signal-cli daemon
+EOF
+chmod 700 ~/.config/hermes-box/services.d/signal
+hb services-reload
+```
+
+The existing 20-second workload loop also reconciles definitions
+automatically. Each service is supervised independently with a bounded
+restart delay; one crashing service cannot block the others. `hb pause` and
+`hb down` synchronously stop all custom services and prevent restart, while
+`hb up`/`hb resume` start them again. Disabling only the Hermes gateway does
+not affect custom services. Container termination is forwarded through the
+log supervisor to each foreground process.
+
+`hb services` distinguishes a stable foreground child (`running`) from a
+live capture wrapper waiting to restart it (`restarting`). This is process
+state, not an application health check; `hb doctor` cannot infer a generic
+health contract for arbitrary service code.
+
+Service output uses the same rotating, redacted durable log pipeline as other
+tx9-owned processes. Sources are distinct and collision-free:
+
+```bash
+tx9 logs media-bot --source service-signal
+tx9 logs media-bot --source all
+```
+
+Drop-ins have the same permissions and network/filesystem access as the
+`agent` user. Treat installing one as installing executable code in the box;
+the filename and regular non-symlink checks prevent accidental shell
+evaluation or symlink execution, but do not sandbox trusted service code.
+
 ## Logs and resource allocation
 
 Runtime output from the agent supervisor, Hermes gateway, and Executor is

docker/entrypoint.sh

@@ -1,8 +1,9 @@
 #!/usr/bin/env bash
 # Container PID-1 workload (run under docker --init so signals behave).
-# Reuses guest/hb-workload verbatim: it reconciles Executor, the Hermes
-# gateway, and the 0.0.0.0 socat bridges every 20s. tx9-logs supervises the
-# loop process itself; Docker's restart policy supervises the container.
+# Reuses guest/hb-workload verbatim: it reconciles custom services, Executor,
+# the Hermes gateway, and the 0.0.0.0 socat bridges every 20s. tx9-logs
+# supervises the loop process itself; Docker's restart policy supervises the
+# container.
 set -uo pipefail
 
 trap 'exit 143' TERM

docs/tx9-cli-design.md

@@ -52,7 +52,7 @@ Docker overview can be read. `tx9 help` remains Docker-independent.
 | `tx9 backup <box>` (aliases `export`, `save`) | Flags: `--path` (default `~/Downloads`), `--password`/env/prompt, `--no-encrypt`. Quiesce → archive agent /data → validate → (encrypt) → verify → `<box>-<timestamp>.tx9`. |
 | `tx9 import <file.tx9>` (aliases `load`, `restore`) | Flags: `--name`, `--password`/env/prompt. Validate before creating anything; restore staged; arrive quiesced + gateway-disabled + fresh token; fail on name collision. |
 | `tx9 mount <add\|list\|remove> ...` | Persist host-directory bind mounts for an agent and recreate only its disposable container. Targets must be below `/mnt`, outside the portable `/data` volume. `add` supports `--read-only` and `--require-mountpoint`. |
-| `tx9 logs <box>` | Query durable agent, Executor, Hermes, Codex, and Claude events. Filters include source, age, text, severity (`--level`, this level and above; unleveled events count as info), count, and normalized JSONL. `tx9 logs export <box>` creates a mode-0600 portable log bundle from both isolated volumes. |
+| `tx9 logs <box>` | Query durable agent, Executor, Hermes, Codex, Claude, and custom-service events. Filters include source, age, text, severity (`--level`, this level and above; unleveled events count as info), count, and normalized JSONL. Custom sources use `service-<name>`. `tx9 logs export <box>` creates a mode-0600 portable log bundle from both isolated volumes. |
 | `tx9 resources <box>` | Show actual container CPU/RAM limits and volume usage versus advisory budgets. `resources set` updates limits live and persists them; `resources reset` restores 4 CPU/8 GiB (agent) and 2 CPU/2 GiB (Executor). |
 | `tx9 gateway <status\|enable\|disable> <box>` | Inspect or control the container-supervised Hermes gateway. Enable requires `--confirm-single-writer`. |
 | `tx9 open <box>` | Print (or open) the authenticated dashboard URL (`?_token=`). |
@@ -108,6 +108,20 @@ the port:
   SQLite histories. Executor data is never mounted into the agent container.
   This provides complete tx9-owned runtime output, not an independent audit
   trail for operations that Executor does not emit to any durable sink.
+- **Portable custom services**: direct executable regular-file children of
+  `${XDG_CONFIG_HOME:-$HOME/.config}/hermes-box/services.d` are supervised as
+  the agent. Names are limited to `[a-z0-9][a-z0-9_-]{0,62}`. The absolute
+  file path is passed as argv directly, never sourced or evaluated; scripts
+  stay foreground and should `exec` their daemon. Each service has an
+  independent `tx9-logs` capture and bounded restart loop under source
+  `service-<name>`. Runtime PID/lock state stays outside `/data`, while the
+  definitions and logs remain on `/data` and therefore travel in backups.
+  `hb services` reports status and ignored entries; `hb services-reload`
+  reconciles immediately in addition to the normal 20-second loop. Quiesce
+  synchronously stops every custom service and blocks restart until
+  `hb up`/`hb resume`; Hermes gateway enable/disable is independent. Drop-ins
+  run with the agent's ordinary access and are trusted executable code, not a
+  sandbox boundary.
 - **Gateway single-writer**: restored boxes never auto-enable the Hermes
   gateway. `tx9 gateway enable <box> --confirm-single-writer` delegates to
   `hb gateway-enable` and stays the only host-side path.

guest/hb

@@ -4,8 +4,9 @@ set -uo pipefail
 # shellcheck disable=SC1091
 [ -f /etc/profile.d/hermes-box.sh ] && . /etc/profile.d/hermes-box.sh
 export TX9_LOG_MAX_BYTES TX9_LOG_MAX_FILES
+HB_BIN_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 # shellcheck source=guest/lib-mcp.sh
-. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/lib-mcp.sh"
+. "$HB_BIN_DIR/lib-mcp.sh"
 
 DATA="${HB_DATA:-/data}"
 AGENT_HOME="$DATA/home/agent"
@@ -20,6 +21,7 @@ GATEWAY_LOCK="$STATE_DIR/gateway.lock"
 GATEWAY_RELOAD_REQUESTS="$STATE_DIR/gateway-reload-requests"
 GATEWAY_RELOAD_LOCK="$STATE_DIR/gateway-reload.lock"
 EXECUTOR_LOCK="$STATE_DIR/executor.lock"
+SERVICES_DIR="$STATE_DIR/services.d"
 WIRE_VERSION=http-v3
 WIRED="$AGENT_HOME/.hermes-box-wired"
 TOKEN_FILE="$AGENT_HOME/.executor/server-control/auth.json"
@@ -64,9 +66,11 @@ init() {
   local managed=(
     "$AGENT_HOME" "$AGENT_HOME/.claude" "$AGENT_HOME/.codex" "$AGENT_HOME/.hermes"
     "$AGENT_HOME/workspace" "$AGENT_HOME/.config" "$AGENT_HOME/.config/hermes-box"
-    "$AGENT_HOME/.local" "$AGENT_HOME/.local/share" "$AGENT_HOME/.local/state" "$LOGS"
+    "$SERVICES_DIR" "$AGENT_HOME/.local" "$AGENT_HOME/.local/share"
+    "$AGENT_HOME/.local/state" "$LOGS"
   )
   mkdir -p "${managed[@]}"
+  chmod 0700 "$SERVICES_DIR" 2>/dev/null || true
   id agent >/dev/null 2>&1 && chown agent:agent "${managed[@]}" 2>/dev/null || true
   if [[ -e "$LEGACY_QUIESCE_FILE" ]]; then
     touch "$QUIESCE_FILE" || return 1
@@ -81,6 +85,20 @@ init() {
   id agent >/dev/null 2>&1 && chown agent:agent "$GATEWAY_DISABLED" "$GATEWAY_POLICY" 2>/dev/null || true
 }
 
+_services_helper() {
+  local helper
+  helper="$(command -v tx9-services 2>/dev/null || true)"
+  [[ -n "$helper" ]] || helper="$HB_BIN_DIR/tx9-services"
+  [[ -x "$helper" ]] || {
+    echo "tx9-services is unavailable" >&2
+    return 1
+  }
+  TX9_SERVICES_CONFIG_ROOT="$STATE_DIR" TX9_SERVICES_LOG_DIR="$LOGS" "$helper" "$@"
+}
+
+_services_reconcile() { _services_helper reconcile; }
+_services_stop() { _services_helper stop-all; }
+
 _port_open() {
   (exec 3<>"/dev/tcp/$EXECUTOR_HOST/${EXECUTOR_PORT:-4788}") 2>/dev/null && { exec 3>&-; return 0; }
   return 1
@@ -333,15 +351,29 @@ _start_gateway() {
 }
 
 up() {
+  local services_failed=0
+  local gateway_status
+
   init
   rm -f "$QUIESCE_FILE"
+  _services_reconcile || {
+    services_failed=1
+    echo "warning: custom service reconciliation failed; run 'hb services-reload' for details" >&2
+  }
   _executor_up || return 1
   _start_gateway
+  gateway_status=$?
+  [[ "$gateway_status" == 0 ]] || return "$gateway_status"
+  [[ "$services_failed" == 0 ]]
 }
 
 reconcile() {
   init
-  [[ ! -e "$QUIESCE_FILE" ]] || return 0
+  if [[ -e "$QUIESCE_FILE" ]]; then
+    _services_stop
+    return
+  fi
+  _services_reconcile || echo "warning: custom service reconciliation failed; continuing core reconciliation" >&2
   HB_STEADY_QUIET=1 _executor_up || return 1
   _start_gateway
 }
@@ -372,13 +404,14 @@ pause() {
   local failed=0
   init
   touch "$QUIESCE_FILE"
+  _services_stop || failed=1
   _stop_gateway || failed=1
   _stop_executor || failed=1
   if command -v hermes-state >/dev/null 2>&1; then
     hermes-state verify --checkpoint >/dev/null || failed=1
   fi
   [[ "$failed" == 0 ]] || return 1
-  echo "Hermes gateway and Executor: quiesced"
+  echo "Custom services, Hermes gateway, and Executor: quiesced"
 }
 
 resume() {
@@ -729,7 +762,7 @@ write_manifest() {
 }
 
 status() {
-  local gateway_status
+  local gateway_status services_count
   if [[ -e "$GATEWAY_DISABLED" ]]; then
     gateway_status=disabled
   elif _gateway_running; then
@@ -747,9 +780,20 @@ status() {
     printf 'platform: no runtime state\n'
   fi
   printf 'mcp:      %s\n' "$([[ -f "$WIRED" ]] && cat "$WIRED" || echo unwired)"
+  services_count="$(find "$SERVICES_DIR" -mindepth 1 -maxdepth 1 -printf x 2>/dev/null | wc -c | tr -d ' ')"
+  printf "services: %s drop-in entries (run 'hb services')\n" "$services_count"
   printf 'data:     %s\n' "$(du -sh "$DATA" 2>/dev/null | cut -f1)"
 }
 
+services() { _services_helper status; }
+
+services_reload() {
+  local failed=0
+  _services_reconcile || failed=1
+  services || failed=1
+  return "$failed"
+}
+
 versions() {
   echo "node:     $(node --version 2>/dev/null || echo -)"
   # sed -n 1p, not head -1: vp prints more after the version line, and head
@@ -822,6 +866,7 @@ doctor() {
     _check "Codex HTTP MCP config" grep -qF "url = \"$(_executor_url)\"" "$CODEX_HOME/config.toml" || failed=1
     _check "Hermes HTTP MCP config" _hermes_http_config || failed=1
   fi
+  echo "info custom services have no generic health contract; inspect process state with 'hb services'"
   [[ "$failed" == 0 ]]
 }
 
@@ -861,9 +906,11 @@ main() {
     write-manifest) write_manifest ;;
     doctor) doctor ;;
     status) status ;;
+    services) services ;;
+    services-reload) services_reload ;;
     versions) versions ;;
     logs) shift; logs "${1:-executor}" ;;
-    *) echo "usage: hb {init|up|reconcile|down|pause|resume|gateway-enable|gateway-disable|gateway-reload-if-requested|verify-state|acknowledge-active-paths|cutover-ready|web|wire-mcp|doctor|status|versions|logs}"; return 1 ;;
+    *) echo "usage: hb {init|up|reconcile|down|pause|resume|services|services-reload|gateway-enable|gateway-disable|gateway-reload-if-requested|verify-state|acknowledge-active-paths|cutover-ready|web|wire-mcp|doctor|status|versions|logs}"; return 1 ;;
   esac
 }
 

guest/hb-workload

@@ -17,6 +17,8 @@ EXECUTOR_TARGET_PORT="${EXECUTOR_PORT:-4788}"
 HERMES_TARGET_PORT="${HERMES_API_PORT:-8642}"
 LOGS="${HB_WORKLOAD_LOGS:-/data/logs}"
 BOX_ENV="${HB_BOX_ENV:-/etc/hermes-box.env}"
+SERVICES_HELPER="${HB_SERVICES_HELPER:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/tx9-services}"
+WORKLOAD_SLEEP_PID=''
 
 _exec_up() { (exec 3<>"/dev/tcp/127.0.0.1/$EXECUTOR_TARGET_PORT") 2>/dev/null && { exec 3>&-; return 0; }; return 1; }
 _api_up() { (exec 3<>"/dev/tcp/127.0.0.1/$HERMES_TARGET_PORT") 2>/dev/null && { exec 3>&-; return 0; }; return 1; }
@@ -54,6 +56,22 @@ _spawn_logged() {
   fi
 }
 
+_stop_custom_services() {
+  [[ -x "$SERVICES_HELPER" ]] || return 0
+  TX9_SERVICES_CONFIG_ROOT="$(dirname "$QUIESCE_FILE")" \
+    TX9_SERVICES_LOG_DIR="$LOGS" "$SERVICES_HELPER" stop-all
+}
+
+_shutdown() {
+  local exit_status="$1"
+  trap - TERM INT
+  [[ -z "$WORKLOAD_SLEEP_PID" ]] || kill -TERM "$WORKLOAD_SLEEP_PID" 2>/dev/null || true
+  _stop_custom_services || echo "custom service shutdown failed during workload termination" >&2
+  _stop_executor_bridge
+  _stop_api_bridge
+  exit "$exit_status"
+}
+
 _gateway_reload_pending() {
   [ -d "$GATEWAY_RELOAD_REQUESTS" ] &&
     [ -n "$(find "$GATEWAY_RELOAD_REQUESTS" -maxdepth 1 -type f -name 'request.*' -print -quit)" ]
@@ -89,6 +107,10 @@ reconcile_once() {
     return
   fi
   if [ -e "$QUIESCE_FILE" ] || [ -e "$LEGACY_QUIESCE_FILE" ]; then
+    # Reconciliation while quiesced is a stop-only operation for portable
+    # custom services. Explicit hb pause performs the synchronous gate; this
+    # also covers a durable marker restored or created between loop cycles.
+    _stop_custom_services || echo "custom service shutdown reconciliation failed" >&2
     _stop_executor_bridge
     _stop_api_bridge
     return
@@ -121,11 +143,19 @@ reconcile_once() {
 }
 
 main() {
+  # Install lifecycle traps only for the executed workload. Regression tests
+  # source this file to exercise reconcile_once and must retain their own
+  # process-level trap policy.
+  trap '_shutdown 143' TERM
+  trap '_shutdown 130' INT
   mkdir -p "$LOGS"
   while true; do
     reconcile_once
     [[ "${HB_WORKLOAD_ONCE:-0}" != 1 ]] || break
-    sleep 20
+    sleep 20 &
+    WORKLOAD_SLEEP_PID=$!
+    wait "$WORKLOAD_SLEEP_PID" 2>/dev/null || true
+    WORKLOAD_SLEEP_PID=''
   done
 }