fix(ssh): reconnecting no longer locks out the remote's background service

docs/internals/remote.md

@@ -165,7 +165,11 @@ renderer uses the forwarded endpoint through the shared connection runtime.
 [SSH cleanup](../../packages/ssh/src/tunnel.ts) stops a remote server only if the
 launcher owns it; a server it discovered already running must survive a client
 disconnect. Reconnection restores the forward before opening the application
-transport.
+transport. When the remote home has a loaded background service, the launch
+waits for that service's server before starting one: the service has no server
+for a few seconds while it restarts or updates, and a server started in that gap
+takes the [ownership lock](../../apps/server/src/serverOwnership.ts) and leaves
+the service idle until someone stops it.
 
 Remote servers can outlive several client releases. Clients must use advertised
 capabilities and handle their absence, rather than assume their own version

packages/ssh/src/launchScript.test.ts

@@ -0,0 +1,162 @@
+import { assert, describe, it } from "@effect/vitest";
+import * as NodeServices from "@effect/platform-node/NodeServices";
+import * as HostProcess from "@t3tools/shared/HostProcess";
+import * as Effect from "effect/Effect";
+import * as FileSystem from "effect/FileSystem";
+import * as Path from "effect/Path";
+import * as Stream from "effect/Stream";
+import { ChildProcess, ChildProcessSpawner } from "effect/process";
+
+import { buildRemoteLaunchScript } from "./tunnel.ts";
+
+// Stands in for both the background service's server and a server the launch
+// starts itself, so the real launch script runs end to end under `sh`.
+const FAKE_SERVER = `const fs = require("node:fs");
+const http = require("node:http");
+const path = require("node:path");
+const fakeDir = process.env.FAKE_DIR;
+const args = process.argv.slice(2);
+const server = http.createServer((_request, response) => response.end("ok"));
+if (args[0] === "service") {
+  const home = args[1];
+  server.listen(0, "127.0.0.1", () => {
+    const port = server.address().port;
+    fs.mkdirSync(path.join(home, "userdata"), { recursive: true });
+    fs.writeFileSync(
+      path.join(home, "userdata", "server-runtime.json"),
+      JSON.stringify({ pid: process.pid, port, origin: "http://127.0.0.1:" + port }),
+    );
+    fs.writeFileSync(path.join(fakeDir, "service.pid"), String(process.pid));
+  });
+} else {
+  const port = Number(args[args.indexOf("--port") + 1]);
+  server.listen(port, "127.0.0.1", () => {
+    fs.writeFileSync(path.join(fakeDir, "launched.pid"), String(process.pid));
+  });
+}
+`;
+
+// Reports the unit as activating and brings its server up, the way a service
+// looks while it switches servers during a restart or update.
+const FAKE_SYSTEMCTL = `#!/bin/sh
+if [ -f "$FAKE_DIR/service-installed" ]; then
+  if [ ! -f "$FAKE_DIR/service-started" ]; then
+    : >"$FAKE_DIR/service-started"
+    node "$FAKE_SERVER" service "$HOME/.t3" >/dev/null 2>&1 &
+  fi
+  echo activating
+  exit 3
+fi
+echo inactive
+exit 3
+`;
+
+const collectText = <E>(stream: Stream.Stream<Uint8Array, E>) =>
+  stream.pipe(
+    Stream.decodeText(),
+    Stream.runFold(
+      () => "",
+      (text, chunk) => text + chunk,
+    ),
+  );
+
+const launchOnFakeRemote = Effect.fn("launchOnFakeRemote")(function* (options: {
+  readonly withService: boolean;
+}) {
+  const fs = yield* FileSystem.FileSystem;
+  const path = yield* Path.Path;
+  const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-ssh-launch-" });
+  const home = path.join(root, "home");
+  const fakeDir = path.join(root, "fake");
+  const bin = path.join(root, "bin");
+  for (const dir of [home, fakeDir, bin]) yield* fs.makeDirectory(dir, { recursive: true });
+  const fakeServer = path.join(root, "fake-server.cjs");
+  yield* fs.writeFileString(fakeServer, FAKE_SERVER);
+  yield* fs.writeFileString(path.join(bin, "systemctl"), FAKE_SYSTEMCTL);
+  yield* fs.chmod(path.join(bin, "systemctl"), 0o755);
+  if (options.withService) {
+    yield* fs.makeDirectory(path.join(home, ".t3", "runtime"), { recursive: true });
+    yield* fs.writeFileString(
+      path.join(home, ".t3", "runtime", "service-state.json"),
+      JSON.stringify({ protocol: 3, activeVersion: "1.2.3" }),
+    );
+    yield* fs.writeFileString(path.join(fakeDir, "service-installed"), "");
+  }
+
+  const pidAt = (name: string) =>
+    fs.readFileString(path.join(fakeDir, name)).pipe(
+      Effect.map(Number),
+      Effect.orElseSucceed(() => undefined),
+    );
+  // Registered after the temp directory, so it stops the servers before the directory goes.
+  yield* Effect.addFinalizer(() =>
+    Effect.forEach(["service.pid", "launched.pid"], (name) =>
+      pidAt(name).pipe(
+        Effect.flatMap((pid) =>
+          Effect.sync(() => {
+            if (pid === undefined) return;
+            try {
+              process.kill(pid);
+            } catch {}
+          }),
+        ),
+      ),
+    ),
+  );
+
+  const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
+  const child = yield* spawner.spawn(
+    ChildProcess.make("sh", ["-s", "--", "test-state-key"], {
+      env: {
+        HOME: home,
+        PATH: [bin, path.dirname(process.execPath), "/usr/bin", "/bin"].join(":"),
+        FAKE_DIR: fakeDir,
+        FAKE_SERVER: fakeServer,
+      },
+      extendEnv: false,
+      stdin: {
+        stream: Stream.make(
+          new TextEncoder().encode(buildRemoteLaunchScript({ nodeScriptPath: fakeServer })),
+        ),
+        endOnDone: true,
+      },
+    }),
+  );
+  const [stdout, stderr, exitCode] = yield* Effect.all(
+    [collectText(child.stdout), collectText(child.stderr), child.exitCode],
+    { concurrency: "unbounded" },
+  );
+  assert.equal(Number(exitCode), 0, stderr);
+  const result = JSON.parse(stdout.trim().split("\n").at(-1) ?? "") as {
+    readonly remotePort: number;
+    readonly serverKind: string;
+  };
+  const runtime = yield* fs
+    .readFileString(path.join(home, ".t3", "userdata", "server-runtime.json"))
+    .pipe(
+      Effect.map((text) => JSON.parse(text) as { readonly port: number }),
+      Effect.orElseSucceed(() => undefined),
+    );
+  return { result, servicePort: runtime?.port, launchedPid: yield* pidAt("launched.pid") };
+});
+
+describe.skipIf(HostProcess.Platform.defaultValue() === "win32")("remote launch script", () => {
+  it.effect("waits for a restarting background service instead of starting its own server", () =>
+    Effect.gen(function* () {
+      const { result, servicePort, launchedPid } = yield* launchOnFakeRemote({
+        withService: true,
+      });
+      assert.equal(result.serverKind, "external");
+      assert.equal(result.remotePort, servicePort);
+      assert.isUndefined(launchedPid);
+    }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
+  );
+
+  it.effect("starts its own server right away when the home has no background service", () =>
+    Effect.gen(function* () {
+      const { result, launchedPid } = yield* launchOnFakeRemote({ withService: false });
+      assert.equal(result.serverKind, "managed");
+      assert.isDefined(launchedPid);
+    }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
+  );
+});

packages/ssh/src/tunnel.ts

@@ -51,7 +51,8 @@ const SSH_READY_PROBE_TIMEOUT_MS = 1_000;
 const TUNNEL_SHUTDOWN_TIMEOUT_MS = 2_000;
 const TUNNEL_STDERR_TAIL_CHARS = 16_384;
 const REMOTE_READY_TIMEOUT_MS = 60_000;
-const REMOTE_LAUNCH_TIMEOUT_MS = 90_000;
+// Covers the wait for a restarting background service plus our own server's readiness.
+const REMOTE_LAUNCH_TIMEOUT_MS = 120_000;
 // Cold archive launches download and extract ~70 MB and may wait on an install lock.
 // Lock waits must outlast both downloads and extraction; SSH must outlast install or wait plus readiness.
 // Leave slack for verification and extraction, which have no timeout.
@@ -60,6 +61,8 @@ const REMOTE_ARCHIVE_DOWNLOAD_SECONDS = 240;
 const REMOTE_ARCHIVE_LOCK_WAIT_SECONDS = 360;
 const REMOTE_ARCHIVE_LAUNCH_TIMEOUT_MS = 900_000;
 const REMOTE_REUSE_READY_TIMEOUT_MS = 2_000;
+// How long a launch waits for this home's restarting background service before starting its own server.
+const REMOTE_SERVICE_WAIT_SECONDS = 30;
 
 export interface RemoteT3RunnerOptions {
   readonly nodeScriptPath?: string | null;
@@ -619,6 +622,23 @@ try {
 }
 NODE
 }
+# True while this home's background service (\`t3 service install\`) is loaded,
+# including between its servers. Names match apps/server/src/cloud/bootService.ts;
+# service-state.json ties the service to this home.
+boot_service_loaded() {
+  [ -f "$DEFAULT_SERVER_HOME/runtime/service-state.json" ] || return 1
+  if command -v systemctl >/dev/null 2>&1; then
+    case "$(systemctl --user is-active t3code.service 2>/dev/null || true)" in
+      active | activating | reloading) return 0 ;;
+    esac
+    return 1
+  fi
+  if command -v launchctl >/dev/null 2>&1; then
+    launchctl print "gui/$(id -u)/com.t3tools.t3code.service" >/dev/null 2>&1
+    return
+  fi
+  return 1
+}
 REMOTE_PID="$(cat "$PID_FILE" 2>/dev/null || true)"
 REMOTE_PORT="$(cat "$PORT_FILE" 2>/dev/null || true)"
 REMOTE_MANAGED="$(cat "$MANAGED_FILE" 2>/dev/null || true)"
@@ -691,6 +711,29 @@ else
   REMOTE_PORT=""
   REMOTE_MANAGED=""
 fi
+# A background service has no server for a few seconds while it restarts or
+# updates. A server started in that gap owns the home and locks the service
+# out until someone stops it, so give the service time to come back first.
+if [ -z "$REMOTE_PORT" ] && boot_service_loaded; then
+  SERVICE_WAIT_COUNT=0
+  while [ "$SERVICE_WAIT_COUNT" -lt "@@T3_SERVICE_WAIT_SECONDS@@" ]; do
+    DEFAULT_RUNTIME_INFO="$(resolve_default_runtime_port 2>/dev/null || true)"
+    if [ -n "$DEFAULT_RUNTIME_INFO" ]; then
+      REMOTE_PORT="\${DEFAULT_RUNTIME_INFO#* }"
+      if wait_ready "@@T3_REUSE_READY_TIMEOUT_MS@@"; then
+        REMOTE_PID=""
+        REMOTE_MANAGED="external"
+        rm -f "$PID_FILE"
+        printf '%s\\n' "$REMOTE_PORT" >"$PORT_FILE"
+        printf 'external\\n' >"$MANAGED_FILE"
+        break
+      fi
+      REMOTE_PORT=""
+    fi
+    SERVICE_WAIT_COUNT=$((SERVICE_WAIT_COUNT + 1))
+    sleep 1
+  done
+fi
 if [ -z "$REMOTE_PORT" ]; then
   REMOTE_PORT="$(pick_port)" || true
   if [ -z "$REMOTE_PORT" ]; then
@@ -825,7 +868,7 @@ export function buildRemoteNodeEnvScript(input?: RemoteT3RunnerOptions): string
   );
 }
 
-function buildRemoteLaunchScript(input?: RemoteT3RunnerOptions): string {
+export function buildRemoteLaunchScript(input?: RemoteT3RunnerOptions): string {
   return applyScriptPlaceholders(REMOTE_LAUNCH_SCRIPT, {
     T3_ARCHIVE_MODE: isNodeScriptRunner(input) ? "0" : "1",
     T3_NODE_ENV_SCRIPT: buildRemoteNodeEnvScript(input),
@@ -836,6 +879,7 @@ function buildRemoteLaunchScript(input?: RemoteT3RunnerOptions): string {
     T3_REMOTE_PORT_SCAN_WINDOW: String(REMOTE_PORT_SCAN_WINDOW),
     T3_READY_TIMEOUT_MS: String(REMOTE_READY_TIMEOUT_MS),
     T3_REUSE_READY_TIMEOUT_MS: String(REMOTE_REUSE_READY_TIMEOUT_MS),
+    T3_SERVICE_WAIT_SECONDS: String(REMOTE_SERVICE_WAIT_SECONDS),
     T3_READY_PROBE_TIMEOUT_MS: String(SSH_READY_PROBE_TIMEOUT_MS),
   });
 }