diff --git a/docs/internals/remote.md b/docs/internals/remote.md index 0c328c50317c5..048fbde6c88a4 100644 --- a/docs/internals/remote.md +++ b/docs/internals/remote.md @@ -165,7 +165,11 @@ renderer uses the forwarded endpoint through the shared connection runtime. [SSH cleanup](../../packages/ssh/src/tunnel.ts) stops a remote server only if the launcher owns it; a server it discovered already running must survive a client disconnect. Reconnection restores the forward before opening the application -transport. +transport. When the remote home has a loaded background service, the launch +waits for that service's server before starting one: the service has no server +for a few seconds while it restarts or updates, and a server started in that gap +takes the [ownership lock](../../apps/server/src/serverOwnership.ts) and leaves +the service idle until someone stops it. Remote servers can outlive several client releases. Clients must use advertised capabilities and handle their absence, rather than assume their own version diff --git a/packages/ssh/src/launchScript.test.ts b/packages/ssh/src/launchScript.test.ts new file mode 100644 index 0000000000000..25f606df45b1f --- /dev/null +++ b/packages/ssh/src/launchScript.test.ts @@ -0,0 +1,162 @@ +import { assert, describe, it } from "@effect/vitest"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Path from "effect/Path"; +import * as Stream from "effect/Stream"; +import { ChildProcess, ChildProcessSpawner } from "effect/process"; + +import { buildRemoteLaunchScript } from "./tunnel.ts"; + +// Stands in for both the background service's server and a server the launch +// starts itself, so the real launch script runs end to end under `sh`. +const FAKE_SERVER = `const fs = require("node:fs"); +const http = require("node:http"); +const path = require("node:path"); +const fakeDir = process.env.FAKE_DIR; +const args = process.argv.slice(2); +const server = http.createServer((_request, response) => response.end("ok")); +if (args[0] === "service") { + const home = args[1]; + server.listen(0, "127.0.0.1", () => { + const port = server.address().port; + fs.mkdirSync(path.join(home, "userdata"), { recursive: true }); + fs.writeFileSync( + path.join(home, "userdata", "server-runtime.json"), + JSON.stringify({ pid: process.pid, port, origin: "http://127.0.0.1:" + port }), + ); + fs.writeFileSync(path.join(fakeDir, "service.pid"), String(process.pid)); + }); +} else { + const port = Number(args[args.indexOf("--port") + 1]); + server.listen(port, "127.0.0.1", () => { + fs.writeFileSync(path.join(fakeDir, "launched.pid"), String(process.pid)); + }); +} +`; + +// Reports the unit as activating and brings its server up, the way a service +// looks while it switches servers during a restart or update. +const FAKE_SYSTEMCTL = `#!/bin/sh +if [ -f "$FAKE_DIR/service-installed" ]; then + if [ ! -f "$FAKE_DIR/service-started" ]; then + : >"$FAKE_DIR/service-started" + node "$FAKE_SERVER" service "$HOME/.t3" >/dev/null 2>&1 & + fi + echo activating + exit 3 +fi +echo inactive +exit 3 +`; + +const collectText = (stream: Stream.Stream) => + stream.pipe( + Stream.decodeText(), + Stream.runFold( + () => "", + (text, chunk) => text + chunk, + ), + ); + +const launchOnFakeRemote = Effect.fn("launchOnFakeRemote")(function* (options: { + readonly withService: boolean; +}) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-ssh-launch-" }); + const home = path.join(root, "home"); + const fakeDir = path.join(root, "fake"); + const bin = path.join(root, "bin"); + for (const dir of [home, fakeDir, bin]) yield* fs.makeDirectory(dir, { recursive: true }); + const fakeServer = path.join(root, "fake-server.cjs"); + yield* fs.writeFileString(fakeServer, FAKE_SERVER); + yield* fs.writeFileString(path.join(bin, "systemctl"), FAKE_SYSTEMCTL); + yield* fs.chmod(path.join(bin, "systemctl"), 0o755); + if (options.withService) { + yield* fs.makeDirectory(path.join(home, ".t3", "runtime"), { recursive: true }); + yield* fs.writeFileString( + path.join(home, ".t3", "runtime", "service-state.json"), + JSON.stringify({ protocol: 3, activeVersion: "1.2.3" }), + ); + yield* fs.writeFileString(path.join(fakeDir, "service-installed"), ""); + } + + const pidAt = (name: string) => + fs.readFileString(path.join(fakeDir, name)).pipe( + Effect.map(Number), + Effect.orElseSucceed(() => undefined), + ); + // Registered after the temp directory, so it stops the servers before the directory goes. + yield* Effect.addFinalizer(() => + Effect.forEach(["service.pid", "launched.pid"], (name) => + pidAt(name).pipe( + Effect.flatMap((pid) => + Effect.sync(() => { + if (pid === undefined) return; + try { + process.kill(pid); + } catch {} + }), + ), + ), + ), + ); + + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const child = yield* spawner.spawn( + ChildProcess.make("sh", ["-s", "--", "test-state-key"], { + env: { + HOME: home, + PATH: [bin, path.dirname(process.execPath), "/usr/bin", "/bin"].join(":"), + FAKE_DIR: fakeDir, + FAKE_SERVER: fakeServer, + }, + extendEnv: false, + stdin: { + stream: Stream.make( + new TextEncoder().encode(buildRemoteLaunchScript({ nodeScriptPath: fakeServer })), + ), + endOnDone: true, + }, + }), + ); + const [stdout, stderr, exitCode] = yield* Effect.all( + [collectText(child.stdout), collectText(child.stderr), child.exitCode], + { concurrency: "unbounded" }, + ); + assert.equal(Number(exitCode), 0, stderr); + const result = JSON.parse(stdout.trim().split("\n").at(-1) ?? "") as { + readonly remotePort: number; + readonly serverKind: string; + }; + const runtime = yield* fs + .readFileString(path.join(home, ".t3", "userdata", "server-runtime.json")) + .pipe( + Effect.map((text) => JSON.parse(text) as { readonly port: number }), + Effect.orElseSucceed(() => undefined), + ); + return { result, servicePort: runtime?.port, launchedPid: yield* pidAt("launched.pid") }; +}); + +describe.skipIf(HostProcess.Platform.defaultValue() === "win32")("remote launch script", () => { + it.effect("waits for a restarting background service instead of starting its own server", () => + Effect.gen(function* () { + const { result, servicePort, launchedPid } = yield* launchOnFakeRemote({ + withService: true, + }); + assert.equal(result.serverKind, "external"); + assert.equal(result.remotePort, servicePort); + assert.isUndefined(launchedPid); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + + it.effect("starts its own server right away when the home has no background service", () => + Effect.gen(function* () { + const { result, launchedPid } = yield* launchOnFakeRemote({ withService: false }); + assert.equal(result.serverKind, "managed"); + assert.isDefined(launchedPid); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); +}); diff --git a/packages/ssh/src/tunnel.ts b/packages/ssh/src/tunnel.ts index 1a98fa7d21a7d..1fa59f25e893a 100644 --- a/packages/ssh/src/tunnel.ts +++ b/packages/ssh/src/tunnel.ts @@ -51,7 +51,8 @@ const SSH_READY_PROBE_TIMEOUT_MS = 1_000; const TUNNEL_SHUTDOWN_TIMEOUT_MS = 2_000; const TUNNEL_STDERR_TAIL_CHARS = 16_384; const REMOTE_READY_TIMEOUT_MS = 60_000; -const REMOTE_LAUNCH_TIMEOUT_MS = 90_000; +// Covers the wait for a restarting background service plus our own server's readiness. +const REMOTE_LAUNCH_TIMEOUT_MS = 120_000; // Cold archive launches download and extract ~70 MB and may wait on an install lock. // Lock waits must outlast both downloads and extraction; SSH must outlast install or wait plus readiness. // Leave slack for verification and extraction, which have no timeout. @@ -60,6 +61,8 @@ const REMOTE_ARCHIVE_DOWNLOAD_SECONDS = 240; const REMOTE_ARCHIVE_LOCK_WAIT_SECONDS = 360; const REMOTE_ARCHIVE_LAUNCH_TIMEOUT_MS = 900_000; const REMOTE_REUSE_READY_TIMEOUT_MS = 2_000; +// How long a launch waits for this home's restarting background service before starting its own server. +const REMOTE_SERVICE_WAIT_SECONDS = 30; export interface RemoteT3RunnerOptions { readonly nodeScriptPath?: string | null; @@ -619,6 +622,23 @@ try { } NODE } +# True while this home's background service (\`t3 service install\`) is loaded, +# including between its servers. Names match apps/server/src/cloud/bootService.ts; +# service-state.json ties the service to this home. +boot_service_loaded() { + [ -f "$DEFAULT_SERVER_HOME/runtime/service-state.json" ] || return 1 + if command -v systemctl >/dev/null 2>&1; then + case "$(systemctl --user is-active t3code.service 2>/dev/null || true)" in + active | activating | reloading) return 0 ;; + esac + return 1 + fi + if command -v launchctl >/dev/null 2>&1; then + launchctl print "gui/$(id -u)/com.t3tools.t3code.service" >/dev/null 2>&1 + return + fi + return 1 +} REMOTE_PID="$(cat "$PID_FILE" 2>/dev/null || true)" REMOTE_PORT="$(cat "$PORT_FILE" 2>/dev/null || true)" REMOTE_MANAGED="$(cat "$MANAGED_FILE" 2>/dev/null || true)" @@ -691,6 +711,29 @@ else REMOTE_PORT="" REMOTE_MANAGED="" fi +# A background service has no server for a few seconds while it restarts or +# updates. A server started in that gap owns the home and locks the service +# out until someone stops it, so give the service time to come back first. +if [ -z "$REMOTE_PORT" ] && boot_service_loaded; then + SERVICE_WAIT_COUNT=0 + while [ "$SERVICE_WAIT_COUNT" -lt "@@T3_SERVICE_WAIT_SECONDS@@" ]; do + DEFAULT_RUNTIME_INFO="$(resolve_default_runtime_port 2>/dev/null || true)" + if [ -n "$DEFAULT_RUNTIME_INFO" ]; then + REMOTE_PORT="\${DEFAULT_RUNTIME_INFO#* }" + if wait_ready "@@T3_REUSE_READY_TIMEOUT_MS@@"; then + REMOTE_PID="" + REMOTE_MANAGED="external" + rm -f "$PID_FILE" + printf '%s\\n' "$REMOTE_PORT" >"$PORT_FILE" + printf 'external\\n' >"$MANAGED_FILE" + break + fi + REMOTE_PORT="" + fi + SERVICE_WAIT_COUNT=$((SERVICE_WAIT_COUNT + 1)) + sleep 1 + done +fi if [ -z "$REMOTE_PORT" ]; then REMOTE_PORT="$(pick_port)" || true if [ -z "$REMOTE_PORT" ]; then @@ -825,7 +868,7 @@ export function buildRemoteNodeEnvScript(input?: RemoteT3RunnerOptions): string ); } -function buildRemoteLaunchScript(input?: RemoteT3RunnerOptions): string { +export function buildRemoteLaunchScript(input?: RemoteT3RunnerOptions): string { return applyScriptPlaceholders(REMOTE_LAUNCH_SCRIPT, { T3_ARCHIVE_MODE: isNodeScriptRunner(input) ? "0" : "1", T3_NODE_ENV_SCRIPT: buildRemoteNodeEnvScript(input), @@ -836,6 +879,7 @@ function buildRemoteLaunchScript(input?: RemoteT3RunnerOptions): string { T3_REMOTE_PORT_SCAN_WINDOW: String(REMOTE_PORT_SCAN_WINDOW), T3_READY_TIMEOUT_MS: String(REMOTE_READY_TIMEOUT_MS), T3_REUSE_READY_TIMEOUT_MS: String(REMOTE_REUSE_READY_TIMEOUT_MS), + T3_SERVICE_WAIT_SECONDS: String(REMOTE_SERVICE_WAIT_SECONDS), T3_READY_PROBE_TIMEOUT_MS: String(SSH_READY_PROBE_TIMEOUT_MS), }); }