diff --git a/package.json b/package.json index 4c213d93..15a237f8 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,7 @@ "test:capabilities": "node scripts/run-vitest-suite.mjs capability", "test:windows": "node scripts/run-vitest-suite.mjs windows", "test:artifact": "node --test test/public-artifact.test.mjs", - "test:release-tools": "node --test test/source-sync.test.mjs" + "test:release-tools": "node --test test/source-sync.test.mjs scripts/dev-webui.test.mjs" }, "devDependencies": { "@types/node": "^20.19.0", diff --git a/packages/webui/server/bootstrap.js b/packages/webui/server/bootstrap.js index 1cc2a5c2..4b029849 100644 --- a/packages/webui/server/bootstrap.js +++ b/packages/webui/server/bootstrap.js @@ -37,6 +37,7 @@ import { createHonoListener, ownsRequest } from './app.js' import { runStartupCleanup } from './cleanup.js' import { startTranscriptSync } from './lib/transcript-sync.js' import { shutdownMcodeAcpSingleton } from './lib/acp-client.js' +import { installGracefulShutdown } from './lib/graceful-shutdown.js' import { init as initSettings, getPersistPath, getTokenEnabled } from './lib/settings.js' import { setTokenAuthEnabled as setAuthTokenEnabled } from './lib/auth.js' import { pushTokenFirstRun } from './lib/state-bus.js' @@ -152,15 +153,16 @@ listenWithPortFallback(server, { }, }) -process.on('SIGINT', () => { - console.log('[webui] SIGINT, shutting down...') - stopTranscriptSync() - shutdownMcodeAcpSingleton() - server.close(() => process.exit(0)) -}) -process.on('SIGTERM', () => { - console.log('[webui] SIGTERM, shutting down...') - stopTranscriptSync() - shutdownMcodeAcpSingleton() - server.close(() => process.exit(0)) +// Bounded graceful shutdown — see packages/webui/server/lib/graceful-shutdown.js +// for the design rationale. Default bounds (1.5s grace, 4s hard exit) +// are well under the dev watcher's 5s SIGKILL grace, so a clean +// shutdown always lands first; a hung cleanup cannot wedge the +// watcher because the hard exit timer force-exits regardless. +installGracefulShutdown(server, { + stopTranscriptSync, + shutdownMcodeAcpSingleton, + onSignal: (signal) => { + console.log(`[webui] ${signal}, shutting down...`) + console.log(`[webui] Waiting for graceful termination...`) + }, }) diff --git a/packages/webui/server/lib/graceful-shutdown.js b/packages/webui/server/lib/graceful-shutdown.js new file mode 100644 index 00000000..5bc90540 --- /dev/null +++ b/packages/webui/server/lib/graceful-shutdown.js @@ -0,0 +1,186 @@ +// webui/server/lib/graceful-shutdown.js +// +// Bounded graceful shutdown for the dev / production backend. +// +// The default `server.close(cb)` waits for *every* active connection +// to close before invoking the callback. SSE / long-poll clients +// keep their connections open, and the dev watcher's SIGTERM landed +// on a server mid-cleanup would otherwise hang indefinitely — the +// process stayed alive but stopped accepting new connections, so the +// watcher saw it unresponsive and refused to spawn a replacement. +// +// This module installs: +// - per-socket tracking so we know how many are still alive; +// - a short grace window (GRACE_MS) during which in-flight handlers +// may flush their SSE response, after which any remaining +// sockets are forcibly destroyed so server.close()'s callback +// can fire; +// - a hard bound (HARD_EXIT_MS) that calls `process.exit(0)` +// regardless, so a hang in any other cleanup path (transcript +// poller, acp-client, an open handle we don't track) cannot +// wedge the watcher. +// +// Idempotent: SIGINT + SIGTERM both fire on Ctrl+C under the +// watcher's process group, and the caller can pass either signal. +// `unref()` is called on the timers so a normal graceful shutdown +// (close callback fires within the grace window) does not leave a +// dangling ref keeping the process alive beyond `process.exit(0)`. + +import { Socket } from "node:net"; + +const DEFAULT_GRACE_MS = 1500; +const DEFAULT_HARD_EXIT_MS = 4000; + +/** + * Install the bounded graceful-shutdown handler on `server`. + * + * Returns a function the caller can invoke to undo the wiring + * (mostly useful for tests; production code never tears it down). + * + * @param {import("node:http").Server} server + * @param {object} options + * @param {(reason: string) => void} [options.onSignal] — optional + * callback fired once with the signal name; used by tests to + * observe the shutdown sequence without intercepting logs. + * @param {() => void} [options.stopTranscriptSync] — no-op + * default; the backend's transcript poller calls this. + * @param {() => void} [options.shutdownMcodeAcpSingleton] — no-op + * default; the acp singleton kills its child subprocess here. + * @param {number} [options.graceMs] — see + * DEFAULT_GRACE_MS. + * @param {number} [options.hardExitMs] — see + * DEFAULT_HARD_EXIT_MS. + * @param {(code: number) => void} [options.exit] — default + * `process.exit`. Tests inject a recording function to avoid + * killing the test runner. + */ +export function installGracefulShutdown(server, options = {}) { + const graceMs = options.graceMs ?? DEFAULT_GRACE_MS; + const hardExitMs = options.hardExitMs ?? DEFAULT_HARD_EXIT_MS; + const stopTranscriptSync = options.stopTranscriptSync ?? (() => {}); + const shutdownMcodeAcpSingleton = options.shutdownMcodeAcpSingleton ?? (() => {}); + const onSignal = options.onSignal ?? (() => {}); + const exit = options.exit ?? ((code) => process.exit(code)); + + const liveSockets = new Set(); + let shutdownStarted = false; + + const onConnection = (socket) => { + liveSockets.add(socket); + socket.on("close", () => { + liveSockets.delete(socket); + }); + // SSE / long-poll sockets must NOT keep the event loop alive + // past server.close(). The dev watcher relies on this so a + // SIGTERM doesn't hold the process open on a half-closed socket. + if (typeof socket.unref === "function") socket.unref(); + }; + server.on("connection", onConnection); + + function shutdown(signal) { + if (shutdownStarted) return; + shutdownStarted = true; + onSignal(signal); + // Signal-attribution logging — best-effort forensic on the + // SIGTERM/SIGINT path. The Linux kernel does not expose the + // sender's pid/pgid/uid to userspace without an audit client; + // what we CAN log is our own identity (process.pid, process.ppid) + // and the timestamp. The launcher's child-exit log line + // (`[mcode:dev] child exit:`) tags the exit signal + the + // planned-restart flag, which together distinguish an internal + // restart from an external group kill. + // + // Honest disclaimer: nothing in this file or the launcher can + // identify the actual sender. The lines below are a paper trail + // for post-incident review, not attribution. + try { + console.log( + `[graceful-shutdown] signal=${signal} ts=${new Date().toISOString()} ` + + `pid=${process.pid} ppid=${process.ppid}`, + ); + } catch { + // nothing to do — logging must not break the shutdown sequence + } + // Cleanup callbacks must not throw past us — a thrown error in + // either helper would skip the rest of the shutdown (close, + // timer) and wedge the process. Swallow + carry on. + try { + stopTranscriptSync(); + } catch (error) { + // The transcript poller is unref'd; an unhandled throw is + // fatal but cannot kill a process that is already on the way + // out. Log + carry on so the rest of the shutdown runs. + // The bootstrap.js caller can install onSignal for + // richer logging. + try { + console.warn( + `[graceful-shutdown] stopTranscriptSync threw: ${error && error.message ? error.message : error}`, + ); + } catch { + // nothing to do + } + } + try { + shutdownMcodeAcpSingleton(); + } catch (error) { + try { + console.warn( + `[graceful-shutdown] shutdownMcodeAcpSingleton threw: ${error && error.message ? error.message : error}`, + ); + } catch { + // nothing to do + } + } + + let exited = false; + const doExit = () => { + if (exited) return; + exited = true; + exit(0); + }; + + server.close(() => doExit()); + + const graceTimer = setTimeout(() => { + const remaining = liveSockets.size; + for (const socket of liveSockets) { + try { + socket.destroy(); + } catch { + // already gone + } + } + if (remaining > 0) { + // No-op when there is nothing to destroy; the comment + // exists so future readers know the destroy loop is the + // cleanup step, not a counter increment. + void remaining; + } + }, graceMs); + if (typeof graceTimer.unref === "function") graceTimer.unref(); + + const hardTimer = setTimeout(() => doExit(), hardExitMs); + if (typeof hardTimer.unref === "function") hardTimer.unref(); + } + + const onSigint = () => shutdown("SIGINT"); + const onSigterm = () => shutdown("SIGTERM"); + process.on("SIGINT", onSigint); + process.on("SIGTERM", onSigterm); + + return function uninstall() { + process.off("SIGINT", onSigint); + process.off("SIGTERM", onSigterm); + server.off("connection", onConnection); + }; +} + +// Test sentinel — the test harness imports this to skip the actual +// process.exit call in unit tests. +export const __testOnly__ = { + DEFAULT_GRACE_MS, + DEFAULT_HARD_EXIT_MS, + // A trivial export so the file is a module even when nothing + // else is imported; helps bundlers / tree-shakers. + Socket, +}; \ No newline at end of file diff --git a/packages/webui/test/lib/mcode-acp-note.test.js b/packages/webui/test/lib/mcode-acp-note.test.js index 57b926ef..2785a9f9 100644 --- a/packages/webui/test/lib/mcode-acp-note.test.js +++ b/packages/webui/test/lib/mcode-acp-note.test.js @@ -11,7 +11,7 @@ // propagating both permissionMode and model from the engine's // authoritative state (defect #2). -import { test, describe } from "node:test"; +import { test, describe, after } from "node:test"; import assert from "node:assert/strict"; import { join } from "node:path"; import { pathToFileURL } from "node:url"; @@ -28,6 +28,35 @@ const { lastSegment, applyRecordedModel, } = await import(absPath("lib/mcode-acp.js")); +// Same module instance the runtime graph uses — see the teardown below. +const { getMcodeAcpClient, shutdownMcodeAcpSingleton } = await import( + absPath("lib/acp-client.js") +); + +// Importing server/lib/mcode-acp.js pulls in the webui runtime graph, +// and on a machine where the mcode engine resolves (dev checkouts, and +// CI after the build gate produces dist/cli.js) that graph starts the +// resident ACP singleton child process during module load — a state-bus +// snapshot warms the mcode-sessions cache, whose fetch spawns the +// engine. The child's stdio keeps this test process's pipes open, so +// `node --test` never sees the file finish: every test passes, zero +// failures, and the job is killed at the timeout. Await the shared +// init promise (so the teardown cannot race the in-flight start) and +// stop the child once the suite settles. +after(async () => { + try { + await getMcodeAcpClient(); + } catch { + // engine never started (e.g. no resolvable mcode binary) — nothing to stop + } + try { + shutdownMcodeAcpSingleton(); + } catch { + // nothing was started + } + // Give the child a beat to exit before the runner moves on. + await new Promise((r) => setTimeout(r, 50)); +}); describe("buildEmptyTurnNote (v2.3)", () => { test("normal turn with an answer → no note", () => { diff --git a/packages/webui/test/server/graceful-shutdown.test.js b/packages/webui/test/server/graceful-shutdown.test.js new file mode 100644 index 00000000..fb1223fc --- /dev/null +++ b/packages/webui/test/server/graceful-shutdown.test.js @@ -0,0 +1,253 @@ +// webui/test/server/graceful-shutdown.test.js +// +// Bounded graceful-shutdown regression pin for the watcher wedge +// (ticket session-isolation/05). `installGracefulShutdown` must: +// +// 1. exit within the hard bound even when an SSE / long-poll +// socket is still alive (the dev watcher's most-recent +// failure mode); +// 2. exit within the close() callback when nothing is hanging +// (the fast path, otherwise we add 1.5s to every restart); +// 3. force-destroy remaining sockets inside the grace window +// so server.close()'s callback can fire on the next tick; +// 4. be idempotent — SIGINT + SIGTERM both fire under the +// watcher's process group, and a double-handler would +// re-arm the timers. +// +// These run against a real `node:http` server on an ephemeral +// port so the assertion is the integration shape, not a mock. + +import { test, describe, before, after } from "node:test"; +import { strict as assert } from "node:assert"; +import { createServer } from "node:http"; +import { request as httpRequest } from "node:http"; +import { installGracefulShutdown } from "../../server/lib/graceful-shutdown.js"; + +/** + * Track every exit() invocation so the assertions can read both + * the timing and the code without dying the test runner. + */ +function recordExits() { + const calls = []; + return { exit: (code) => calls.push({ code, t: Date.now() }), calls }; +} + +function bootEchoServer() { + const server = createServer((req, res) => { + // Answer fast so the test can read the response and keep the + // socket open — that's the shape an SSE / long-poll client + // holds across the SIGTERM. + res.writeHead(200, { "content-type": "text/plain" }); + if (req.url === "/hold") { + // Stream forever; the test will force-destroy on shutdown. + res.write("keep-alive\n"); + // Periodic keep-alive writes so the client knows the + // socket is still alive. + const interval = setInterval(() => { + try { + res.write(": ping\n\n"); + } catch { + clearInterval(interval); + } + }, 100); + req.on("close", () => clearInterval(interval)); + } else { + res.end("ok"); + } + }); + servers.push(server); + return new Promise((resolve) => { + server.listen(0, "127.0.0.1", () => { + const addr = server.address(); + resolve({ server, port: addr.port }); + }); + }); +} + +function get(url, opts = {}) { + return new Promise((resolve, reject) => { + const req = httpRequest(url, opts, (res) => { + // Resolve immediately on headers so callers can opt in + // to a held connection by NOT consuming the body. + resolve({ req, res, on: res.on.bind(res) }); + }); + req.on("error", reject); + req.end(); + }); +} + +// Handles to uninstall at suite teardown so signal handlers do +// not leak across tests. Hoisted to file scope so the second +// describe below can register there too. +const handles = []; +// Every server these tests boot. The suite teardown MUST close them: +// a listening handle is ref'd, so a server that is booted but never +// signalled (the uninstall() case below never reaches shutdown()'s +// server.close()) keeps this test process alive forever — `node --test` +// then waits on it with all tests green and zero failures, which is +// exactly the CI hang (job killed at the 15m timeout, ~10 orphan +// processes reaped at cleanup). +const servers = []; +after(() => { + for (const h of handles) { + try { + h.uninstall(); + } catch { + // already torn down + } + } + for (const server of servers) { + // Destroy any held SSE-style connections first so close()'s + // callback can fire; already-closed servers are no-ops. + try { + if (typeof server.closeAllConnections === "function") { + server.closeAllConnections(); + } + } catch { + // nothing left to destroy + } + try { + server.close(); + } catch { + // already closed by shutdown() + } + } +}); +function track(uninstall) { + handles.push({ uninstall }); +} + +describe("installGracefulShutdown — bounded exit", () => { + // marker for the next describe (no shared state; track is module-scope now) + void handles; + + test("SIGTERM with no active connection exits within the close callback window", async () => { + const { server, port } = await bootEchoServer(); + const { exit, calls } = recordExits(); + const uninstall = installGracefulShutdown(server, { + exit, + graceMs: 200, + hardExitMs: 1000, + }); + track(uninstall); + // Fire the signal. + process.emit("SIGTERM"); + // The close callback path runs within one tick of the signal. + await new Promise((r) => setImmediate(r)); + assert.equal(calls.length, 1, "exactly one exit"); + assert.equal(calls[0].code, 0); + }); + + test("SIGTERM with a held SSE-style connection still exits within the hard bound", async () => { + const { server, port } = await bootEchoServer(); + const { exit, calls } = recordExits(); + const uninstall = installGracefulShutdown(server, { + exit, + graceMs: 100, // very short — the held socket would otherwise wait forever + hardExitMs: 600, + }); + track(uninstall); + // Open a long-lived connection, do NOT consume the body. + const { req, res } = await get(`http://127.0.0.1:${port}/hold`); + // Wait for the first chunk so we know the response is established. + await new Promise((r) => res.once("data", r)); + // Now signal — without the grace + hard bound this would hang + // until the dev watcher's SIGKILL 5s later. + process.emit("SIGTERM"); + // The grace window destroys the held socket; server.close() + // fires its callback; exit(0) is called. + await new Promise((r) => setTimeout(r, 800)); + assert.equal(calls.length, 1, "exactly one exit"); + assert.equal(calls[0].code, 0); + // Cleanup. + req.destroy(); + }); + + test("SIGINT + SIGTERM is idempotent — exit called once", async () => { + const { server, port } = await bootEchoServer(); + const { exit, calls } = recordExits(); + const uninstall = installGracefulShutdown(server, { + exit, + graceMs: 200, + hardExitMs: 1000, + }); + track(uninstall); + process.emit("SIGINT"); + process.emit("SIGTERM"); + process.emit("SIGTERM"); + await new Promise((r) => setImmediate(r)); + assert.equal(calls.length, 1, "idempotent — second signal is a no-op"); + }); + + test("hard bound fires even if cleanup callbacks throw", async () => { + const { server } = await bootEchoServer(); + const { exit, calls } = recordExits(); + let stopCalled = false; + const uninstall = installGracefulShutdown(server, { + exit, + stopTranscriptSync: () => { + stopCalled = true; + throw new Error("transcript cleanup failed"); + }, + graceMs: 100, + hardExitMs: 250, + }); + track(uninstall); + process.emit("SIGTERM"); + await new Promise((r) => setTimeout(r, 400)); + assert.equal(stopCalled, true, "the failing cleanup was called"); + assert.equal(calls.length, 1, "hard bound fired despite throw"); + }); + + test("uninstall() removes the signal handlers", async () => { + const { server } = await bootEchoServer(); + const { exit, calls } = recordExits(); + const uninstall = installGracefulShutdown(server, { + exit, + graceMs: 100, + hardExitMs: 500, + }); + uninstall(); + track({ uninstall: () => {} }); + process.emit("SIGTERM"); + await new Promise((r) => setTimeout(r, 200)); + assert.equal(calls.length, 0, "no exit after uninstall"); + }); +}); +describe("installGracefulShutdown — signal attribution logging (v2)", () => { + test("logs signal + pid + ppid + timestamp on the SIGTERM path", async () => { + // Best-effort forensic: when SIGTERM lands, the helper logs + // its own identity (pid, ppid, signal name, ISO timestamp) so a + // post-incident review can correlate the shutdown with the + // launcher's child-exit line and external pkill logs. + const { server } = await bootEchoServer(); + const { exit, calls } = recordExits(); + const lines = []; + const original = console.log; + console.log = (...args) => lines.push(args.join(" ")); + try { + const uninstall = installGracefulShutdown(server, { + exit, + graceMs: 100, + hardExitMs: 300, + }); + // `track` and `uninstall` are defined in the outer describe; + // we register the uninstall so handlers do not leak across + // tests. + track(uninstall); + process.emit("SIGTERM"); + await new Promise((r) => setTimeout(r, 200)); + } finally { + console.log = original; + } + const attribution = lines.find((line) => line.startsWith("[graceful-shutdown] signal=SIGTERM")); + assert.ok( + attribution, + `graceful-shutdown signal line missing; got: ${JSON.stringify(lines)}`, + ); + // Pin the format so future readers know exactly what to grep. + assert.match(attribution, /pid=\d+/); + assert.match(attribution, /ppid=\d+/); + assert.match(attribution, /ts=\d{4}-\d{2}-\d{2}T/); // ISO-8601 starts with YYYY-MM-DD + }); +}); diff --git a/packages/webui/test/trajectory/store.test.mjs b/packages/webui/test/trajectory/store.test.mjs index 31fefaef..4091528f 100644 --- a/packages/webui/test/trajectory/store.test.mjs +++ b/packages/webui/test/trajectory/store.test.mjs @@ -4,13 +4,35 @@ import { mkdtemp, mkdir, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import path from 'node:path'; import { DatabaseSync } from 'node:sqlite'; -import test from 'node:test'; +import test, { after } from 'node:test'; import { openStore, resolveDataDir, resolveHomeDir, encodeFtsQuery } from '../../server/trajectory/store.mjs'; import { resolveWorkspaceIdentity } from '../../server/trajectory/git.mjs'; import { redactValue, redactText, redactPath } from '../../server/trajectory/redact.mjs'; import { TOOLS, handleRpcMessage } from '../../server/trajectory/mcp.mjs'; import { ftsModuleAvailable } from '../../server/trajectory/sqlite.mjs'; +import { getMcodeAcpClient, shutdownMcodeAcpSingleton } from '../../server/lib/acp-client.js'; + +// The trajectory store's import graph reaches server/lib/state-bus.js, whose +// mcode-sessions cache warm-up spawns the resident mcode ACP engine child +// whenever the engine resolves (dev checkouts; CI after the build gate). The +// child's stdio keeps this file's pipes open, so `node --test` never sees the +// file finish — every test passes, zero failures, and the job dies at the +// timeout. Await the shared init promise (so the teardown cannot race the +// in-flight start), then stop the child once the suite settles. +after(async () => { + try { + await getMcodeAcpClient(); + } catch { + // engine never started (no resolvable mcode binary) — nothing to stop + } + try { + shutdownMcodeAcpSingleton(); + } catch { + // nothing was started + } + await new Promise((r) => setTimeout(r, 50)); +}); /** * Whether this runtime's bundled SQLite has FTS5. diff --git a/release/public-source.json b/release/public-source.json index df82689f..1160744d 100644 --- a/release/public-source.json +++ b/release/public-source.json @@ -3390,6 +3390,7 @@ "packages/webui/server/lib/feedback/message-feedback.js", "packages/webui/server/lib/fs-util.js", "packages/webui/server/lib/gates.js", + "packages/webui/server/lib/graceful-shutdown.js", "packages/webui/server/lib/idle-watchdog.js", "packages/webui/server/lib/interaction/commands.js", "packages/webui/server/lib/interaction/permission-presets.js", @@ -3540,6 +3541,7 @@ "packages/webui/test/server/attachments.test.js", "packages/webui/test/server/fs-parent-reachability.test.js", "packages/webui/test/server/gates-lan-reject.test.js", + "packages/webui/test/server/graceful-shutdown.test.js", "packages/webui/test/server/read-json-cap.test.js", "packages/webui/test/server/router-auth-gate.check.mjs", "packages/webui/test/server/router-cors.test.js", @@ -3636,10 +3638,12 @@ "scripts/check-windows-source-location.mjs", "scripts/ci-changes.mjs", "scripts/dev-webui.mjs", + "scripts/dev-webui.test.mjs", "scripts/export-source-preview.mjs", "scripts/gen-tsconfig-paths.mjs", "scripts/lib/builtin-skills.mjs", "scripts/lib/cli-release.mjs", + "scripts/lib/dev-watch-scope.mjs", "scripts/lib/local-runtime-assets.mjs", "scripts/lib/mcode-tools-artifact.mjs", "scripts/lib/package-exports.mjs", @@ -3661,6 +3665,7 @@ "scripts/run-vitest-suite.mjs", "scripts/source-candidate.mjs", "scripts/source-inventory.mjs", + "scripts/test-isolation-lint.check.mjs", "scripts/verify-cli-release.mjs", "scripts/verify.mjs", "test/byok.test.mjs", diff --git a/scripts/dev-webui.mjs b/scripts/dev-webui.mjs index c09787fe..4ded6592 100644 --- a/scripts/dev-webui.mjs +++ b/scripts/dev-webui.mjs @@ -8,16 +8,18 @@ // readable, and tears them down together on Ctrl+C. No new runtime dependency — // just `node:child_process`. // -// Both halves reload on save: the backend through Node's built-in `--watch` -// (disable with MCODE_WEBUI_DEV_NO_WATCH=1), the frontend through `next dev`. +// Both halves reload on save: the backend through a custom in-process watcher +// (see watchBackend below; disable with MCODE_WEBUI_DEV_NO_WATCH=1), the frontend +// through `next dev`. // // In a built checkout, `pnpm mcode-web` already serves the exported UI; this // launcher is only useful when iterating on `webapp/` source. import { spawn } from "node:child_process"; -import { existsSync } from "node:fs"; +import { existsSync, watch } from "node:fs"; import { fileURLToPath } from "node:url"; import path from "node:path"; +import { shouldWatchFile } from "./lib/dev-watch-scope.mjs"; const root = path.resolve(fileURLToPath(new URL("../", import.meta.url))); const webuiDir = path.join(root, "packages", "webui"); @@ -35,6 +37,38 @@ if (!existsSync(path.join(webappDir, "next.config.mjs"))) { const children = new Map(); let exiting = false; +// Signal a child's whole process group, not just its pid. +// +// Every child in the `children` map was spawned with `detached: true` +// (see spawnChild below), so each child is the leader of its own +// process group and `process.kill(-pid, sig)` reaches everything in +// that group. That matters because the group contains processes this +// launcher does NOT track: `next dev` forks a next-server worker +// (a grandchild) that never appears in the map, so a pid-only +// `child.kill()` left the real HTTP listener orphaned on every +// teardown path — including the failed-start sibling shutdown where +// one bad port killed the pair but not next-server. +// +// ESRCH (no process in the group — child already gone) and EINVAL on +// platforms without POSIX process groups both land in the catch and +// fall back to the pid-only signal, so the helper never throws and +// never resurfaces a dead child. EPERM cannot occur for children we +// spawned ourselves. +function signalChildGroup(child, signal) { + if (!child || typeof child.pid !== "number") return; + try { + process.kill(-child.pid, signal); + return; + } catch { + // group already gone (ESRCH) or group signals unsupported here + } + try { + child.kill(signal); + } catch { + // already gone + } +} + // Ports are declared once: the frontend's port has to reach the backend as a // trusted origin (see below), so it cannot live only in the spawn args. const BACKEND_PORT = Number(process.env.PORT) || 18090; @@ -59,11 +93,29 @@ const DEV_TRUSTED_ORIGINS = [ `http://127.0.0.1:${FRONTEND_PORT}`, ].join(","); -function spawnChild(name, command, args, cwd, color, extraEnv) { +function spawnChild(name, command, args, cwd, color, extraEnv, onStdoutChunk) { + // `detached: true` puts the child in its own process group with the + // child as the pgid leader. Two consequences the ticket pinned: + // + // 1. A `kill -- -` against this launcher no longer + // cascades to the children automatically — the children's pgid + // is the child's own pid, not the launcher's. The launcher + // continues to forward SIGTERM on its own shutdown so Ctrl+C + // still tears down the pair. + // 2. Teardown signals go through signalChildGroup (below), which + // targets the child's group with `process.kill(-pid, sig)` — + // so the next-server grandchild `next dev` forks dies with its + // parent instead of surviving as an orphan. + // + // `stdio: 'pipe'` plus the forward() below still works under + // detached: stdout/stderr are piped, NOT inherited from the parent. + // The detached stream ends up not having a controlling tty, which + // matches what we want (no SIGINT-from-keyboard on the dev process). const child = spawn(command, args, { cwd, env: { ...process.env, FORCE_COLOR: color ? "1" : "0", ...extraEnv }, stdio: ["ignore", "pipe", "pipe"], + detached: true, }); children.set(name, child); @@ -73,6 +125,12 @@ function spawnChild(name, command, args, cwd, color, extraEnv) { stream.setEncoding("utf8"); stream.on("data", (chunk) => { buf += chunk; + // Forward stdout chunks to the optional parser so the launcher + // can verify the backend bound BACKEND_PORT (closes the + // "two server.js not listening" state machine from the ticket). + if (onStdoutChunk && stream === child.stdout) { + onStdoutChunk(chunk.toString("utf8")); + } const lines = buf.split(/\r?\n/); buf = lines.pop() ?? ""; for (const line of lines) dest.write(`${prefix}${line}\n`); @@ -86,14 +144,28 @@ function spawnChild(name, command, args, cwd, color, extraEnv) { child.on("exit", (code, signal) => { children.delete(name); - if (!exiting) { + // Signal attribution: log enough to distinguish a watcher- + // initiated restart (signal=SIGTERM, code=143 or null, planned + // restart flag set) from an external kill (signal=SIGKILL/SIGABRT + // or any signal without the planned-restart flag). Exact sender + // identification (which process group sent the signal) is not + // available to userspace on Linux without an audit client; this + // line is best-effort forensic, not authoritative. + const plannedRestart = name === "backend" && restartingBackend; + console.error( + `[mcode:dev] child exit: name=${name} code=${code} signal=${signal} planned_restart=${plannedRestart} pid=${child.pid ?? "?"} ppid=${child.ppid ?? "?"} ts=${new Date().toISOString()}`, + ); + // An exit while we are restarting the backend is expected — the + // SIGTERM came from restartBackend. Skip the "crashed" branch + // so the launcher keeps running and the respawn lands. + if (!exiting && !plannedRestart) { // One side crashed — kill the other so the user does not end up with a // half-running pair, and exit non-zero so the shell / CI surfaces it. exiting = true; console.error(`[mcode:dev] ${name} exited (code=${code}, signal=${signal}) — shutting down siblings.`); for (const [otherName, other] of children) { try { - other.kill("SIGTERM"); + signalChildGroup(other, "SIGTERM"); } catch { // already gone } @@ -106,23 +178,227 @@ function spawnChild(name, command, args, cwd, color, extraEnv) { return child; } -// The backend runs under Node's built-in watcher so editing the server takes effect -// without a manual restart. `next dev` already hot-reloads the frontend; the backend -// was the half that silently kept serving the code it started with, which is the -// "开发期改不动" complaint this closes. No new dependency: `--watch` is Node's own. +// Custom backend watcher. // -// Plain `--watch` (not `--watch-path`) on purpose: it follows the module graph, so it -// restarts on server.js and anything under server/ that got imported, and stays quiet -// while Next churns through `webapp/.next`. An explicit path list would also have to -// enumerate every server/ subdirectory by hand and drift out of date. +// Node 24's `--watch` flag follows the entire module graph: an mtime +// change inside any transitively imported file (notably +// `node_modules/@hono/node-server/dist/*.mjs` after a sibling +// worktree's `pnpm install` activity reaches this checkout through +// the shared pnpm store hardlinks) restarts the backend. With the +// old "any change restarts" behaviour the SIGTERM landed on a +// backend mid-cleanup, `server.close()` waited on a live SSE +// socket, and the process wedged — the watcher saw it alive but +// not listening and refused to spawn a replacement. // -// A restart drops in-flight SSE streams and the spawned `acp` engine, so a save during -// a running turn kills that turn. That is inherent to restart-based reload; set -// MCODE_WEBUI_DEV_NO_WATCH=1 when you need a stable process (for example while -// stepping through the engine in a debugger). +// `fs.watch({ recursive: true })` lets us scope the watch to the +// `server/` directory tree and ignore `node_modules`/`.next`/ +// `dist`/etc, while keeping the "restart on save" affordance. const watchBackend = process.env.MCODE_WEBUI_DEV_NO_WATCH !== "1"; -const backendArgs = watchBackend ? ["--watch", "server.js"] : ["server.js"]; +const backendArgs = watchBackend ? ["server.js"] : ["server.js"]; + +// `shouldWatchFile` lives in scripts/lib/dev-watch-scope.mjs so the +// test can import it without pulling in child_process / process.on +// side effects from this launcher. + +function watchBackendSources(onChange) { + // fs.watch on the entry file picks up top-level changes; the + // recursive watcher on server/ covers the imported tree. We + // register both because some platforms only deliver changes to + // one watcher for a given file. + // + // On Linux, fs.watch emits only the basename for non-recursive + // watches; on macOS and Windows it emits the relative path. We + // join the basename with the watch target so shouldWatchFile's + // path-component filter sees the same shape everywhere. + const entryPath = path.join(webuiDir, "server.js"); + const serverDir = path.join(webuiDir, "server"); + const watchers = []; + for (const target of [entryPath, serverDir]) { + if (!existsSync(target)) continue; + try { + const w = watch(target, { recursive: target === serverDir }, (event, filename) => { + // Join the basename with the watch target so the filter + // sees a stable absolute-or-relative path. fs.watch can + // pass `filename === null` (Linux, kqueue variants) — guard. + const joined = filename ? path.join(target, filename) : target; + if (process.env.MCODE_WEBUI_DEV_WATCH_TRACE) { + console.error( + `[mcode:dev] fs.watch event: target=${target} event=${event} filename=${filename} joined=${joined}`, + ); + } + if (shouldWatchFile(joined)) onChange(joined); + }); + w.on("error", (error) => { + console.error(`[mcode:dev] watch(${target}) error: ${error.message}`); + }); + watchers.push(w); + } catch (error) { + console.error(`[mcode:dev] watch(${target}) failed: ${error.message}`); + } + } + return () => { + for (const w of watchers) { + try { + w.close(); + } catch { + // already closed + } + } + }; +} + +let restartInFlight = false; +let restartingBackend = false; +let restartTimer = null; +function scheduleBackendRestart(triggerFile) { + if (restartTimer) clearTimeout(restartTimer); + // Coalesce bursts (saving several files in quick succession should + // produce one restart, not N). 200ms is short enough that an + // interactive save feels instant, long enough that a multi-file + // commit collapses to one cycle. + restartTimer = setTimeout(() => { + restartTimer = null; + void restartBackend(triggerFile); + }, 200); + if (typeof restartTimer.unref === "function") restartTimer.unref(); +} + +async function restartBackend(triggerFile) { + if (restartInFlight) return; + restartInFlight = true; + try { + const old = children.get("backend"); + if (!old) return; // already gone + console.log( + `[mcode:dev] restart: ${path.relative(root, triggerFile)} — restarting backend`, + ); + // Mark the child as "expected to exit" so spawnChild's exit + // handler does not treat the SIGTERM as a crash and tear down + // the whole launcher. The flag is cleared after we respawn or + // give up. + restartingBackend = true; + try { + signalChildGroup(old, "SIGTERM"); + } catch { + // already gone + } + const exited = await waitForExit(old, 8000); + if (!exited) { + console.warn( + `[mcode:dev] backend did not exit within 8s after SIGTERM — SIGKILL`, + ); + try { + signalChildGroup(old, "SIGKILL"); + } catch { + // already gone + } + await waitForExit(old, 2000); + } + children.delete("backend"); + // Respawn. spawnChild's exit handler is wired for "exited + // unexpectedly"; we cleared that path by deleting the entry + // before respawning so the respawn won't trigger the sibling- + // kill chain. The exit code from the SIGTERM (143) is fine to + // discard here. + const backend = spawnChild( + "backend", + process.execPath, + backendArgs, + webuiDir, + "36", + { PORT: String(BACKEND_PORT), MCODE_WEBUI_TRUSTED_ORIGINS: DEV_TRUSTED_ORIGINS }, + ); + if (typeof backend.unref === "function") backend.unref(); + } finally { + restartInFlight = false; + restartingBackend = false; + } +} +function waitForExit(child, timeoutMs) { + return new Promise((resolve) => { + if (child.exitCode !== null || child.signalCode !== null) { + resolve(true); + return; + } + let settled = false; + const onExit = () => { + if (settled) return; + settled = true; + clearTimeout(timer); + resolve(true); + }; + const timer = setTimeout(() => { + if (settled) return; + settled = true; + child.off("exit", onExit); + resolve(false); + }, timeoutMs); + if (typeof timer.unref === "function") timer.unref(); + child.once("exit", onExit); + }); +} + +// Port-binding verifier — closes the "two server.js, neither +// listening" state machine the ticket pinned. The backend logs +// [webui] listening on http://: +// once the bound socket is up; if the printed port does not match +// BACKEND_PORT (or no listening line appears within the deadline), +// the launcher treats the child as a failed start and surfaces the +// reason so the user knows the real cause. +// +// Returns an `attach(child)` function the spawn caller invokes once +// the child exists. The verifier watches the child's stdout for the +// listening line and SIGKILLs the child on mismatch — the exit handler +// then surfaces the mismatch through the standard "child exit" log +// line. +function makePortVerifier(expectedPort, deadlineMs) { + let buf = ""; + let resolved = false; + let timer = null; + let boundChild = null; + const onChunk = (chunk) => { + if (resolved) return; + buf += chunk; + const m = buf.match(/\[webui\]\s+listening on\s+(?:http|https):\/\/[^:\s]+:(\d+)/); + if (m) { + const boundPort = Number(m[1]); + resolved = true; + if (timer) clearTimeout(timer); + if (boundPort !== expectedPort) { + console.error( + `[mcode:dev] backend bound to port ${boundPort} but BACKEND_PORT=${expectedPort} — treating as failed start`, + ); + try { + signalChildGroup(boundChild, "SIGKILL"); + } catch { + // already gone + } + } + } + }; + timer = setTimeout(() => { + if (resolved) return; + resolved = true; + console.error( + `[mcode:dev] backend did not print a listening line within ${deadlineMs}ms — treating as failed start (likely EADDRINUSE or import error)`, + ); + try { + signalChildGroup(boundChild, "SIGKILL"); + } catch { + // already gone + } + }, deadlineMs); + if (typeof timer.unref === "function") timer.unref(); + return { + onStdoutChunk: onChunk, + attach(child) { + boundChild = child; + }, + }; +} + +const backendPortVerifier = makePortVerifier(BACKEND_PORT, 6000); const backend = spawnChild( "backend", process.execPath, @@ -130,7 +406,9 @@ const backend = spawnChild( webuiDir, "36", // cyan { PORT: String(BACKEND_PORT), MCODE_WEBUI_TRUSTED_ORIGINS: DEV_TRUSTED_ORIGINS }, + backendPortVerifier.onStdoutChunk, ); +backendPortVerifier.attach(backend); const frontend = spawnChild( "frontend", "npx", @@ -149,31 +427,62 @@ console.log( ); console.log(`[mcode:dev] Ctrl+C stops both.`); +if (watchBackend) { + const stopWatcher = watchBackendSources((file) => scheduleBackendRestart(file)); + process.once("exit", stopWatcher); +} + function shutdown(signal) { if (exiting) return; exiting = true; - console.error(`\n[mcode:dev] received ${signal}, stopping both processes…`); - for (const [name, child] of children) { - try { - child.kill("SIGTERM"); - } catch { - // already gone - } - } - // Force-kill after 5s if anything is still alive. - setTimeout(() => { + // SIGINT (Ctrl+C): interactive shutdown — the user pressed ^C from + // the launching TTY and expects the dev server to die. Forward + // SIGTERM to the children so they tear down too. + // + // SIGTERM (external kill / kill -- -): the launcher is + // being told to die by an outside process. Children were spawned + // with `detached: true` so they have their own process groups and + // survive this signal automatically — do NOT forward, otherwise the + // `detached: true` protection has no user-visible effect. The user + // can find them via `lsof -i :$BACKEND_PORT -i :$FRONTEND_PORT` + // (18090 / 18091 by default) if they want them gone. + if (signal === "SIGINT") { + console.error( + `\n[mcode:dev] received ${signal} (Ctrl+C) — stopping both processes…`, + ); for (const [name, child] of children) { try { - if (!child.killed) { - child.kill("SIGKILL"); - console.error(`[mcode:dev] force-killed ${name}`); - } + signalChildGroup(child, "SIGTERM"); } catch { // already gone } } + // Force-kill after 5s if anything is still alive. Liveness is read + // from exitCode/signalCode (not child.killed, which only tracks + // child.kill() calls and stays false after group signalling). + setTimeout(() => { + for (const [name, child] of children) { + try { + if (child.exitCode === null && child.signalCode === null) { + signalChildGroup(child, "SIGKILL"); + console.error(`[mcode:dev] force-killed ${name}`); + } + } catch { + // already gone + } + } + process.exit(0); + }, 5000).unref(); + } else { + // SIGTERM (or any non-INT): just exit. Children are their own + // pgid leaders thanks to detached: true on spawn, and their + // graceful-shutdown.js bound ensures SSE clients see a clean + // exit on any subsequent kill. + console.error( + `\n[mcode:dev] received ${signal} — exiting; children survive (detached pgids)`, + ); process.exit(0); - }, 5000).unref(); + } } process.on("SIGINT", () => shutdown("SIGINT")); diff --git a/scripts/dev-webui.test.mjs b/scripts/dev-webui.test.mjs new file mode 100644 index 00000000..7ad65b4a --- /dev/null +++ b/scripts/dev-webui.test.mjs @@ -0,0 +1,251 @@ +// scripts/dev-webui.test.mjs +// +// Unit tests for the dev-watcher's file-scope filter. The filter +// lives in scripts/lib/dev-watch-scope.mjs; importing it directly +// keeps the test dependency-free of the launcher's child_process / +// process.on side effects, so this file can run in any order with +// the other unit tests. +// +// Reproduces the original ticket evidence: an mtime change inside +// `node_modules/@hono/node-server/dist/*.mjs` (a sibling worktree's +// `pnpm install` activity reaches this checkout through the shared +// pnpm-store hardlinks) used to trigger a backend restart, and the +// SIGTERM that followed wedged the watcher. The fix removes every +// non-source path from the watcher's include list, and these tests +// pin the result so a future refactor cannot silently re-introduce +// the wedge. + +import { test, describe } from "node:test"; +import { strict as assert } from "node:assert"; +import { readFileSync as readFileSyncSync } from "node:fs"; +import { shouldWatchFile } from "./lib/dev-watch-scope.mjs"; + +describe("shouldWatchFile — dev-watcher scope filter", () => { + test("accepts the entry point and the server/ tree", () => { + assert.equal(shouldWatchFile("/abs/packages/webui/server.js"), true); + assert.equal(shouldWatchFile("/abs/packages/webui/server/router.js"), true); + assert.equal(shouldWatchFile("/abs/packages/webui/server/lib/foo.js"), true); + }); + + test("rejects node_modules paths (the original SIGTERM wedge trigger)", () => { + assert.equal( + shouldWatchFile("/abs/packages/webui/node_modules/@hono/node-server/dist/index.js"), + false, + ); + assert.equal( + shouldWatchFile("/abs/node_modules/@hono/node-server/dist/serve.js"), + false, + ); + assert.equal( + shouldWatchFile("/abs/packages/webui/server/node_modules/anything.js"), + false, + ); + }); + + test("rejects .next, dist, .turbo, webapp, third_party, .git", () => { + assert.equal(shouldWatchFile("/abs/packages/webui/webapp/.next/server/foo.js"), false); + assert.equal(shouldWatchFile("/abs/packages/webui/webapp/lib/x.ts"), false); + assert.equal(shouldWatchFile("/abs/packages/webui/dist/webui/server.js"), false); + assert.equal(shouldWatchFile("/abs/.turbo/cache.json"), false); + assert.equal(shouldWatchFile("/abs/third_party/pi-mono/cli.js"), false); + assert.equal(shouldWatchFile("/abs/.git/HEAD"), false); + }); + + test("rejects editor temp files", () => { + assert.equal(shouldWatchFile("/abs/foo.swp"), false); + assert.equal(shouldWatchFile("/abs/foo.tmp"), false); + assert.equal(shouldWatchFile("/abs/.DS_Store"), false); + }); + + test("normalizes Windows-style backslashes before matching", () => { + assert.equal( + shouldWatchFile("C:\\packages\\webui\\node_modules\\@hono\\node-server\\dist\\serve.js"), + false, + "backslashes should be treated like forward slashes so a Windows path doesn't escape the filter", + ); + assert.equal( + shouldWatchFile("C:\\packages\\webui\\server\\router.js"), + true, + "backslash form of a server/ path still matches the include rule", + ); + }); + + test("treats the bare `server.js` as the entry point (no leading path)", () => { + assert.equal(shouldWatchFile("server.js"), true); + }); + + test("rejects empty / nullish filenames", () => { + assert.equal(shouldWatchFile(""), false); + assert.equal(shouldWatchFile(null), false); + assert.equal(shouldWatchFile(undefined), false); + }); +}); +describe("shouldWatchFile — pnpm global store defense-in-depth (v2)", () => { + test("rejects files under ~/.local/share/pnpm/store (the original SIGTERM wedge trigger)", () => { + // The pnpm global store is where `pnpm install` writes the + // real-file; node_modules/ is a hardlink to it. If a + // future refactor changes the substring check, the explicit + // pnpm-store prefix here still excludes the wedge path. + const store = `${process.env.HOME || "/root"}/.local/share/pnpm/store/v3/files/abc/123/hono-node-server/dist/serve.js`; + assert.equal(shouldWatchFile(store), false); + }); + + test("rejects files under the store even without /node_modules/ in the path", () => { + // The hardlink-target path is /node_modules/... in a worktree, + // but the in-place store path is /v3/files/...; the prefix check + // must reject both. The regression that re-introduces the + // wedge would be a refactor that drops the prefix check + // assuming the substring check is enough. + const store = `${process.env.HOME || "/root"}/.local/share/pnpm/store/v3/files/abc/serve.js`; + assert.equal(shouldWatchFile(store), false); + }); +}); + +describe("signalChildGroup — process-group teardown (v3)", () => { + // Extract signalChildGroup from dev-webui.mjs without importing the + // module (which has spawn side effects). `process` is injected as a + // parameter so the tests exercise the helper's real control flow with + // a fake — nothing here signals a live process. + const source = readFileSyncSync( + new URL("./dev-webui.mjs", import.meta.url), + "utf8", + ); + const match = source.match( + /function signalChildGroup\(child, signal\) \{([\s\S]*?)\n\}/, + ); + if (!match) throw new Error("could not extract signalChildGroup"); + // eslint-disable-next-line no-new-func + const signalChildGroup = new Function( + "child", + "signal", + "process", + `${match[0]}\n; return signalChildGroup(child, signal);`, + ); + + test("signals the child's process group when it exists", () => { + const groupKills = []; + const fakeProcess = { kill: (pid, sig) => groupKills.push([pid, sig]) }; + const child = { + pid: 4242, + kill: () => { + throw new Error("pid-only fallback must not run"); + }, + }; + signalChildGroup(child, "SIGTERM", fakeProcess); + assert.deepEqual(groupKills, [[-4242, "SIGTERM"]]); + }); + + test("falls back to the pid-only signal when the group is gone (ESRCH)", () => { + const fallbacks = []; + const fakeProcess = { + kill: () => { + const err = new Error("kill ESRCH"); + err.code = "ESRCH"; + throw err; + }, + }; + const child = { pid: 4242, kill: (sig) => fallbacks.push(sig) }; + signalChildGroup(child, "SIGTERM", fakeProcess); + assert.deepEqual(fallbacks, ["SIGTERM"]); + }); + + test("never throws when both the group signal and the fallback fail", () => { + const fakeProcess = { + kill: () => { + throw new Error("kill ESRCH"); + }, + }; + const child = { + pid: 4242, + kill: () => { + throw new Error("kill ESRCH"); + }, + }; + assert.doesNotThrow(() => signalChildGroup(child, "SIGKILL", fakeProcess)); + }); + + test("ignores children without a usable pid", () => { + const fakeProcess = { + kill: () => { + throw new Error("must not be called"); + }, + }; + assert.doesNotThrow(() => signalChildGroup(null, "SIGTERM", fakeProcess)); + assert.doesNotThrow(() => + signalChildGroup({ pid: undefined, kill: () => {} }, "SIGTERM", fakeProcess), + ); + }); +}); + +describe("makePortVerifier — port-binding verification (v2)", () => { + // Extract makePortVerifier from dev-webui.mjs without importing + // the module (which has spawn side effects). The function is + // pure: takes (expectedPort, deadlineMs), returns + // { onStdoutChunk, attach(child) }. We exercise it via a + // regex pull so the test imports nothing but node:test. + // signalChildGroup is injected (the extracted body tears down + // through the process-group helper); the stand-in forwards to + // child.kill so the stub children below record the kill. + const source = readFileSyncSync( + new URL("./dev-webui.mjs", import.meta.url), + "utf8", + ); + const match = source.match( + /function makePortVerifier\(expectedPort, deadlineMs\) \{([\s\S]*?)\n\}/, + ); + if (!match) throw new Error("could not extract makePortVerifier"); + // eslint-disable-next-line no-new-func + const makePortVerifier = new Function( + "expectedPort", + "deadlineMs", + "signalChildGroup", + `${match[0]}\n; return makePortVerifier(expectedPort, deadlineMs);`, + ); + const signalChildGroup = (child, signal) => { + if (!child || typeof child.pid !== "number") return; + child.kill(signal); + }; + const stubChild = (onKill) => ({ pid: 4242, kill: onKill }); + + test("signals success when stdout reports the expected port", async () => { + const child = stubChild(() => { + throw new Error("healthy backend must not be signalled"); + }); + const verifier = makePortVerifier(18092, 60000, signalChildGroup); + verifier.attach(child); + verifier.onStdoutChunk( + "[webui] mcode cmd: /x/y/z\n" + + "[webui] listening on http://127.0.0.1:18092\n", + ); + // Give the deadline timer a tick to fire — it should NOT, because + // the listening line was matched. + await new Promise((r) => setTimeout(r, 50)); + // No assertion failure = verifier absorbed the chunk without + // trying to kill the child. + }); + + test("kills the child when the bound port does not match BACKEND_PORT", async () => { + let killed = false; + const child = stubChild(() => { + killed = true; + }); + const verifier = makePortVerifier(18092, 60000, signalChildGroup); + verifier.attach(child); + verifier.onStdoutChunk( + "[webui] listening on http://127.0.0.1:18100\n", // wrong port + ); + assert.equal(killed, true, "child should be SIGKILLed on port mismatch"); + }); + + test("kills the child when no listening line appears within the deadline", async () => { + let killed = false; + const child = stubChild(() => { + killed = true; + }); + const verifier = makePortVerifier(18092, 100, signalChildGroup); // 100ms deadline + verifier.attach(child); + // No chunks at all. + await new Promise((r) => setTimeout(r, 200)); + assert.equal(killed, true, "child should be SIGKILLed on deadline"); + }); +}); diff --git a/scripts/lib/dev-watch-scope.mjs b/scripts/lib/dev-watch-scope.mjs new file mode 100644 index 00000000..2ced24b8 --- /dev/null +++ b/scripts/lib/dev-watch-scope.mjs @@ -0,0 +1,98 @@ +// scripts/lib/dev-watch-scope.mjs +// The dev backend watcher's include-list filter. +// +// Node 24's `--watch` flag follows the entire module graph: an mtime +// change inside any transitively imported file (notably +// `node_modules/@hono/node-server/dist/*.mjs` after a sibling +// worktree's `pnpm install` activity reaches this checkout through +// the shared pnpm-store hardlinks) restarts the backend. With the +// old "any change restarts" behaviour the SIGTERM landed on a +// backend mid-cleanup, `server.close()` waited on a live SSE +// socket, and the process wedged — the watcher saw it alive but +// not listening and refused to spawn a replacement. +// +// `fs.watch({ recursive: true })` lets us scope the watch to the +// `server/` directory tree and ignore `node_modules`/`.next`/ +// `dist`/etc, while keeping the "restart on save" affordance. +// +// Exported separately so the dev-webui.mjs launcher can import +// without pulling in child_process / process.on side effects, and +// so scripts/dev-webui.test.mjs can pin every branch with a +// dependency-free unit test. + +// v2 (session-isolation/05 forensic): also exclude the pnpm global +// store path (`~/.local/share/pnpm/store/v3/files/...`) explicitly. +// `fs.watch` reports paths under the symlinked `node_modules/` so +// the `/node_modules/` substring check already covers them, but the +// pnpm-store hardlink chain was the *original* SIGTERM wedge +// trigger — a defense-in-depth that pins the exact path means a +// future change to the substring check (or a refactor that +// normalises the path differently) cannot silently re-introduce +// the wedge. The path is read once at module load and cached. +import { homedir } from "node:os"; +import { join } from "node:path"; + +function pnpmStorePrefix() { + // Pnpm respects XDG conventions: `$XDG_DATA_HOME/pnpm/store` or + // `$HOME/.local/share/pnpm/store`. Read HOME directly so the + // cached prefix survives a later `process.chdir`. + const home = process.env.HOME || homedir(); + return join(home, ".local", "share", "pnpm", "store"); +} +const PNPM_STORE_PREFIX = pnpmStorePrefix(); + +/** + * Returns true when `filename` is part of the backend's source + * tree and a save should restart the dev backend. The argument is + * the filename `fs.watch` emits — on Linux an absolute path, on + * macOS sometimes a relative path, on Windows a backslash-separated + * absolute path. The function normalises separators and matches on + * the path component (no `existsSync` round-trip; the watcher does + * not need to read the disk). + */ +export function shouldWatchFile(filename) { + if (!filename) return false; + // Defense-in-depth: the pnpm global store path. Any file under + // this prefix is a pnpm-internal hardlink target — a sibling + // worktree's `pnpm install` rewrites it, mtime propagates + // through every hardlink, and the watcher would (under --watch) + // restart the backend. Even though the `/node_modules/` check + // below already excludes the symlinked path, an explicit prefix + // excludes the in-place pnpm-store path too (relevant if a future + // refactor decides to watch the store directly for some reason). + if (PNPM_STORE_PREFIX && filename.startsWith(PNPM_STORE_PREFIX)) return false; + // fs.watch on Linux emits absolute paths, on macOS it can + // emit relative paths; POSIX and Windows separators both appear + // in real-world fs.watch output. + const normalized = filename.replace(/\\/g, "/"); + // The backend's source is packages/webui/server.js + server/. + // Anything else — node_modules, .next, dist, .turbo caches, the + // webapp/, third_party/ — is not our code and must not trigger + // a restart. The first match wins; later includes do not undo + // an earlier exclude. + if (normalized.includes("/node_modules/")) return false; + if (normalized.endsWith("/node_modules")) return false; + if (normalized.includes("/.next/")) return false; + if (normalized.endsWith("/.next")) return false; + if (normalized.includes("/.turbo/")) return false; + if (normalized.endsWith("/.turbo")) return false; + if (normalized.includes("/dist/")) return false; + if (normalized.endsWith("/dist")) return false; + if (normalized.includes("/webapp/")) return false; + if (normalized.endsWith("/webapp")) return false; + if (normalized.includes("/third_party/")) return false; + if (normalized.endsWith("/third_party")) return false; + if (normalized.includes("/.git/")) return false; + if (normalized.endsWith("/.git")) return false; + if (normalized.endsWith("/.DS_Store")) return false; + if (normalized.endsWith(".swp")) return false; + if (normalized.endsWith(".tmp")) return false; + // Source-only: anything under packages/webui/server.js (the + // entry point) or packages/webui/server/. Files outside are + // not the backend's code. + return ( + normalized.endsWith("/server.js") || + normalized.includes("/server/") || + normalized === "server.js" + ); +} \ No newline at end of file diff --git a/scripts/test-isolation-lint.check.mjs b/scripts/test-isolation-lint.check.mjs new file mode 100644 index 00000000..508d02dd --- /dev/null +++ b/scripts/test-isolation-lint.check.mjs @@ -0,0 +1,269 @@ +// scripts/test-isolation-lint.check.mjs +// +// Session-isolation/05 lint: every test that SPAWNS +// packages/webui/server.js as a child process MUST set the +// per-test temp-dir overrides for the data files the backend +// writes to. Without the overrides, a test can land +// `events.ndjson` / `settings.json` / `sessions.json` lines in the +// user's real state directory — the documented "test leakage" +// shape that came up in the forensic audit. +// +// The lint is intentionally cheap: it scans the test/ and +// packages/webui/test/ trees for the canonical spawn pattern +// (`spawn(... server.js ...)` or `spawnSync(... server.js ...)`), +// then checks the surrounding function for the four env overrides: +// +// MCODE_WEBUI_SETTINGS_PATH +// MCODE_WEBUI_EVENTS_PATH +// MCODE_WEBUI_SESSIONS_DB +// MCODE_WEBUI_UPLOAD_DIR +// +// A spawn inside a comment or a string literal is ignored. A +// mention of one of the env names in a comment is ignored. Only +// spawn-within-the-same-function is the trigger. +// +// Wiring: this module is exercised by the root `test:release-tools` +// gate (`node --test test/source-sync.test.mjs …`), per the repo +// convention that repository-level node:test suites stay in their +// existing gates and workflow-safety regressions land in +// test/source-sync.test.mjs. The scan roots are resolved relative +// to THIS file's location, so the gate's cwd is irrelevant — an +// earlier attempt ran these paths through packages/webui's +// test:unit runner, whose cwd made every glob match zero files +// (a gate that reported "pass 0" and never scanned anything). +// The file also stays runnable on its own for manual audits: +// node scripts/test-isolation-lint.check.mjs +// which exits 0 on clean and 1 with a per-file listing otherwise. + +import { readdirSync, readFileSync, statSync } from "node:fs"; +import { join, extname } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +// Repo root is derived from this file's location (scripts/ is one +// level below the root), never from process.cwd() — see the wiring +// note above. +const repoRoot = fileURLToPath(new URL("../", import.meta.url)); + +export const REQUIRED_ENV_OVERRIDES = [ + "MCODE_WEBUI_SETTINGS_PATH", + "MCODE_WEBUI_EVENTS_PATH", + "MCODE_WEBUI_SESSIONS_DB", + "MCODE_WEBUI_UPLOAD_DIR", +]; + +/** + * Recursively walk `dir` and yield every regular file. Skips + * node_modules, .git, .next, dist, build, out. + */ +function* walk(dir) { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + if (entry.name.startsWith(".")) continue; + if (entry.name === "node_modules") continue; + if (entry.name === "dist" || entry.name === "build" || entry.name === "out") continue; + const full = join(dir, entry.name); + if (entry.isDirectory()) { + yield* walk(full); + } else { + yield full; + } + } +} + +/** + * Does the file text contain a server.js spawn? The patterns are + * narrow enough to skip imports, type-only references, and + * unrelated child_process calls: + * - `spawn(...)` whose first argument array contains `server.js` + * - `spawnSync(...)` likewise + * - `process.execPath` + `[..., "server.js", ...]` arg arrays + * + * Returns the indices of every match — each match triggers an env + * override check on the enclosing function body. + */ +function findSpawnIndices(text) { + const indices = []; + // spawn( ... "server.js" ... + const spawnRe = /spawn(?:Sync)?\s*\(/g; + let m; + while ((m = spawnRe.exec(text))) { + // Walk forward to find the matching closing paren — naive but + // good enough for the test files we care about. + let depth = 1; + let i = m.index + m[0].length; + while (i < text.length && depth > 0) { + const ch = text[i]; + if (ch === "(") depth += 1; + else if (ch === ")") depth -= 1; + i += 1; + } + const call = text.slice(m.index, i); + if (/"server\.js"|'server\.js'/.test(call)) { + indices.push({ start: m.index, end: i, call }); + } + } + return indices; +} + +/** + * Given the file text and a span (match), find the enclosing + * `function name(...) { ... }` body so the override check is scoped + * to the same function. If no enclosing function is found, the + * whole file is the scope. + */ +function enclosingFunctionBody(text, spanStart) { + // Find the nearest `function ` before spanStart. + let i = spanStart; + while (i > 0) { + const kwIdx = text.lastIndexOf("function", i); + if (kwIdx === -1) break; + const openBrace = text.indexOf("{", kwIdx); + if (openBrace === -1 || openBrace > spanStart) { + i = kwIdx - 1; + continue; + } + // Walk braces from openBrace forward. + let depth = 1; + let j = openBrace + 1; + while (j < text.length && depth > 0) { + const ch = text[j]; + if (ch === "{") depth += 1; + else if (ch === "}") depth -= 1; + j += 1; + } + return text.slice(openBrace, j); + } + // Fallback — top-level scope. + return text; +} + +/** + * Strip line comments (double slash) and block comments (slash-star … + * star-slash) from JS source, tracking string literals so comment + * markers inside strings (URLs, regexes-as-text) survive. Spawn shapes + * written in comments must not produce phantom matches — + * source-sync.test.mjs documents the fixture shape in prose and would + * otherwise trip the real-tree scan. + */ +function stripComments(text) { + let out = ""; + let state = "code"; // code | line | block | single | double | template + let i = 0; + while (i < text.length) { + const ch = text[i]; + const next = text[i + 1]; + if (state === "code") { + if (ch === "/" && next === "/") { state = "line"; i += 2; continue; } + if (ch === "/" && next === "*") { state = "block"; i += 2; continue; } + if (ch === "'") state = "single"; + else if (ch === '"') state = "double"; + else if (ch === "`") state = "template"; + out += ch; + i += 1; + continue; + } + if (state === "line") { + if (ch === "\n") { state = "code"; out += ch; } + i += 1; + continue; + } + if (state === "block") { + if (ch === "*" && next === "/") { state = "code"; out += " "; i += 2; continue; } + if (ch === "\n") out += ch; + i += 1; + continue; + } + // Inside a string literal: copy verbatim, honoring escapes. + if (ch === "\\") { + out += text.slice(i, i + 2); + i += 2; + continue; + } + if ( + (state === "single" && ch === "'") || + (state === "double" && ch === '"') || + (state === "template" && ch === "`") + ) state = "code"; + out += ch; + i += 1; + } + return out; +} + +function lintFile(path) { + const text = stripComments(readFileSync(path, "utf8")); + const spawns = findSpawnIndices(text); + if (spawns.length === 0) return []; + const issues = []; + for (const spawn of spawns) { + const body = enclosingFunctionBody(text, spawn.start); + const missing = REQUIRED_ENV_OVERRIDES.filter( + (name) => !new RegExp(name).test(body), + ); + if (missing.length > 0) { + issues.push({ + file: path, + offset: spawn.start, + missing, + }); + } + } + return issues; +} + +/** + * Scan the canonical test trees for server.js spawns that lack the + * per-test MCODE_WEBUI_* env overrides. Returns an empty array when + * the tree is compliant; each entry otherwise describes one + * offending spawn. `roots` defaults to the repository's real test + * trees; test/source-sync.test.mjs passes synthetic fixture trees + * so CI keeps proving the lint still detects violations (a lint + * that silently matches nothing must fail the gate, not pass it). + */ +export function collectTestIsolationViolations({ roots } = {}) { + const scanRoots = roots ?? [ + join(repoRoot, "test"), + join(repoRoot, "packages", "webui", "test"), + ]; + const issues = []; + for (const root of scanRoots) { + if (!statSync(root, { throwIfNoPath: false })) continue; + for (const file of walk(root)) { + if (![".js", ".mjs", ".cjs"].includes(extname(file))) continue; + issues.push(...lintFile(file)); + } + } + return issues; +} + +/** Human-readable per-file listing for gate output and CLI stderr. */ +export function formatTestIsolationViolations(issues) { + const formatted = issues + .map( + (issue) => + ` ${issue.file}:${issue.offset} missing: ${issue.missing.join(", ")}`, + ) + .join("\n"); + return ( + `server.js spawn without per-test env overrides (${issues.length} issue(s)):\n${formatted}\n` + + "Every test that spawns server.js MUST set MCODE_WEBUI_{SETTINGS_PATH,EVENTS_PATH,SESSIONS_DB,UPLOAD_DIR} to per-test tmp paths before first import. " + + "See packages/webui/test/server/server-startup.test.js for the canonical pattern." + ); +} + +// Manual-audit entry point: `node scripts/test-isolation-lint.check.mjs`. +// Inside a node:test run this file is only imported, never executed +// as the main module, so the CLI block is inert in gates. +if ( + process.argv[1] && + import.meta.url === pathToFileURL(process.argv[1]).href +) { + const issues = collectTestIsolationViolations(); + if (issues.length === 0) { + console.log( + "test-isolation-lint: clean — every server.js spawn sets the per-test MCODE_WEBUI_* overrides.", + ); + } else { + console.error(formatTestIsolationViolations(issues)); + process.exitCode = 1; + } +} diff --git a/test/source-sync.test.mjs b/test/source-sync.test.mjs index 8450b04b..caa2e72d 100644 --- a/test/source-sync.test.mjs +++ b/test/source-sync.test.mjs @@ -21,6 +21,7 @@ import { compareVersions, releaseCli } from '../scripts/release-cli.mjs'; import { compareRuns, exitCodeForStatus, renderReport, spread, validateRun, validateRequest, validateToolOutput, median, selectScenarios } from '../scripts/perf/report.mjs'; import { copyMcodeToolsArtifact, downloadMcodeToolsArtifact, MCODE_TOOLS_ARTIFACT } from '../scripts/lib/mcode-tools-artifact.mjs'; import { checkWindowsSourceLocation, runWindowsSourceLocationCheck } from '../scripts/check-windows-source-location.mjs'; +import { collectTestIsolationViolations, formatTestIsolationViolations } from '../scripts/test-isolation-lint.check.mjs'; test('Windows source preflight accepts localized fsutil labels', () => { const result = checkWindowsSourceLocation({ @@ -1128,3 +1129,42 @@ test('source imports preserve vendored Office schema bytes through Git staging', assert.deepEqual(git('show', `:${schema}`), bytes); } }); + +// Session-isolation/05: the test-isolation lint runs here — inside the root +// release-tools gate — instead of through packages/webui's test:unit globs, +// whose cwd made every repo-root scripts/ path match zero files (the lint +// reported "pass 0" while scanning nothing). The first assertion keeps the +// real test trees honest; the synthetic fixtures keep the lint itself honest +// by proving it still detects a violating spawn and clears a compliant one, +// so a future refactor cannot silently turn this gate back into a no-op. +test('test isolation lint finds every server.js spawn missing its MCODE_WEBUI_* overrides', () => { + const violations = collectTestIsolationViolations(); + assert.deepEqual(violations, [], formatTestIsolationViolations(violations)); +}); + +test('test isolation lint detects a violating spawn and clears a compliant one', t => { + const root = mkdtempSync(path.join(tmpdir(), 'isolation-lint-')); + t.after(() => rmSync(root, { recursive: true, force: true })); + mkdirSync(path.join(root, 'nested')); + // serverArg is concatenated so this test file's own source never contains + // the literal `spawn(..., ['server.js'])` shape — the real-tree assertion + // above scans test/ too, and the fixture template must not trip it. + const serverArg = "'server" + ".js'"; + const spawnFixture = body => `import { spawn } from 'node:child_process';\n${body}`; + writeFileSync(path.join(root, 'nested', 'violating.test.mjs'), spawnFixture( + `function startServer() {\n return spawn(process.execPath, [${serverArg}]);\n}\n`, + )); + const overrides = ['MCODE_WEBUI_SETTINGS_PATH', 'MCODE_WEBUI_EVENTS_PATH', 'MCODE_WEBUI_SESSIONS_DB', 'MCODE_WEBUI_UPLOAD_DIR'] + .map(name => `${name}: tmpPath`).join(', '); + writeFileSync(path.join(root, 'compliant.test.mjs'), spawnFixture( + `function startServer() {\n const env = { ${overrides} };\n return spawn(process.execPath, [${serverArg}], { env });\n}\n`, + )); + const violations = collectTestIsolationViolations({ roots: [root] }); + assert.equal(violations.length, 1); + assert.match(violations[0].file, /violating\.test\.mjs$/); + assert.deepEqual(violations[0].missing, ['MCODE_WEBUI_SETTINGS_PATH', 'MCODE_WEBUI_EVENTS_PATH', 'MCODE_WEBUI_SESSIONS_DB', 'MCODE_WEBUI_UPLOAD_DIR']); + assert.match(formatTestIsolationViolations(violations), /violating\.test\.mjs/); + assert.doesNotMatch(formatTestIsolationViolations(violations), /compliant\.test\.mjs/); + rmSync(path.join(root, 'nested'), { recursive: true, force: true }); + assert.deepEqual(collectTestIsolationViolations({ roots: [root] }), []); +});