From bfc83541e411a94e9c203c6032c52841c1422d1b Mon Sep 17 00:00:00 2001 From: "@daniel-lxs" <57051444+daniel-lxs@users.noreply.github.com> Date: Tue, 20 Oct 2026 05:32:06 +0000 Subject: [PATCH 1/3] fix: retain development source events during inotify rebuilds --- .../__tests__/watchman-script.native.test.ts | 225 ++++++++++++++++++ scripts/watchman.sh | 91 +++++-- 2 files changed, 299 insertions(+), 17 deletions(-) create mode 100644 apps/dev/src/services/__tests__/watchman-script.native.test.ts diff --git a/apps/dev/src/services/__tests__/watchman-script.native.test.ts b/apps/dev/src/services/__tests__/watchman-script.native.test.ts new file mode 100644 index 0000000000..b774fa62e9 --- /dev/null +++ b/apps/dev/src/services/__tests__/watchman-script.native.test.ts @@ -0,0 +1,225 @@ +import { spawn, spawnSync, type ChildProcess } from 'node:child_process'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const nativeInotify = spawnSync('which', ['inotifywait'], { encoding: 'utf8' }); +const hasInotify = process.platform === 'linux' && nativeInotify.status === 0; +const scriptSource = fileURLToPath( + new URL('../../../../../scripts/watchman.sh', import.meta.url), +); +const isAlive = (pid: number) => { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +}; + +// Native coverage requires inotify-tools. The fixture retains the real shell, +// monitor, event stream, child processes and disk; only compilation is a small +// disposable command so that its read/publish ordering is deterministic. +describe.skipIf(!hasInotify)('shared dev watcher with native inotify', () => { + let root: string; + let app: string; + let child: ChildProcess | undefined; + let exited: Promise | undefined; + let output: string; + + beforeEach(async () => { + root = await fs.mkdtemp(path.join(os.tmpdir(), 'dev-watch-native-')); + app = path.join(root, 'apps/proof'); + await Promise.all( + ['scripts', 'packages', 'bin', 'tmp', 'apps/proof/src'].map((dir) => + fs.mkdir(path.join(root, dir), { recursive: true }), + ), + ); + await fs.copyFile(scriptSource, path.join(root, 'scripts/watchman.sh')); + // Force the supported fallback even on hosts with Watchman installed. + await fs.writeFile(path.join(root, 'bin/watchman'), '#!/bin/sh\nexit 1\n', { + mode: 0o755, + }); + await fs.writeFile( + path.join(root, 'bin/inotifywait'), + `#!/bin/sh\necho $$ > '${app}/monitor-pid'\nexec '${nativeInotify.stdout.trim()}' "$@"\n`, + { mode: 0o755 }, + ); + await fs.writeFile(path.join(app, 'src/revision.ts'), 'first'); + await fs.writeFile( + path.join(app, 'build.mjs'), + ` +import fs from 'node:fs/promises'; +const text = await fs.readFile('src/revision.ts', 'utf8'); +await fs.appendFile('builds', text + '\\n'); +if (text === 'fail') process.exit(17); +if (text === 'second') { + await fs.writeFile('second-started', ''); + while (true) { + try { await fs.access('release-second'); break; } + catch { await new Promise(resolve => setTimeout(resolve, 20)); } + } +} +await fs.writeFile('published', text); +`, + ); + await fs.writeFile( + path.join(app, 'run.mjs'), + ` +import fs from 'node:fs'; +fs.appendFileSync('runtime-pids', process.pid + '\\n'); +setInterval(() => {}, 1000); +`, + ); + output = ''; + child = undefined; + exited = undefined; + }); + + const start = () => { + child = spawn( + 'bash', + [ + path.join(root, 'scripts/watchman.sh'), + 'proof', + `${process.execPath} build.mjs`, + `${process.execPath} run.mjs`, + ], + { + cwd: app, + detached: true, + env: { + ...process.env, + PATH: `${root}/bin:${process.env.PATH}`, + TMPDIR: path.join(root, 'tmp'), + }, + stdio: ['ignore', 'pipe', 'pipe'], + }, + ); + child.stdout!.on('data', (chunk) => { + output += chunk; + }); + child.stderr!.on('data', (chunk) => { + output += chunk; + }); + exited = new Promise((resolve) => child!.once('exit', resolve)); + }; + const ready = async () => { + await vi.waitFor(() => expect(output).toContain('Watching for changes'), { + timeout: 5_000, + }); + await vi.waitFor( + async () => + expect( + await fs.readFile(path.join(app, 'runtime-pids'), 'utf8'), + ).not.toBe(''), + { timeout: 5_000 }, + ); + }; + const expectCleaned = async () => { + const monitor = Number( + await fs.readFile(path.join(app, 'monitor-pid'), 'utf8'), + ); + const runtimes = await fs + .readFile(path.join(app, 'runtime-pids'), 'utf8') + .catch(() => ''); + await vi.waitFor(() => { + expect(isAlive(monitor)).toBe(false); + for (const pid of runtimes.trim().split(/\s+/).filter(Boolean)) + expect(isAlive(Number(pid))).toBe(false); + }); + expect(await fs.readdir(path.join(root, 'tmp'))).toEqual([]); + }; + + afterEach(async () => { + await fs.writeFile(path.join(app, 'release-second'), ''); + if (child?.pid) { + try { + process.kill(-child.pid, 'SIGKILL'); + } catch { + /* already exited */ + } + await exited; + } + await fs.rm(root, { recursive: true, force: true }); + }); + + it.each(['initial', 'later'])( + 'retains edits during the %s compilation and reaps owned children on idle TERM', + async (build) => { + if (build === 'initial') + await fs.writeFile(path.join(app, 'src/revision.ts'), 'second'); + start(); + if (build === 'later') { + await ready(); + await fs.writeFile(path.join(app, 'src/revision.ts'), 'second'); + } + await vi.waitFor(() => fs.access(path.join(app, 'second-started')), { + timeout: 5_000, + }); + await fs.writeFile(path.join(app, 'src/revision.ts'), 'third'); + await fs.writeFile(path.join(app, 'release-second'), ''); + await vi.waitFor( + async () => + expect(await fs.readFile(path.join(app, 'published'), 'utf8')).toBe( + 'third', + ), + { timeout: 5_000 }, + ); + child!.kill('SIGTERM'); + await vi.waitFor(() => expect(child!.exitCode).toBe(143), { + timeout: 5_000, + }); + await expectCleaned(); + }, + 15_000, + ); + + it('continues watching and recovers after a later compiler failure', async () => { + start(); + await ready(); + await fs.writeFile(path.join(app, 'src/revision.ts'), 'fail'); + await vi.waitFor( + async () => + expect(await fs.readFile(path.join(app, 'builds'), 'utf8')).toContain( + 'fail\n', + ), + { timeout: 5_000 }, + ); + await fs.writeFile(path.join(app, 'src/revision.ts'), 'third'); + await vi.waitFor( + async () => + expect(await fs.readFile(path.join(app, 'published'), 'utf8')).toBe( + 'third', + ), + { timeout: 5_000 }, + ); + child!.kill('SIGTERM'); + await vi.waitFor(() => expect(child!.exitCode).toBe(143), { + timeout: 5_000, + }); + await expectCleaned(); + }, 15_000); + + it('cleans the monitor and temporary pipe when the initial build fails', async () => { + await fs.writeFile(path.join(app, 'src/revision.ts'), 'fail'); + start(); + await vi.waitFor(() => expect(child!.exitCode).toBe(17), { + timeout: 5_000, + }); + await expectCleaned(); + await expect(fs.access(path.join(app, 'runtime-pids'))).rejects.toThrow(); + }, 10_000); + + it('fails and cleans the runtime when the event monitor exits', async () => { + start(); + await ready(); + const monitor = Number( + await fs.readFile(path.join(app, 'monitor-pid'), 'utf8'), + ); + process.kill(monitor, 'SIGTERM'); + await vi.waitFor(() => expect(child!.exitCode).toBe(1), { timeout: 5_000 }); + await expectCleaned(); + }, 10_000); +}); diff --git a/scripts/watchman.sh b/scripts/watchman.sh index 5176636c2a..d0c12d5e40 100755 --- a/scripts/watchman.sh +++ b/scripts/watchman.sh @@ -70,40 +70,97 @@ if command -v inotifywait &>/dev/null; then CURRENT_PATH="$PATH" + RUN_PID="" + WATCH_PID="" + EVENT_DIR="" + cleanup_inotify() { + local status=$? + trap - EXIT INT TERM + exec 3<&- + if [[ -n "$WATCH_PID" ]]; then + kill "$WATCH_PID" 2>/dev/null || true + wait "$WATCH_PID" 2>/dev/null || true + fi + if [[ -n "$RUN_PID" ]]; then + kill "$RUN_PID" 2>/dev/null || true + wait "$RUN_PID" 2>/dev/null || true + fi + if [[ -n "$EVENT_DIR" ]]; then + rm -rf "$EVENT_DIR" + fi + exit "$status" + } + trap cleanup_inotify EXIT + trap 'exit 130' INT + trap 'exit 143' TERM + + # Keep one monitor alive through compilation. A one-shot watch loses every + # edit made while the build is running. Its readiness line also fences the + # initial build so that startup edits cannot fall through the same gap. + EVENT_DIR=$(mktemp -d) + mkfifo "$EVENT_DIR/events" + LC_ALL=C inotifywait -m -r \ + --include '\.ts$' \ + --format '%w%f' \ + -e modify,create,delete,move \ + "$REPO_ROOT/$APP_REL/src" \ + "$REPO_ROOT/packages" \ + >"$EVENT_DIR/events" 2>&1 & + WATCH_PID=$! + exec 3<"$EVENT_DIR/events" + WATCH_READY=false + while IFS= read -r event <&3; do + if [[ "$event" == "Watches established." ]]; then + WATCH_READY=true + break + fi + printf '%s\n' "$event" + done + if [[ "$WATCH_READY" != true ]]; then + echo "[$SERVICE_NAME:inotifywait] Failed to establish file watches." >&2 + exit 1 + fi + echo "[$SERVICE_NAME:inotifywait] Initial build..." - bash -lc "$BUILD_CMD" + bash -lc "$BUILD_CMD" 3<&- echo "[$SERVICE_NAME:inotifywait] Starting runtime..." - bash -c "export PATH='$CURRENT_PATH' && cd '$APP_DIR' && $RUN_CMD" & + bash -c "export PATH='$CURRENT_PATH' && cd '$APP_DIR' && $RUN_CMD" 3<&- & RUN_PID=$! - cleanup_inotify() { - kill "$RUN_PID" 2>/dev/null || true - } - trap cleanup_inotify EXIT INT TERM - echo "[$SERVICE_NAME:inotifywait] Watching for changes in $APP_REL/src/ and packages/..." - while true; do - inotifywait -r -q \ - --include '\.ts$' \ - -e modify,create,delete,move \ - "$REPO_ROOT/$APP_REL/src" \ - "$REPO_ROOT/packages" \ - 2>/dev/null || true - + while IFS= read -r event <&3; do sleep 0.2 + # Coalesce the pre-build burst only. Events received during the build stay + # queued for the next pass. Bound draining so continuous edits cannot starve + # compilation, and treat monitor EOF as failure rather than a rebuild loop. + for ((i = 0; i < 1000; i++)); do + if IFS= read -r -t 0.01 event <&3; then + continue + else + read_status=$? + if [[ "$read_status" -eq 1 ]]; then + echo "[$SERVICE_NAME:inotifywait] File watcher stopped." >&2 + exit 1 + fi + break + fi + done echo "[$SERVICE_NAME:inotifywait] Change detected, rebuilding..." kill "$RUN_PID" 2>/dev/null || true wait "$RUN_PID" 2>/dev/null || true - bash -c "export PATH='$CURRENT_PATH' && cd '$APP_DIR' && $BUILD_CMD" || true + RUN_PID="" + bash -c "export PATH='$CURRENT_PATH' && cd '$APP_DIR' && $BUILD_CMD" 3<&- || true echo "[$SERVICE_NAME:inotifywait] Restarting runtime..." - bash -c "export PATH='$CURRENT_PATH' && cd '$APP_DIR' && $RUN_CMD" & + bash -c "export PATH='$CURRENT_PATH' && cd '$APP_DIR' && $RUN_CMD" 3<&- & RUN_PID=$! done + echo "[$SERVICE_NAME:inotifywait] File watcher stopped." >&2 + exit 1 fi # ── No file watcher available ─────────────────────────────────────────────── From 50c084dfd3fb2085b68b6003aef943e0c40fdce3 Mon Sep 17 00:00:00 2001 From: "@daniel-lxs" <57051444+daniel-lxs@users.noreply.github.com> Date: Tue, 20 Oct 2026 06:00:20 +0000 Subject: [PATCH 2/3] fix: preserve PM2 IPC descriptors in the development watcher --- .../__tests__/watchman-script.native.test.ts | 58 ++++++++++++++++--- scripts/watchman.sh | 23 +++++--- 2 files changed, 65 insertions(+), 16 deletions(-) diff --git a/apps/dev/src/services/__tests__/watchman-script.native.test.ts b/apps/dev/src/services/__tests__/watchman-script.native.test.ts index b774fa62e9..3cd828b138 100644 --- a/apps/dev/src/services/__tests__/watchman-script.native.test.ts +++ b/apps/dev/src/services/__tests__/watchman-script.native.test.ts @@ -27,6 +27,7 @@ describe.skipIf(!hasInotify)('shared dev watcher with native inotify', () => { let child: ChildProcess | undefined; let exited: Promise | undefined; let output: string; + let ipcMessages: unknown[]; beforeEach(async () => { root = await fs.mkdtemp(path.join(os.tmpdir(), 'dev-watch-native-')); @@ -52,6 +53,7 @@ describe.skipIf(!hasInotify)('shared dev watcher with native inotify', () => { ` import fs from 'node:fs/promises'; const text = await fs.readFile('src/revision.ts', 'utf8'); +if (process.send) await fs.appendFile('build-ipc-fds', (await fs.readlink('/proc/self/fd/3')) + '\\n'); await fs.appendFile('builds', text + '\\n'); if (text === 'fail') process.exit(17); if (text === 'second') { @@ -69,15 +71,18 @@ await fs.writeFile('published', text); ` import fs from 'node:fs'; fs.appendFileSync('runtime-pids', process.pid + '\\n'); +fs.appendFileSync('runtime-revisions', fs.readFileSync('published', 'utf8') + '\\n'); +process.send?.({ kind: 'watch-runtime', pid: process.pid }); setInterval(() => {}, 1000); `, ); output = ''; + ipcMessages = []; child = undefined; exited = undefined; }); - const start = () => { + const start = (ipc = false) => { child = spawn( 'bash', [ @@ -94,7 +99,11 @@ setInterval(() => {}, 1000); PATH: `${root}/bin:${process.env.PATH}`, TMPDIR: path.join(root, 'tmp'), }, - stdio: ['ignore', 'pipe', 'pipe'], + // PM2 also launches shell interpreters with an inherited Node IPC + // socket on fd3. Keep that real channel in the regression boundary. + stdio: ipc + ? ['ignore', 'pipe', 'pipe', 'ipc'] + : ['ignore', 'pipe', 'pipe'], }, ); child.stdout!.on('data', (chunk) => { @@ -103,6 +112,7 @@ setInterval(() => {}, 1000); child.stderr!.on('data', (chunk) => { output += chunk; }); + child.on('message', (message) => ipcMessages.push(message)); exited = new Promise((resolve) => child!.once('exit', resolve)); }; const ready = async () => { @@ -127,7 +137,10 @@ setInterval(() => {}, 1000); await vi.waitFor(() => { expect(isAlive(monitor)).toBe(false); for (const pid of runtimes.trim().split(/\s+/).filter(Boolean)) - expect(isAlive(Number(pid))).toBe(false); + expect( + isAlive(Number(pid)), + `runtime ${pid} remains after shell exit`, + ).toBe(false); }); expect(await fs.readdir(path.join(root, 'tmp'))).toEqual([]); }; @@ -145,12 +158,16 @@ setInterval(() => {}, 1000); await fs.rm(root, { recursive: true, force: true }); }); - it.each(['initial', 'later'])( - 'retains edits during the %s compilation and reaps owned children on idle TERM', - async (build) => { + it.each([ + { build: 'initial', ipc: false }, + { build: 'later', ipc: false }, + { build: 'later', ipc: true }, + ])( + 'retains edits during $build compilation (IPC=$ipc) and reaps owned children on idle TERM', + async ({ build, ipc }) => { if (build === 'initial') await fs.writeFile(path.join(app, 'src/revision.ts'), 'second'); - start(); + start(ipc); if (build === 'later') { await ready(); await fs.writeFile(path.join(app, 'src/revision.ts'), 'second'); @@ -167,11 +184,31 @@ setInterval(() => {}, 1000); ), { timeout: 5_000 }, ); + // Publication happens inside the build. Establish the claimed idle + // runtime boundary before sending the shutdown signal. + await vi.waitFor( + async () => + expect( + await fs.readFile(path.join(app, 'runtime-revisions'), 'utf8'), + ).toContain('third\n'), + { timeout: 5_000 }, + ); child!.kill('SIGTERM'); await vi.waitFor(() => expect(child!.exitCode).toBe(143), { timeout: 5_000, }); await expectCleaned(); + if (ipc) { + expect(ipcMessages).toEqual( + expect.arrayContaining([ + expect.objectContaining({ kind: 'watch-runtime' }), + ]), + ); + expect( + await fs.readFile(path.join(app, 'build-ipc-fds'), 'utf8'), + ).toMatch(/^socket:\[\d+\]/); + expect(output).not.toMatch(/Assertion|Aborted/); + } }, 15_000, ); @@ -195,6 +232,13 @@ setInterval(() => {}, 1000); ), { timeout: 5_000 }, ); + await vi.waitFor( + async () => + expect( + await fs.readFile(path.join(app, 'runtime-revisions'), 'utf8'), + ).toContain('third\n'), + { timeout: 5_000 }, + ); child!.kill('SIGTERM'); await vi.waitFor(() => expect(child!.exitCode).toBe(143), { timeout: 5_000, diff --git a/scripts/watchman.sh b/scripts/watchman.sh index d0c12d5e40..bdfe8d9a9f 100755 --- a/scripts/watchman.sh +++ b/scripts/watchman.sh @@ -73,10 +73,13 @@ if command -v inotifywait &>/dev/null; then RUN_PID="" WATCH_PID="" EVENT_DIR="" + EVENT_FD="" cleanup_inotify() { local status=$? trap - EXIT INT TERM - exec 3<&- + if [[ -n "$EVENT_FD" ]]; then + exec {EVENT_FD}<&- + fi if [[ -n "$WATCH_PID" ]]; then kill "$WATCH_PID" 2>/dev/null || true wait "$WATCH_PID" 2>/dev/null || true @@ -107,9 +110,11 @@ if command -v inotifywait &>/dev/null; then "$REPO_ROOT/packages" \ >"$EVENT_DIR/events" 2>&1 & WATCH_PID=$! - exec 3<"$EVENT_DIR/events" + # Let Bash choose an unused descriptor: PM2 reserves fd3 for Node IPC, which + # build/runtime children must continue to inherit unchanged. + exec {EVENT_FD}<"$EVENT_DIR/events" WATCH_READY=false - while IFS= read -r event <&3; do + while IFS= read -r -u "$EVENT_FD" event; do if [[ "$event" == "Watches established." ]]; then WATCH_READY=true break @@ -122,20 +127,20 @@ if command -v inotifywait &>/dev/null; then fi echo "[$SERVICE_NAME:inotifywait] Initial build..." - bash -lc "$BUILD_CMD" 3<&- + bash -lc "$BUILD_CMD" {EVENT_FD}<&- echo "[$SERVICE_NAME:inotifywait] Starting runtime..." - bash -c "export PATH='$CURRENT_PATH' && cd '$APP_DIR' && $RUN_CMD" 3<&- & + bash -c "export PATH='$CURRENT_PATH' && cd '$APP_DIR' && $RUN_CMD" {EVENT_FD}<&- & RUN_PID=$! echo "[$SERVICE_NAME:inotifywait] Watching for changes in $APP_REL/src/ and packages/..." - while IFS= read -r event <&3; do + while IFS= read -r -u "$EVENT_FD" event; do sleep 0.2 # Coalesce the pre-build burst only. Events received during the build stay # queued for the next pass. Bound draining so continuous edits cannot starve # compilation, and treat monitor EOF as failure rather than a rebuild loop. for ((i = 0; i < 1000; i++)); do - if IFS= read -r -t 0.01 event <&3; then + if IFS= read -r -t 0.01 -u "$EVENT_FD" event; then continue else read_status=$? @@ -153,10 +158,10 @@ if command -v inotifywait &>/dev/null; then wait "$RUN_PID" 2>/dev/null || true RUN_PID="" - bash -c "export PATH='$CURRENT_PATH' && cd '$APP_DIR' && $BUILD_CMD" 3<&- || true + bash -c "export PATH='$CURRENT_PATH' && cd '$APP_DIR' && $BUILD_CMD" {EVENT_FD}<&- || true echo "[$SERVICE_NAME:inotifywait] Restarting runtime..." - bash -c "export PATH='$CURRENT_PATH' && cd '$APP_DIR' && $RUN_CMD" 3<&- & + bash -c "export PATH='$CURRENT_PATH' && cd '$APP_DIR' && $RUN_CMD" {EVENT_FD}<&- & RUN_PID=$! done echo "[$SERVICE_NAME:inotifywait] File watcher stopped." >&2 From e23c6196eeaba676de9964702cc57ea744b81217 Mon Sep 17 00:00:00 2001 From: "@daniel-lxs" <57051444+daniel-lxs@users.noreply.github.com> Date: Tue, 20 Oct 2026 06:26:10 +0000 Subject: [PATCH 3/3] test: require native watcher regression coverage in Linux CI --- .github/workflows/CI.yml | 4 ++++ .../src/services/__tests__/watchman-script.native.test.ts | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 6592a51e85..f0973e9e58 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -181,6 +181,10 @@ jobs: run: pnpm --filter @roomote/db db:push:test --force env: DATABASE_URL: postgres://postgres:password@localhost:5432/test + - name: Install native watcher test dependency + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends inotify-tools - name: Run unit tests run: pnpm test:ci env: diff --git a/apps/dev/src/services/__tests__/watchman-script.native.test.ts b/apps/dev/src/services/__tests__/watchman-script.native.test.ts index 3cd828b138..8388a291ce 100644 --- a/apps/dev/src/services/__tests__/watchman-script.native.test.ts +++ b/apps/dev/src/services/__tests__/watchman-script.native.test.ts @@ -6,6 +6,11 @@ import { fileURLToPath } from 'node:url'; const nativeInotify = spawnSync('which', ['inotifywait'], { encoding: 'utf8' }); const hasInotify = process.platform === 'linux' && nativeInotify.status === 0; +if (process.env.CI && process.platform === 'linux' && !hasInotify) { + throw new Error( + 'Linux CI requires inotify-tools for native dev-watcher tests', + ); +} const scriptSource = fileURLToPath( new URL('../../../../../scripts/watchman.sh', import.meta.url), );