diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 2d443ac0d..48d262372 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -181,6 +181,10 @@ jobs: run: pnpm --filter @roomote/db db:push:test --force env: DATABASE_URL: postgres://postgres:password@localhost:5432/test + - name: Install native watcher test dependency + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends inotify-tools - name: Run unit tests run: pnpm test:ci env: diff --git a/apps/dev/src/services/__tests__/watchman-script.native.test.ts b/apps/dev/src/services/__tests__/watchman-script.native.test.ts new file mode 100644 index 000000000..8388a291c --- /dev/null +++ b/apps/dev/src/services/__tests__/watchman-script.native.test.ts @@ -0,0 +1,274 @@ +import { spawn, spawnSync, type ChildProcess } from 'node:child_process'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const nativeInotify = spawnSync('which', ['inotifywait'], { encoding: 'utf8' }); +const hasInotify = process.platform === 'linux' && nativeInotify.status === 0; +if (process.env.CI && process.platform === 'linux' && !hasInotify) { + throw new Error( + 'Linux CI requires inotify-tools for native dev-watcher tests', + ); +} +const scriptSource = fileURLToPath( + new URL('../../../../../scripts/watchman.sh', import.meta.url), +); +const isAlive = (pid: number) => { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +}; + +// Native coverage requires inotify-tools. The fixture retains the real shell, +// monitor, event stream, child processes and disk; only compilation is a small +// disposable command so that its read/publish ordering is deterministic. +describe.skipIf(!hasInotify)('shared dev watcher with native inotify', () => { + let root: string; + let app: string; + let child: ChildProcess | undefined; + let exited: Promise | undefined; + let output: string; + let ipcMessages: unknown[]; + + beforeEach(async () => { + root = await fs.mkdtemp(path.join(os.tmpdir(), 'dev-watch-native-')); + app = path.join(root, 'apps/proof'); + await Promise.all( + ['scripts', 'packages', 'bin', 'tmp', 'apps/proof/src'].map((dir) => + fs.mkdir(path.join(root, dir), { recursive: true }), + ), + ); + await fs.copyFile(scriptSource, path.join(root, 'scripts/watchman.sh')); + // Force the supported fallback even on hosts with Watchman installed. + await fs.writeFile(path.join(root, 'bin/watchman'), '#!/bin/sh\nexit 1\n', { + mode: 0o755, + }); + await fs.writeFile( + path.join(root, 'bin/inotifywait'), + `#!/bin/sh\necho $$ > '${app}/monitor-pid'\nexec '${nativeInotify.stdout.trim()}' "$@"\n`, + { mode: 0o755 }, + ); + await fs.writeFile(path.join(app, 'src/revision.ts'), 'first'); + await fs.writeFile( + path.join(app, 'build.mjs'), + ` +import fs from 'node:fs/promises'; +const text = await fs.readFile('src/revision.ts', 'utf8'); +if (process.send) await fs.appendFile('build-ipc-fds', (await fs.readlink('/proc/self/fd/3')) + '\\n'); +await fs.appendFile('builds', text + '\\n'); +if (text === 'fail') process.exit(17); +if (text === 'second') { + await fs.writeFile('second-started', ''); + while (true) { + try { await fs.access('release-second'); break; } + catch { await new Promise(resolve => setTimeout(resolve, 20)); } + } +} +await fs.writeFile('published', text); +`, + ); + await fs.writeFile( + path.join(app, 'run.mjs'), + ` +import fs from 'node:fs'; +fs.appendFileSync('runtime-pids', process.pid + '\\n'); +fs.appendFileSync('runtime-revisions', fs.readFileSync('published', 'utf8') + '\\n'); +process.send?.({ kind: 'watch-runtime', pid: process.pid }); +setInterval(() => {}, 1000); +`, + ); + output = ''; + ipcMessages = []; + child = undefined; + exited = undefined; + }); + + const start = (ipc = false) => { + child = spawn( + 'bash', + [ + path.join(root, 'scripts/watchman.sh'), + 'proof', + `${process.execPath} build.mjs`, + `${process.execPath} run.mjs`, + ], + { + cwd: app, + detached: true, + env: { + ...process.env, + PATH: `${root}/bin:${process.env.PATH}`, + TMPDIR: path.join(root, 'tmp'), + }, + // PM2 also launches shell interpreters with an inherited Node IPC + // socket on fd3. Keep that real channel in the regression boundary. + stdio: ipc + ? ['ignore', 'pipe', 'pipe', 'ipc'] + : ['ignore', 'pipe', 'pipe'], + }, + ); + child.stdout!.on('data', (chunk) => { + output += chunk; + }); + child.stderr!.on('data', (chunk) => { + output += chunk; + }); + child.on('message', (message) => ipcMessages.push(message)); + exited = new Promise((resolve) => child!.once('exit', resolve)); + }; + const ready = async () => { + await vi.waitFor(() => expect(output).toContain('Watching for changes'), { + timeout: 5_000, + }); + await vi.waitFor( + async () => + expect( + await fs.readFile(path.join(app, 'runtime-pids'), 'utf8'), + ).not.toBe(''), + { timeout: 5_000 }, + ); + }; + const expectCleaned = async () => { + const monitor = Number( + await fs.readFile(path.join(app, 'monitor-pid'), 'utf8'), + ); + const runtimes = await fs + .readFile(path.join(app, 'runtime-pids'), 'utf8') + .catch(() => ''); + await vi.waitFor(() => { + expect(isAlive(monitor)).toBe(false); + for (const pid of runtimes.trim().split(/\s+/).filter(Boolean)) + expect( + isAlive(Number(pid)), + `runtime ${pid} remains after shell exit`, + ).toBe(false); + }); + expect(await fs.readdir(path.join(root, 'tmp'))).toEqual([]); + }; + + afterEach(async () => { + await fs.writeFile(path.join(app, 'release-second'), ''); + if (child?.pid) { + try { + process.kill(-child.pid, 'SIGKILL'); + } catch { + /* already exited */ + } + await exited; + } + await fs.rm(root, { recursive: true, force: true }); + }); + + it.each([ + { build: 'initial', ipc: false }, + { build: 'later', ipc: false }, + { build: 'later', ipc: true }, + ])( + 'retains edits during $build compilation (IPC=$ipc) and reaps owned children on idle TERM', + async ({ build, ipc }) => { + if (build === 'initial') + await fs.writeFile(path.join(app, 'src/revision.ts'), 'second'); + start(ipc); + if (build === 'later') { + await ready(); + await fs.writeFile(path.join(app, 'src/revision.ts'), 'second'); + } + await vi.waitFor(() => fs.access(path.join(app, 'second-started')), { + timeout: 5_000, + }); + await fs.writeFile(path.join(app, 'src/revision.ts'), 'third'); + await fs.writeFile(path.join(app, 'release-second'), ''); + await vi.waitFor( + async () => + expect(await fs.readFile(path.join(app, 'published'), 'utf8')).toBe( + 'third', + ), + { timeout: 5_000 }, + ); + // Publication happens inside the build. Establish the claimed idle + // runtime boundary before sending the shutdown signal. + await vi.waitFor( + async () => + expect( + await fs.readFile(path.join(app, 'runtime-revisions'), 'utf8'), + ).toContain('third\n'), + { timeout: 5_000 }, + ); + child!.kill('SIGTERM'); + await vi.waitFor(() => expect(child!.exitCode).toBe(143), { + timeout: 5_000, + }); + await expectCleaned(); + if (ipc) { + expect(ipcMessages).toEqual( + expect.arrayContaining([ + expect.objectContaining({ kind: 'watch-runtime' }), + ]), + ); + expect( + await fs.readFile(path.join(app, 'build-ipc-fds'), 'utf8'), + ).toMatch(/^socket:\[\d+\]/); + expect(output).not.toMatch(/Assertion|Aborted/); + } + }, + 15_000, + ); + + it('continues watching and recovers after a later compiler failure', async () => { + start(); + await ready(); + await fs.writeFile(path.join(app, 'src/revision.ts'), 'fail'); + await vi.waitFor( + async () => + expect(await fs.readFile(path.join(app, 'builds'), 'utf8')).toContain( + 'fail\n', + ), + { timeout: 5_000 }, + ); + await fs.writeFile(path.join(app, 'src/revision.ts'), 'third'); + await vi.waitFor( + async () => + expect(await fs.readFile(path.join(app, 'published'), 'utf8')).toBe( + 'third', + ), + { timeout: 5_000 }, + ); + await vi.waitFor( + async () => + expect( + await fs.readFile(path.join(app, 'runtime-revisions'), 'utf8'), + ).toContain('third\n'), + { timeout: 5_000 }, + ); + child!.kill('SIGTERM'); + await vi.waitFor(() => expect(child!.exitCode).toBe(143), { + timeout: 5_000, + }); + await expectCleaned(); + }, 15_000); + + it('cleans the monitor and temporary pipe when the initial build fails', async () => { + await fs.writeFile(path.join(app, 'src/revision.ts'), 'fail'); + start(); + await vi.waitFor(() => expect(child!.exitCode).toBe(17), { + timeout: 5_000, + }); + await expectCleaned(); + await expect(fs.access(path.join(app, 'runtime-pids'))).rejects.toThrow(); + }, 10_000); + + it('fails and cleans the runtime when the event monitor exits', async () => { + start(); + await ready(); + const monitor = Number( + await fs.readFile(path.join(app, 'monitor-pid'), 'utf8'), + ); + process.kill(monitor, 'SIGTERM'); + await vi.waitFor(() => expect(child!.exitCode).toBe(1), { timeout: 5_000 }); + await expectCleaned(); + }, 10_000); +}); diff --git a/scripts/watchman.sh b/scripts/watchman.sh index 5176636c2..bdfe8d9a9 100755 --- a/scripts/watchman.sh +++ b/scripts/watchman.sh @@ -70,40 +70,102 @@ if command -v inotifywait &>/dev/null; then CURRENT_PATH="$PATH" + RUN_PID="" + WATCH_PID="" + EVENT_DIR="" + EVENT_FD="" + cleanup_inotify() { + local status=$? + trap - EXIT INT TERM + if [[ -n "$EVENT_FD" ]]; then + exec {EVENT_FD}<&- + fi + if [[ -n "$WATCH_PID" ]]; then + kill "$WATCH_PID" 2>/dev/null || true + wait "$WATCH_PID" 2>/dev/null || true + fi + if [[ -n "$RUN_PID" ]]; then + kill "$RUN_PID" 2>/dev/null || true + wait "$RUN_PID" 2>/dev/null || true + fi + if [[ -n "$EVENT_DIR" ]]; then + rm -rf "$EVENT_DIR" + fi + exit "$status" + } + trap cleanup_inotify EXIT + trap 'exit 130' INT + trap 'exit 143' TERM + + # Keep one monitor alive through compilation. A one-shot watch loses every + # edit made while the build is running. Its readiness line also fences the + # initial build so that startup edits cannot fall through the same gap. + EVENT_DIR=$(mktemp -d) + mkfifo "$EVENT_DIR/events" + LC_ALL=C inotifywait -m -r \ + --include '\.ts$' \ + --format '%w%f' \ + -e modify,create,delete,move \ + "$REPO_ROOT/$APP_REL/src" \ + "$REPO_ROOT/packages" \ + >"$EVENT_DIR/events" 2>&1 & + WATCH_PID=$! + # Let Bash choose an unused descriptor: PM2 reserves fd3 for Node IPC, which + # build/runtime children must continue to inherit unchanged. + exec {EVENT_FD}<"$EVENT_DIR/events" + WATCH_READY=false + while IFS= read -r -u "$EVENT_FD" event; do + if [[ "$event" == "Watches established." ]]; then + WATCH_READY=true + break + fi + printf '%s\n' "$event" + done + if [[ "$WATCH_READY" != true ]]; then + echo "[$SERVICE_NAME:inotifywait] Failed to establish file watches." >&2 + exit 1 + fi + echo "[$SERVICE_NAME:inotifywait] Initial build..." - bash -lc "$BUILD_CMD" + bash -lc "$BUILD_CMD" {EVENT_FD}<&- echo "[$SERVICE_NAME:inotifywait] Starting runtime..." - bash -c "export PATH='$CURRENT_PATH' && cd '$APP_DIR' && $RUN_CMD" & + bash -c "export PATH='$CURRENT_PATH' && cd '$APP_DIR' && $RUN_CMD" {EVENT_FD}<&- & RUN_PID=$! - cleanup_inotify() { - kill "$RUN_PID" 2>/dev/null || true - } - trap cleanup_inotify EXIT INT TERM - echo "[$SERVICE_NAME:inotifywait] Watching for changes in $APP_REL/src/ and packages/..." - while true; do - inotifywait -r -q \ - --include '\.ts$' \ - -e modify,create,delete,move \ - "$REPO_ROOT/$APP_REL/src" \ - "$REPO_ROOT/packages" \ - 2>/dev/null || true - + while IFS= read -r -u "$EVENT_FD" event; do sleep 0.2 + # Coalesce the pre-build burst only. Events received during the build stay + # queued for the next pass. Bound draining so continuous edits cannot starve + # compilation, and treat monitor EOF as failure rather than a rebuild loop. + for ((i = 0; i < 1000; i++)); do + if IFS= read -r -t 0.01 -u "$EVENT_FD" event; then + continue + else + read_status=$? + if [[ "$read_status" -eq 1 ]]; then + echo "[$SERVICE_NAME:inotifywait] File watcher stopped." >&2 + exit 1 + fi + break + fi + done echo "[$SERVICE_NAME:inotifywait] Change detected, rebuilding..." kill "$RUN_PID" 2>/dev/null || true wait "$RUN_PID" 2>/dev/null || true - bash -c "export PATH='$CURRENT_PATH' && cd '$APP_DIR' && $BUILD_CMD" || true + RUN_PID="" + bash -c "export PATH='$CURRENT_PATH' && cd '$APP_DIR' && $BUILD_CMD" {EVENT_FD}<&- || true echo "[$SERVICE_NAME:inotifywait] Restarting runtime..." - bash -c "export PATH='$CURRENT_PATH' && cd '$APP_DIR' && $RUN_CMD" & + bash -c "export PATH='$CURRENT_PATH' && cd '$APP_DIR' && $RUN_CMD" {EVENT_FD}<&- & RUN_PID=$! done + echo "[$SERVICE_NAME:inotifywait] File watcher stopped." >&2 + exit 1 fi # ── No file watcher available ───────────────────────────────────────────────