feat(runtime): event-loop watchdog for dev and serve

CHANGELOG.md

@@ -11,6 +11,46 @@ split, begin at v1.0.0 — see [VERSIONING.md](VERSIONING.md) for exactly what
 
 ## [Unreleased]
 
+### Added
+
+- **Event-loop watchdog: a server whose main thread gets stuck no longer hangs
+  silently until someone restarts it.** One synchronous loop (catastrophic
+  regex backtracking, a conversion that never terminates) blocks the single
+  thread that serves every request, so every URL stops answering at once, with
+  nothing in the logs. `dune dev` and `dune serve` now run a small watchdog
+  Worker that expects a heartbeat from the main thread every second. After 15
+  seconds of silence (`DUNE_WATCHDOG_TIMEOUT_MS`; 2 minutes until the server is
+  up, `DUNE_WATCHDOG_STARTUP_TIMEOUT_MS`, so a large site's initial indexing
+  can't put a supervised process into a kill/restart loop) it writes a report
+  to `<site>/.dune/hangs/` naming the requests that were in flight (path
+  segments longer than 24 characters are cut, so tokens in URLs don't land in
+  it; the newest 20 reports are kept; plus a native stack sample on macOS), logs
+  the same lines to stderr, and SIGKILLs the process so a supervisor (systemd
+  `Restart=`; the `dune dev` wrapper exits 137) sees a dead process instead of
+  a zombie. The kill always happens: writing the report is abandoned after 5
+  seconds. `DUNE_WATCHDOG=warn` reports without killing and is chosen
+  automatically when the process runs under `--inspect`/`--inspect-brk` (a
+  breakpoint pauses only the main thread); `DUNE_WATCHDOG=off` disables it. A
+  process that is suspended and resumed (laptop sleep, Ctrl-Z) is not mistaken
+  for a hang: the stall clock is monotonic and the Worker discards a gap in its
+  own ticks. It kills with SIGKILL rather than `Deno.exit()` because an exit
+  requested from a Worker only ends the Worker — the spinning main thread
+  carries on.
+- **Telling the watchdog about legitimately synchronous work.** The watchdog
+  can't tell an infinite loop from a long computation, only whether the loop is
+  making progress. `@dune/core/watchdog` exports two functions for code that
+  blocks on purpose: `watchdogProgress()`, called per item inside a long
+  synchronous loop (counts as a heartbeat; rate-limited, so cheap), and
+  `watchdogGrace(ms, reason)`, a declared busy window for a single long native
+  call (capped at 10 minutes, nests, expires on its own, logged each time). A
+  loop that calls `watchdogProgress()` on every iteration of an unbounded loop
+  looks alive forever; use it only where the loop has a natural end. Measured
+  on a 4,474-page site: startup indexing and the search build never held the
+  event loop for over a second, so none of Dune's own loops needed either call.
+- The main thread also logs `[dune] event loop was blocked for ~N.Ns` whenever
+  it was held for over a second, which shows a creeping problem before it
+  becomes a hang.
+
 ### Fixed
 
 - **Co-located audio and video could not seek, and would not play in

deno.json

@@ -54,7 +54,8 @@
     "./i18n": "./src/i18n/rtl.ts",
     "./api": "./src/api/handlers.ts",
     "./bootstrap": "./src/runtime/bootstrap.ts",
-    "./fresh-app": "./src/runtime/server.ts"
+    "./fresh-app": "./src/runtime/server.ts",
+    "./watchdog": "./src/runtime/watchdog.ts"
   },
   "imports": {
     "@dune/plugin-admin": "jsr:@dune/plugin-admin@^3.0",

src/cli/dev.ts

@@ -22,6 +22,7 @@ import { materializeRemoteIslands } from "./remote-islands.ts";
 import { checkLockfileStaleness } from "./lockfile.ts";
 import { resolveContentDirPath } from "../content/content-root.ts";
 import { registerEsbuildWorkerCleanup } from "./esbuild-cleanup.ts";
+import { startWatchdog } from "../runtime/watchdog.ts";
 
 export interface DevOptions {
   port?: number;
@@ -36,6 +37,11 @@ export async function devCommand(root: string, options: DevOptions = {}) {
   // invalidate any relative paths computed against the original cwd.
   root = resolve(root);
 
+  // A server whose event loop is stuck can't answer anything, log anything or
+  // be stopped gracefully; see runtime/watchdog.ts. Dev has no graceful-exit
+  // path to hook (Ctrl-C / --watch restarts end the process), so no stop().
+  const watchdog = startWatchdog({ root, label: "dune dev" });
+
   registerEsbuildWorkerCleanup();
 
   // ── Lockfile staleness check (advisory in dev) ───────────────────────────
@@ -61,6 +67,7 @@ export async function devCommand(root: string, options: DevOptions = {}) {
     await manager.init(root, { port, debug, dev: true });
     console.log(`\n  🌐 http://localhost:${port} (${manager.siteCount()} sites)\n`);
     Deno.serve({ port, handler: (req, info) => manager.handle(req, info) });
+    watchdog.ready();
     return;
   }
 
@@ -207,6 +214,8 @@ export async function devCommand(root: string, options: DevOptions = {}) {
     const { app, notifyReload } = await createDuneApp(ctx, { root, port, debug, dev: true });
     // Wire the content watcher's notifyReload reference now that the app is built.
     notifyContentReload = notifyReload;
+    // Startup is over: from here on the normal (short) stall limit applies.
+    watchdog.ready();
     return app;
   }, { port });
 }

src/cli/serve.ts

@@ -32,6 +32,7 @@ import type { Scheduler, ScheduledAction } from "../workflow/mod.ts";
 import { resolveContentDirPath } from "../content/content-root.ts";
 import { createEmailClient, createEmailProvider } from "../email/mod.ts";
 import { checkLockfileStaleness } from "./lockfile.ts";
+import { startWatchdog } from "../runtime/watchdog.ts";
 
 export interface ServeOptions {
   port?: number;
@@ -106,6 +107,10 @@ export async function serveCommand(root: string, options: ServeOptions = {}) {
   // path and we Deno.chdir() later, which would invalidate relative paths.
   root = resolve(root);
 
+  // See runtime/watchdog.ts. Its Worker can't be unref'd, so both graceful-
+  // shutdown paths below stop it explicitly or the process would never exit.
+  const watchdog = startWatchdog({ root, label: "dune serve" });
+
   // ── Lockfile staleness check ─────────────────────────────────────────────
   if (await checkLockfileStaleness(root)) {
     if (frozen) {
@@ -170,9 +175,11 @@ export async function serveCommand(root: string, options: ServeOptions = {}) {
       },
     );
 
+    watchdog.ready();
     await server.finished;
     await drainInFlight(() => inFlight, drainDeadlineMs);
     cleanup();
+    watchdog.stop();
     console.log("[dune] shutdown complete");
     return;
   }
@@ -374,6 +381,7 @@ export async function serveCommand(root: string, options: ServeOptions = {}) {
     },
   );
 
+  watchdog.ready();
   await server.finished;
 
   // Safety-net drain poll — in normal operation inFlight is already 0 here
@@ -387,5 +395,6 @@ export async function serveCommand(root: string, options: ServeOptions = {}) {
   // flush step is required here.
 
   cleanup();
+  watchdog.stop();
   console.log("[dune] shutdown complete");
 }

src/runtime/server.ts

@@ -19,6 +19,7 @@ import { mountPlugins } from "../plugins/loader.ts";
 import { mountDuneAuth } from "../auth/mount.ts";
 import { getUser, USER_HEADER } from "../auth/types.ts";
 import { isAdminPath, withSecurityHeaders } from "../cli/serve-utils.ts";
+import { trackRequest } from "./watchdog.ts";
 import { duneRoutes } from "../routing/routes.ts";
 import { buildPluginClientBundles } from "../cli/client-bundles.ts";
 import { createApiHandler } from "../api/handlers.ts";
@@ -346,6 +347,17 @@ export async function createDuneApp(
     return fc.next();
   });
 
+  // 0a. Tell the event-loop watchdog which requests are in flight, so a hang
+  // report can name the one that never finished. No-op without a watchdog.
+  app.use(async (fc) => {
+    const done = trackRequest(fc.req.method, fc.url.pathname);
+    try {
+      return await fc.next();
+    } finally {
+      done();
+    }
+  });
+
   // 0b. Standard security headers on EVERY response, whichever route family
   // produced it — plugin publicRoutes, plugin mount() routes, /api/*, /health,
   // sitemap and feeds, static files. They used to be applied by wrapping