feat(cli): dune ps probes health and reads the instance registry

CHANGELOG.md

@@ -13,6 +13,25 @@ split, begin at v1.0.0 — see [VERSIONING.md](VERSIONING.md) for exactly what
 
 ### Added
 
+- **`dune ps` now tells a hung server from a healthy one, and finds servers
+  that registered themselves.** Every listening instance is sent a `GET
+  /health/live` (which shows up in its access log): to the port it registered
+  when it did, otherwise to every port it listens on, counting only dune's own
+  answer so a debugger's inspector port can't make a hung server look healthy.
+  A blocked event loop still lets the kernel accept the connection but never
+  answers it, so an instance that fails three probes in a row (3 seconds each,
+  about 10 seconds in all) is shown as **NOT RESPONDING** and counted
+  separately — it looks perfectly healthy to `lsof`. One failed probe proves
+  nothing, and a port that speaks HTTP but not like dune gives "can't tell",
+  not a hung verdict. Instances that wrote an entry to the instance registry
+  are also listed even if their command line doesn't look like dune, count as
+  confirmed, and show their `@dune/core` version. `dune stop --idle` also takes
+  confirmed instances that are listening but not responding, once they are at
+  least 2 minutes old. With the watchdog in its default mode a hung server is
+  killed within seconds, so this mostly matters under `DUNE_WATCHDOG=warn`/`off`;
+  a server stuck in a loop can't handle SIGTERM, so `dune stop <pid> --force` is
+  how to end one.
+
 - **Running instances now register themselves.** `dune dev` and `dune serve`
   write a small record (pid, process start time, mode, site root, title, port,
   `@dune/core` version) to `~/.dune/instances/<pid>.json`

src/cli/instances.ts

@@ -42,6 +42,7 @@
 
 import { dirname, fromFileUrl, isAbsolute, join, resolve } from "@std/path";
 import { parseUserYaml } from "../security/safe-yaml.ts";
+import { listRegisteredInstances } from "../runtime/instance-registry.ts";
 
 /** A dune dev/serve-shaped process (tree) found running on this machine. */
 export interface DuneInstance {
@@ -66,6 +67,15 @@ export interface DuneInstance {
    * so we can't tell.
    */
   portStatus: "listening" | "not-listening" | "unknown";
+  /**
+   * Whether a listening instance answered an HTTP request: `"responding"`,
+   * `"not-responding"` (accepts connections but gives no answer in time —
+   * the signature of a blocked event loop, or just a very busy server), or
+   * `"unknown"` (not listening, or not probed).
+   */
+  responsive: "responding" | "not-responding" | "unknown";
+  /** `@dune/core` version, when the instance registered itself (see runtime/instance-registry.ts). */
+  version: string | null;
   /** `ps`'s own elapsed-time string (e.g. "01:23:45" or "2-09:03:55"). */
   elapsed: string | null;
   /** `elapsed` in seconds, or null when unparseable. */
@@ -397,6 +407,70 @@ async function confirmSiteRoot(dirs: (string | null)[]): Promise<string | null>
   return null;
 }
 
+// ── Health probe ─────────────────────────────────────────────────────────────
+
+export const PROBE_TIMEOUT_MS = 3_000;
+const PROBE_ATTEMPTS = 3;
+const PROBE_GAP_MS = 500;
+
+/** What answered on a port: dune's own health check, some other HTTP service, or nothing in time. */
+export type PortProbe = "dune" | "other" | "none";
+
+/**
+ * Ask `port` for `/health/live`. A blocked event loop still lets the kernel
+ * accept the TCP connection, but nothing ever answers it, which is exactly what
+ * the timeout catches. A JSON body with a `status` field is dune's own answer;
+ * anything else that speaks HTTP (a debugger's inspector port, another tool)
+ * is "other". Probes 127.0.0.1, not `localhost`, so IPv6 resolution can't
+ * change the result. Note this sends a real GET to the instance, which shows
+ * up in its access log.
+ */
+export async function probePort(port: number, timeoutMs = PROBE_TIMEOUT_MS): Promise<PortProbe> {
+  try {
+    const res = await fetch(`http://127.0.0.1:${port}/health/live`, {
+      signal: AbortSignal.timeout(timeoutMs),
+      redirect: "manual",
+    });
+    const text = (await res.text()).slice(0, 512);
+    try {
+      const body = JSON.parse(text);
+      if (res.ok && body && typeof body === "object" && typeof body.status === "string") return "dune";
+    } catch { /* not JSON */ }
+    return "other";
+  } catch {
+    return "none";
+  }
+}
+
+/**
+ * Is this instance answering? Probes the port the instance itself registered
+ * when there is one (any HTTP answer counts), otherwise every listening port
+ * (only dune's own answer counts, so a debugger's inspector port can't make a
+ * hung server look healthy). A single failure proves nothing — a busy server
+ * can be slow for a few seconds — so it takes `attempts` failures in a row,
+ * spaced `gapMs` apart. If some port speaks HTTP but none answers like dune,
+ * the verdict is "unknown" rather than a claim that it is hung.
+ */
+export async function judgeResponsive(
+  ports: number[],
+  registeredPort: number | null,
+  probe: (port: number) => Promise<PortProbe>,
+  attempts = PROBE_ATTEMPTS,
+  gapMs = PROBE_GAP_MS,
+): Promise<"responding" | "not-responding" | "unknown"> {
+  if (ports.length === 0) return "unknown";
+  const useRegistered = registeredPort !== null && ports.includes(registeredPort);
+  const candidates = useRegistered ? [registeredPort] : ports;
+  let sawOther = false;
+  for (let i = 0; i < attempts; i++) {
+    const results = await Promise.all(candidates.map(probe));
+    if (results.some((r) => (useRegistered ? r !== "none" : r === "dune"))) return "responding";
+    if (results.includes("other")) sawOther = true;
+    if (i < attempts - 1) await new Promise((r) => setTimeout(r, gapMs));
+  }
+  return sawOther ? "unknown" : "not-responding";
+}
+
 /**
  * Find every dune dev/serve instance on this machine.
  *
@@ -406,8 +480,15 @@ async function confirmSiteRoot(dirs: (string | null)[]): Promise<string | null>
  * are null in that case, and the caller should say so plainly.
  */
 export async function findDuneInstances(
-  opts: { onWarning?: (message: string) => void } = {},
+  opts: {
+    onWarning?: (message: string) => void;
+    probe?: (port: number) => Promise<PortProbe>;
+    probeAttempts?: number;
+    probeGapMs?: number;
+  } = {},
 ): Promise<DuneInstance[]> {
+  const probe = opts.probe ?? probePort;
+  const registeredPorts = new Map<number, number | null>(); // instance pid -> port it registered
   const allRows = await readProcessTable();
   // Only this user's processes: another user's server is not ours to list or
   // signal (the kill would be refused anyway). Our own uid comes from our own
@@ -506,12 +587,64 @@ export async function findDuneInstances(
       title,
       ports,
       portStatus: !haveLsof ? "unknown" : ports.length > 0 ? "listening" : "not-listening",
+      responsive: "unknown",
+      version: null,
       elapsed: c.elapsed,
       ageSeconds: parseElapsedSeconds(c.elapsed),
       confirmed: confirmedRoot !== null,
       startTimes: Object.fromEntries(pids.flatMap((p) => byPid.get(p) ? [[p, byPid.get(p)!.started]] : [])),
       command: c.command,
     });
   }
+
+  // 5. Instances that registered themselves (runtime/instance-registry.ts):
+  // attach what they report about themselves, and add any the command-line
+  // matching missed (a server started in an unusual way).
+  for (const rec of await listRegisteredInstances()) {
+    const existing = instances.find((i) => i.pids.includes(rec.pid));
+    if (existing) {
+      existing.version = rec.version;
+      existing.confirmed = true; // the process says so itself
+      registeredPorts.set(existing.pid, rec.port);
+      if (existing.root === null) {
+        existing.root = rec.root;
+        existing.title = rec.title ?? (await readSiteTitle(rec.root));
+      }
+      continue;
+    }
+    const row = byPid.get(rec.pid);
+    if (!row) continue;
+    const pids = [rec.pid, ...descendantsOf(rec.pid, children)];
+    const ports = haveLsof ? await getListeningPorts(pids) : [];
+    instances.push({
+      pid: rec.pid,
+      pids,
+      mode: rec.mode,
+      root: rec.root,
+      title: rec.title ?? (await readSiteTitle(rec.root)),
+      ports,
+      portStatus: !haveLsof ? "unknown" : ports.length > 0 ? "listening" : "not-listening",
+      responsive: "unknown",
+      version: rec.version,
+      elapsed: row.elapsed,
+      ageSeconds: parseElapsedSeconds(row.elapsed),
+      confirmed: true,
+      startTimes: Object.fromEntries(pids.flatMap((p) => byPid.get(p) ? [[p, byPid.get(p)!.started]] : [])),
+      command: row.command,
+    });
+    registeredPorts.set(rec.pid, rec.port);
+  }
+
+  // 6. Probe every listening instance, all at once.
+  await Promise.all(instances.map(async (inst) => {
+    inst.responsive = await judgeResponsive(
+      inst.ports,
+      registeredPorts.get(inst.pid) ?? null,
+      probe,
+      opts.probeAttempts,
+      opts.probeGapMs,
+    );
+  }));
+
   return instances.sort((a, b) => a.pid - b.pid);
 }

src/cli/ps.ts

@@ -16,7 +16,12 @@ export interface PsDeps {
 }
 
 export function describePort(inst: DuneInstance): string {
-  if (inst.portStatus === "listening") return inst.ports.map((p) => `http://localhost:${p}`).join(", ");
+  if (inst.portStatus === "listening") {
+    const urls = inst.ports.map((p) => `http://localhost:${p}`).join(", ");
+    return inst.responsive === "not-responding"
+      ? `${urls} — NOT RESPONDING (no answer to a health check; the server may be hung)`
+      : urls;
+  }
   if (inst.portStatus === "not-listening") return "not listening (no open port; likely a hung or leftover process)";
   return "port unknown (lsof not available)";
 }
@@ -27,7 +32,7 @@ export function formatInstance(inst: DuneInstance, opts: { fullCommand?: boolean
     ? `  (+${inst.pids.length - 1} child process${inst.pids.length > 2 ? "es" : ""})`
     : "";
   return [
-    `  PID ${inst.pid}  ${inst.mode.padEnd(5)}  up ${inst.elapsed ?? "?"}${extra}`,
+    `  PID ${inst.pid}  ${inst.mode.padEnd(5)}  up ${inst.elapsed ?? "?"}${inst.version ? `  v${inst.version}` : ""}${extra}`,
     `    ${inst.title ?? "(untitled)"}`,
     `    ${inst.root ?? "(site root unconfirmed — no config/site.yaml via --root, PWD or live cwd)"}`,
     `    ${describePort(inst)}`,
@@ -65,9 +70,13 @@ export async function psCommand(deps: PsDeps = {}): Promise<void> {
   for (const inst of all) for (const line of formatInstance(inst)) log(line);
 
   const serving = all.filter((i) => i.portStatus === "listening").length;
-  const idle = all.filter((i) => i.portStatus === "not-listening").length;
-  log(`  ${all.length} instance${all.length === 1 ? "" : "s"}: ${serving} listening, ${idle} not listening\n`);
-  if (idle > 0) {
-    log("  `dune stop --idle` stops only the not-listening ones; `dune stop <pid>` stops one instance.");
+  const stuck = all.filter((i) => i.responsive === "not-responding").length;
+  const notListening = all.filter((i) => i.portStatus === "not-listening").length;
+  log(
+    `  ${all.length} instance${all.length === 1 ? "" : "s"}: ${serving} listening` +
+      `${stuck > 0 ? ` (${stuck} not responding)` : ""}, ${notListening} not listening\n`,
+  );
+  if (stuck + notListening > 0) {
+    log("  `dune stop --idle` stops only the ones that aren't serving (not listening or not responding); `dune stop <pid>` stops one instance.");
   }
 }

src/cli/stop.ts

@@ -165,7 +165,7 @@ export interface StopOptions {
   pid?: string;
   /** The instance listening on this port. */
   port?: string;
-  /** Only instances that are running but not listening (hung or leftover). */
+  /** Only instances that aren't serving: running but not listening, or listening but not responding (hung or leftover). */
   idle?: boolean;
   /** Every instance `dune ps` lists. */
   all?: boolean;
@@ -182,8 +182,9 @@ export const STOP_USAGE = `Usage: dune stop <pid> | --port <n> | --idle | --all
 
   <pid>        the instance this process belongs to (see \`dune ps\`)
   --port <n>   the instance listening on that port
-  --idle       only instances that are running but have not been
-               listening for at least 2 minutes (younger ones may still
+  --idle       only instances that aren't serving: running but not
+               listening, or listening but not responding to a health
+               check, for at least 2 minutes (younger ones may still
                be starting)
   --all        every confirmed instance \`dune ps\` lists
   --force      SIGKILL processes that ignore SIGTERM (default: report them)
@@ -252,12 +253,12 @@ export function selectInstances(all: DuneInstance[], opts: StopOptions): Selecti
   }
 
   if (opts.idle) {
-    const notServing = confirmed.filter((i) => i.portStatus === "not-listening");
+    const notServing = confirmed.filter((i) => i.portStatus === "not-listening" || i.responsive === "not-responding");
     const mature = notServing.filter((i) => i.ageSeconds === null || i.ageSeconds >= IDLE_MIN_AGE_SECONDS);
     const young = notServing.length - mature.length;
     if (young > 0) {
       notes.push(
-        `${plural(young, "instance", "instances")} not listening yet ${young === 1 ? "was" : "were"} left out because ` +
+        `${plural(young, "instance", "instances")} not serving yet ${young === 1 ? "was" : "were"} left out because ` +
           `${young === 1 ? "it is" : "they are"} under ${IDLE_MIN_AGE_SECONDS / 60} minutes old and may still be starting.`,
       );
     }
@@ -273,7 +274,7 @@ export function selectInstances(all: DuneInstance[], opts: StopOptions): Selecti
       const why = [...notes].map((n) => `\n  ${n}`).join("");
       return {
         ok: false,
-        message: `No idle instances: ${confirmed.length === 0 ? "none of the running instances is a confirmed dune site" : `all ${confirmed.length} confirmed running instance${confirmed.length === 1 ? " is" : "s are"} listening`}.${why}`,
+        message: `No idle instances: ${confirmed.length === 0 ? "none of the running instances is a confirmed dune site" : `all ${confirmed.length} confirmed running instance${confirmed.length === 1 ? " is" : "s are"} listening and responding`}.${why}`,
         exitCode: 0,
       };
     }

tests/cli/ps_helpers.ts

@@ -74,6 +74,8 @@ export function fakeInstance(overrides: Partial<DuneInstance> = {}): DuneInstanc
     title: "Fake",
     ports: [],
     portStatus: "not-listening",
+    responsive: "unknown",
+    version: null,
     elapsed: "01:00",
     ageSeconds: 3600,
     confirmed: true,