From 9f893157c13b9bb925a73c23fdb918603c47924d Mon Sep 17 00:00:00 2001 From: Tyler Skelton Date: Wed, 7 Oct 2026 10:20:45 -0700 Subject: [PATCH 1/3] Keep the time budget whatever the backend does A bench run given --for 540s on Grok Build logged job_end durationMs 3569961: one ACP prompt turn hung for an hour and nothing bounded the wait for its reply. - Every wait on an ACP agent (session start, session/prompt, the closing summary) is bounded by the turn's deadline. At the deadline Maat sends session/cancel, gives the agent 500 ms, then ends its whole process tree: backends now lead their own process group (src/proctree.ts), SIGTERM then SIGKILL, and any group left at exit is killed. - The job then ends with the normal deadline outcome: the sealed bar judges the tree on disk, and a pass there stays unverified without a claim (7abf0fa). - Stall watchdog: an ACP agent that sends nothing for MAAT_BACKEND_STALL_MS (default 5 min, 0 = off; Maat's own tool time does not count) is a provider stall. Its turn is cancelled, the journal records an error with providerIssue/providerStall, and job_end carries providerStall: a provider issue, not a task failure. - The same pattern elsewhere: the HTTP closing summary had no deadline at all; it now gets a short grace past the budget (a tenth of it, 1-30 s). askModel takes deadlineAt and cuts each ask, retry and overload pause at the budget; review, arbiter, criteria drafting and the reference draft pass it. OpenCode asks and the auth probe are bounded and kill their process trees. - bench/local/run.py records providerStall from job_end. --- bench/local/run.py | 2 +- src/acp.ts | 188 ++++++++++++++++++++++++-- src/ask.ts | 31 ++++- src/backend.ts | 21 ++- src/cli.tsx | 4 + src/criteria.ts | 7 + src/engine.ts | 107 ++++++++++++++- src/opencode.ts | 21 ++- src/proctree.ts | 88 +++++++++++++ src/types.ts | 6 + test/acp-deadline.test.ts | 271 ++++++++++++++++++++++++++++++++++++++ 11 files changed, 719 insertions(+), 27 deletions(-) create mode 100644 src/proctree.ts create mode 100644 test/acp-deadline.test.ts diff --git a/bench/local/run.py b/bench/local/run.py index ccac39d..a3e5e1b 100644 --- a/bench/local/run.py +++ b/bench/local/run.py @@ -141,7 +141,7 @@ def run_molt(d: Path, prompt: str, log: Path) -> dict: review = ev.get("review") disagree = ev.get("checksDisagree") or [] # Only what job_end carried; absent keys stay absent (older builds). - extra = {k: ev[k] for k in ("revealed", "deadline", "endedBy", "retired", "build", "tier", "tierReason") if k in ev} + extra = {k: ev[k] for k in ("revealed", "deadline", "endedBy", "retired", "build", "tier", "tierReason", "providerStall") if k in ev} # The provider's daily cap, not the work: every later task would fail the # same way (2026-10-05: eleven tasks per arm "failed" in 140 s, 0 turns). capped = provider_capped(out, steps) diff --git a/src/acp.ts b/src/acp.ts index eea6e42..ff32d0c 100644 --- a/src/acp.ts +++ b/src/acp.ts @@ -81,11 +81,14 @@ import { execFile } from "node:child_process"; import { promisify } from "node:util"; import { errorText } from "./format.js"; +import { env as readEnv } from "./env.js"; +import { PROBE_TIMEOUT_MS } from "./watchdog.js"; import { RpcPeer, type RpcMessage } from "./jsonrpc.js"; -import type { BackendEvent, MoltTool, ToolRunner } from "./backend.js"; +import type { BackendEvent, MoltTool, SendLimits, ToolRunner } from "./backend.js"; import { GROK_BUILD_URL, OPENCODE_URL } from "./endpoint.js"; import { OPENCODE_CONFIG, OPENCODE_DEFAULT_MODEL, opencodeChildEnv, opencodeModel, opencodeModelProblem } from "./opencode.js"; import { estTokens } from "./types.js"; +import { groupSpawn, killTree, trackGroup } from "./proctree.js"; const exec = promisify(execFile); @@ -422,18 +425,30 @@ export async function acpHealth( */ async function probeAuth(spec: AcpAgentSpec): Promise<{ authenticated: boolean; detail?: string }> { const conn = new AcpConnection(spec); + let timer: ReturnType | undefined; try { - await conn.start(); - await conn.request("initialize", { - protocolVersion: 1, - clientCapabilities: { fs: { readTextFile: false, writeTextFile: false }, terminal: false }, + // Bounded like every other probe: an agent that takes the handshake and + // never answers must not hold `doctor` or the picker for ever. + const expired = new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error(`${spec.label} did not answer within ${PROBE_TIMEOUT_MS / 1000}s`)), PROBE_TIMEOUT_MS); + timer.unref?.(); }); - await conn.request("session/new", { cwd: process.cwd(), mcpServers: [] }); + const handshake = (async () => { + await conn.start(); + await conn.request("initialize", { + protocolVersion: 1, + clientCapabilities: { fs: { readTextFile: false, writeTextFile: false }, terminal: false }, + }); + await conn.request("session/new", { cwd: process.cwd(), mcpServers: [] }); + })(); + handshake.catch(() => {}); + await Promise.race([handshake, expired]); return { authenticated: true }; } catch (e) { const text = errorText(e); return { authenticated: false, detail: /auth/iu.test(text) ? "not signed in" : text }; } finally { + if (timer) clearTimeout(timer); await conn.close(); } } @@ -445,6 +460,22 @@ async function probeAuth(spec: AcpAgentSpec): Promise<{ authenticated: boolean; /** A method the agent calls on molt, and what molt answers. */ export type ClientHandler = (method: string, params: unknown) => Promise; +/** How long an agent told `session/cancel` gets to end its turn before it is killed. */ +export const CANCEL_GRACE_MS = 500; + +/** + * Silence from an ACP agent, in ms, after which its provider is taken to have + * stalled: `MAAT_BACKEND_STALL_MS`, default five minutes, 0 for never. Maat's + * own tool calls do not count as the agent's silence. + */ +export const BACKEND_STALL_MS = 5 * 60_000; + +export function backendStallMs(raw: string | undefined = readEnv("BACKEND_STALL_MS")): number { + if (raw === undefined || raw.trim() === "") return BACKEND_STALL_MS; + const n = Number(raw); + return Number.isFinite(n) && n >= 0 ? n : BACKEND_STALL_MS; +} + /** * Newline-delimited JSON-RPC over a child process's stdio. * @@ -466,6 +497,8 @@ export class AcpConnection { spawnFn?: typeof spawn; onNotify?: (method: string, params: unknown) => void; onRequest?: ClientHandler; + /** Anything at all arrived from the agent: the stall watchdog's clock. */ + onActivity?: () => void; } = {}, ) { const onRequest = opts.onRequest; @@ -501,8 +534,11 @@ export class AcpConnection { ...this.spec.env, ...(this.spec.env ? { PWD: this.opts.cwd ?? process.cwd() } : {}), }, + // Its own process group, so ending it ends what it started (src/proctree.ts). + ...groupSpawn(), }); this.child = child; + trackGroup(child); // A write to a child that has died raises EPIPE as an 'error' event on its // stdin, and an 'error' event nobody listens for is thrown — in the window, // from Electron's main process. It is the same fact as the exit below. @@ -532,6 +568,7 @@ export class AcpConnection { } private feed(chunk: string): void { + this.opts.onActivity?.(); this.peer.feed(chunk); } @@ -547,7 +584,9 @@ export class AcpConnection { async close(): Promise { this.fail(new Error("session closed")); this.child?.stdin?.end(); - this.child?.kill(); + // The agent and everything it started: a helper left running holds the + // job's directory and the machine's cores after the job has ended. + killTree(this.child); } } @@ -711,6 +750,9 @@ export class McpToolServer { await new Promise((resolve) => { if (!this.server) return resolve(); this.server.close(() => resolve()); + // An agent killed mid-call leaves its connection open; waiting for it + // to drain is waiting on a process that no longer exists. + this.server.closeAllConnections?.(); }); } } @@ -869,11 +911,22 @@ export class AcpSession { } private async start(): Promise { - const { spec, cwd, systemPrompt, tools, runTool } = this.opts; + const { spec, cwd, systemPrompt, tools } = this.opts; // A model this agent may not run is refused before anything is spawned. const pick = acpModelFor(spec, this.opts.model); if (pick.problem) throw new Error(pick.problem); this.want = pick.model; + // A tool Maat is running is Maat's time, not the agent's silence: a test + // suite that takes ten minutes is not a stalled provider. + const runTool: ToolRunner = async (...a) => { + this.busy += 1; + try { + return await this.opts.runTool(...a); + } finally { + this.busy -= 1; + this.touch(); + } + }; const mcp = new McpToolServer(tools, runTool, (event) => this.events.push({ kind: "host", event }), ); @@ -888,6 +941,7 @@ export class AcpSession { ...(this.opts.spawnFn ? { spawnFn: this.opts.spawnFn } : {}), onNotify: (method, params) => this.onNotify(method, params), onRequest: (method, params) => this.onRequest(method, params), + onActivity: () => this.touch(), }); this.conn = conn; await conn.start(); @@ -1129,14 +1183,115 @@ export class AcpSession { * loop uses: the model has stopped calling tools and produced an answer, so * the bar can run. */ - async *send(messages: readonly string[]): AsyncGenerator> { + /** When the agent last said anything, or Maat last finished a tool for it. */ + private lastActivity = Date.now(); + /** Maat tool calls running for the agent right now. */ + private busy = 0; + + private touch(): void { + this.lastActivity = Date.now(); + } + + /** + * Wait for `p`, but never past the deadline, and never through a stall. + * + * An ACP prompt turn has no clock of its own: a bench run given 540 s sat in + * one `session/prompt` for an hour, because nothing bounded the wait for + * its reply. Whatever the agent does, Maat keeps its own deadline here. + */ + private async bounded( + p: Promise, + limits: SendLimits, + ): Promise<{ ok: true; value: T } | { ok: false; stop: "deadline" | "stall"; silentMs: number }> { + const settled = p.then((value) => ({ ok: true as const, value })); + // Abandoned on a stop; a late rejection is nobody's to handle. + settled.catch(() => {}); + const stallMs = limits.stallMs && limits.stallMs > 0 ? limits.stallMs : 0; + if (limits.deadlineAt === undefined && !stallMs) return settled; + for (;;) { + const now = Date.now(); + if (limits.deadlineAt !== undefined && now >= limits.deadlineAt) { + return { ok: false, stop: "deadline", silentMs: now - this.lastActivity }; + } + const stallAt = stallMs ? (this.busy > 0 ? now + stallMs : this.lastActivity + stallMs) : Infinity; + if (now >= stallAt) return { ok: false, stop: "stall", silentMs: now - this.lastActivity }; + const wake = Math.min(limits.deadlineAt ?? Infinity, stallAt) - now; + let timer: ReturnType | undefined; + const woke = await Promise.race([ + settled, + new Promise((r) => { + timer = setTimeout(() => r(null), Math.max(1, wake)); + }), + ]); + clearTimeout(timer); + if (woke) return woke; + } + } + + /** + * Stop the agent: `session/cancel` (the protocol's own way to end a prompt + * turn), a moment for it to answer, then the whole process tree. + * + * The kill is not optional. An agent that ignores the cancel — or is hung + * past hearing it — is exactly the one that must not outlive the job. + */ + private async interrupt(answered?: Promise): Promise { + if (this.conn && this.sessionId) { + try { + this.conn.notify("session/cancel", { sessionId: this.sessionId }); + } catch { + // A dead pipe; the kill below is what counts. + } + if (answered) { + let timer: ReturnType | undefined; + await Promise.race([ + answered.catch(() => {}), + new Promise((r) => { + timer = setTimeout(r, CANCEL_GRACE_MS); + }), + ]); + clearTimeout(timer); + } + } + await this.close(); + } + + /** The `done` for a wait Maat cut short. The session is closed by then. */ + private stoppedEvent(stop: "deadline" | "stall", silentMs: number): BackendEvent { + const label = this.opts.spec.label; + const error = + stop === "deadline" + ? `the time budget ran out while ${label} was still working — its turn was cancelled` + : `${label} sent nothing for ${Math.round(silentMs / 1000)}s — provider stall, its turn was cancelled`; + return { ...this.finish(error), stopped: stop, ...(stop === "stall" ? { silentMs } : {}) }; + } + + /** + * Send messages and read back everything until the model stops. + * + * Returns at `session/prompt`'s reply, which is the same boundary molt's own + * loop uses: the model has stopped calling tools and produced an answer, so + * the bar can run. With `limits`, it also returns — with a `done` that says + * `stopped` — when the deadline passes or the agent stalls; by then the agent + * has been cancelled and its process tree ended. + */ + async *send(messages: readonly string[], limits: SendLimits = {}): AsyncGenerator> { + this.touch(); if (!this.started) { + const starting = this.start(); + starting.catch(() => {}); + let got; try { - await this.start(); + got = await this.bounded(starting, limits); } catch (e) { yield doneEvent("", errorText(e)); return; } + if (!got.ok) { + await this.interrupt(); + yield { ...doneEvent("", `${this.opts.spec.label} did not start before the ${got.stop === "deadline" ? "time budget ran out" : "stall allowance ran out"}`), stopped: got.stop }; + return; + } } const text = [this.preface, ...messages].filter(Boolean).join("\n\n"); this.preface = undefined; @@ -1165,7 +1320,14 @@ export class AcpSession { this.reader ??= this.events.drain(); for (;;) { - const next = await this.reader.next(); + const got = await this.bounded(this.reader.next(), limits); + if (!got.ok) { + const ev = this.stoppedEvent(got.stop, got.silentMs); + await this.interrupt(answered); + yield ev; + return; + } + const next = got.value; if (next.done) break; yield next.value; if (next.value.kind === "done") break; @@ -1174,7 +1336,7 @@ export class AcpSession { } /** The step's `done`, with molt's own count of what went over the wire. */ - private finish(error?: string): BackendEvent { + private finish(error?: string): Extract, { kind: "done" }> { const prompt = estTokens(this.conversation); this.conversation += this.reply; return { @@ -1214,7 +1376,7 @@ export class AcpSession { * Zero tokens is the truth here and only here: nothing was sent, so there is * nothing to have counted. Once a session exists, `finish` estimates instead. */ -function doneEvent(text: string, error?: string): BackendEvent { +function doneEvent(text: string, error?: string): Extract, { kind: "done" }> { return { kind: "done", text, diff --git a/src/ask.ts b/src/ask.ts index 5d98748..3dffb8e 100644 --- a/src/ask.ts +++ b/src/ask.ts @@ -48,6 +48,11 @@ export type AskOptions = { /** The CLI runner for the opencode ask path. Tests only. */ cliRun?: (cmd: string, args: string[], opts: object) => Promise<{ stdout: string }>; timeoutMs?: number; + /** + * The run's time budget, as an epoch ms. No ask, retry or pause waits past + * it: each ask's allowance is cut to what is left, and none starts after. + */ + deadlineAt?: number; /** Pauses between asks after an overloaded provider (ASK_OVERLOAD_BACKOFF_MS). Tests only. */ overloadBackoffMs?: number[]; /** @@ -77,10 +82,25 @@ export const ASK_MAX_TOKENS = 2_000; /** How much larger the retry is when a reply came back empty at the ceiling. */ export const ASK_RETRY_FACTOR = 4; +/** Milliseconds left before `deadlineAt`, or undefined when there is none. */ +function leftMs(deadlineAt: number | undefined): number | undefined { + return deadlineAt === undefined ? undefined : deadlineAt - Date.now(); +} + +/** The answer when the time budget is spent before an ask could be made. */ +function outOfTime(opts: AskOptions): Asked { + return { ok: false, error: `the time budget ran out${opts.what ? ` before ${opts.what}` : ""}` }; +} + export async function askModel(opts: AskOptions): Promise { const pauses = opts.overloadBackoffMs ?? ASK_OVERLOAD_BACKOFF_MS; + if ((leftMs(opts.deadlineAt) ?? 1) <= 0) return outOfTime(opts); let asked = await askSized(opts); for (let i = 0; i < pauses.length && !asked.ok && asked.transient; i++) { + // A pause that would end past the budget is not taken: the ask after it + // could not be waited for anyway. + const left = leftMs(opts.deadlineAt); + if (left !== undefined && left <= (pauses[i] ?? 0)) break; await new Promise((r) => setTimeout(r, pauses[i])); asked = await askSized(opts); } @@ -93,7 +113,7 @@ async function askSized(opts: AskOptions): Promise { // never started the answer. Once, with room: a model that empties a four // times larger ceiling the same way is not going to answer, and the second // failure says exactly that instead of "the reply was empty". - if (first.ok && first.cutOff && first.text.trim() === "") { + if (first.ok && first.cutOff && first.text.trim() === "" && (leftMs(opts.deadlineAt) ?? 1) > 0) { const bigger = (opts.maxTokens ?? ASK_MAX_TOKENS) * ASK_RETRY_FACTOR; const second = await askOnce(opts, bigger); if (second.ok && second.cutOff && second.text.trim() === "") { @@ -114,10 +134,16 @@ async function askOnce(opts: AskOptions, maxTokens: number): Promise { // Sized from this question's own prompt, and at local-hardware rates for a // self-hosted server (watchdog.ts localSpeed): the NUC's drafter timed out // on a fixed 2,000-token guess and a laptop's rates, and sealed nothing. - const limitMs = askTimeoutMs(maxTokens, opts.timeoutMs, { + const sized = askTimeoutMs(maxTokens, opts.timeoutMs, { promptTokens: Math.ceil((opts.system.length + opts.prompt.length) / 4), ...(isSelfHosted(opts.baseUrl) ? { speed: localSpeed() } : {}), }); + // Never past the run's time budget, whichever transport answers. A limit + // of 0 means "none" to every transport below, so a spent budget is refused + // here rather than passed on as 0. + const left = leftMs(opts.deadlineAt); + if (left !== undefined && left <= 0) return outOfTime(opts); + const limitMs = left === undefined ? sized : sized > 0 ? Math.min(sized, left) : left; const removed = removedSubscriptionProblem(opts.baseUrl); if (removed) return { ok: false, error: removed }; @@ -125,6 +151,7 @@ async function askOnce(opts: AskOptions, maxTokens: number): Promise { if (isOpencode(opts.baseUrl)) { const asked = await opencodeAsk({ timeoutMs: limitMs, + deadlineAt: opts.deadlineAt, model: opts.model, systemPrompt: opts.system, prompt: opts.prompt, diff --git a/src/backend.ts b/src/backend.ts index faaed85..1aea400 100644 --- a/src/backend.ts +++ b/src/backend.ts @@ -52,8 +52,27 @@ export type BackendEvent = /** Cumulative for the session. The engine takes the delta. */ cumulativeCostUsd: number; error?: string; + /** + * Maat stopped waiting: the turn's clock ran out (`deadline`), or the + * agent said nothing at all for the stall allowance (`stall`). The agent + * was told to cancel and its process tree was ended; the session is gone. + */ + stopped?: "deadline" | "stall"; + /** For `stall`: how long the agent had been silent. */ + silentMs?: number; }; +/** + * Bounds on one `send`. Every wait on the agent is cut at `deadlineAt`, and a + * silence longer than `stallMs` (outside Maat's own tool calls) is a stall. + */ +export type SendLimits = { + /** Epoch ms after which nothing more is waited for. */ + deadlineAt?: number; + /** Silence from the agent, in ms, that counts as a stalled provider. 0 or absent: none. */ + stallMs?: number; +}; + /** * The three methods the engine needs from a subprocess backend. * @@ -62,7 +81,7 @@ export type BackendEvent = * and neither should the loop. */ export interface BackendSession { - send(messages: readonly string[]): AsyncGenerator>; + send(messages: readonly string[], limits?: SendLimits): AsyncGenerator>; close(): Promise; costSoFarUsd(): number; /** diff --git a/src/cli.tsx b/src/cli.tsx index fc937e3..4698fc0 100644 --- a/src/cli.tsx +++ b/src/cli.tsx @@ -1220,6 +1220,8 @@ async function autoDraft( // stage of each draft read this and never the folder the work is changing. const snapshot = inputs?.snapshot ?? drafterSnapshotFor(args); const none: ReturnType = { taskChecks: [], taskNotes: [] }; + // Under --for, no draft, critique or retry waits past the run's budget. + const deadlineAt = args.forMs ? Date.now() + args.forMs : undefined; // Drafted, then read cold by a critic against the task text: a check that // invents or guesses is dropped (with a task quote), and a draft where // nothing runs the deliverable is asked for once more. See criteria.ts. @@ -1236,6 +1238,7 @@ async function autoDraft( cwd: args.cwd, reasoningEffort: judgeEffort(args.reasoningChecks ?? args.reasoning), latency: engine.askLatency, + deadlineAt, // What is ready when a time budget stops the wait (RunOptions.criteriaSoFar). onProgress: (d) => { if (soFar) soFar.draft = d; @@ -1300,6 +1303,7 @@ function startReference(args: Args): Promise<{ check: Check; note: Record { if (!r.ok) { process.stderr.write(`maat: no reference check — ${r.why}\n`); diff --git a/src/criteria.ts b/src/criteria.ts index d54a503..bc359e9 100644 --- a/src/criteria.ts +++ b/src/criteria.ts @@ -594,6 +594,7 @@ async function edgeProbes(opts: Parameters[0], view: strin acpSpawn: opts.acpSpawn, cliRun: opts.cliRun, timeoutMs: opts.timeoutMs, + deadlineAt: opts.deadlineAt, reasoningEffort: opts.reasoningEffort, latency: opts.latency, }).catch(() => null); @@ -634,6 +635,8 @@ export async function draftCriteria(opts: { cliRun?: (cmd: string, args: string[], opts: object) => Promise<{ stdout: string }>; /** How long the HTTP question may wait for its answer; see askTimeoutMs. Tests only. */ timeoutMs?: number; + /** The run's time budget, as an epoch ms (AskOptions.deadlineAt). */ + deadlineAt?: number; /** Pause before re-asking after empty replies (EMPTY_DRAFT_DELAY_MS). Tests only. */ emptyRetryDelayMs?: number; reasoningEffort?: string; @@ -702,6 +705,7 @@ export async function draftCriteria(opts: { acpSpawn: opts.acpSpawn, cliRun: opts.cliRun, timeoutMs: opts.timeoutMs, + deadlineAt: opts.deadlineAt, reasoningEffort: opts.reasoningEffort, latency: opts.latency, }); @@ -722,6 +726,7 @@ export async function draftCriteria(opts: { acpSpawn: opts.acpSpawn, cliRun: opts.cliRun, timeoutMs: opts.timeoutMs, + deadlineAt: opts.deadlineAt, reasoningEffort: opts.reasoningEffort, latency: opts.latency, }); @@ -746,6 +751,7 @@ export async function draftCriteria(opts: { acpSpawn: opts.acpSpawn, cliRun: opts.cliRun, timeoutMs: opts.timeoutMs, + deadlineAt: opts.deadlineAt, reasoningEffort: opts.reasoningEffort, latency: opts.latency, }); @@ -1003,6 +1009,7 @@ async function critiqued( acpSpawn: opts.acpSpawn, cliRun: opts.cliRun, timeoutMs: opts.timeoutMs, + deadlineAt: opts.deadlineAt, reasoningEffort: opts.reasoningEffort, latency: opts.latency, }); diff --git a/src/engine.ts b/src/engine.ts index 1b69b42..38900bb 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -112,7 +112,7 @@ import { parseLenient } from "./lenient-json.js"; import { LONG_RATE_LIMIT_MS, longQuotaText, rateLimitResetAt, untilText, providerErrorText, normalizeMessage, readStream, transientProviderError, type ProviderError, type StreamAccumulator, type Usage } from "./stream.js"; import { Fragments, SafeStream } from "./live.js"; import { Transcript, toolDetail } from "./transcript.js"; -import { acpAgentFor, acpHealth, acpModels, AcpSession, isAcp } from "./acp.js"; +import { acpAgentFor, acpHealth, acpModels, AcpSession, backendStallMs, isAcp } from "./acp.js"; import { type BackendSession, type ToolRunner } from "./backend.js"; import { endpointProblem, removedSubscriptionProblem } from "./endpoint.js"; import { @@ -1156,6 +1156,15 @@ export function changesSomething(name: string, rawArgs: string): boolean { * sealing what is ready: a tenth of the budget, between 45 s and 2 min. No * budget, no bound — a person at the keyboard can wait. */ +/** + * The grace a closing summary gets past the deadline: a tenth of the budget, + * between 1 s and 30 s. The summary is a courtesy; a run given 540 s must not + * spend another five minutes on it (or an hour, on a backend that hangs). + */ +export function deadlineGraceMs(budgetMs: number): number { + return Math.min(30_000, Math.max(1_000, Math.round(budgetMs / 10))); +} + export function criteriaWaitMs(budgetMs: number): number | undefined { if (!budgetMs) return undefined; return Math.min(120_000, Math.max(45_000, Math.round(budgetMs / 10))); @@ -1431,6 +1440,13 @@ export type EngineConfig = { * REQUEST_IDLE_MS (src/watchdog.ts); 0 turns it off. */ requestIdleMs?: number; + /** + * Silence from a subprocess (ACP) backend, in ms, after which its provider + * is taken to have stalled and the turn ends as a provider issue. Unset + * means `MAAT_BACKEND_STALL_MS`, then BACKEND_STALL_MS (src/acp.ts); 0 + * turns it off. + */ + backendStallMs?: number; /** * The allowance before a request's first byte, overriding the one scaled * from the prompt and output sizes (see firstByteMs in src/watchdog.ts). @@ -2178,6 +2194,8 @@ export class Engine { * ran; two of them had passing work on disk. */ private turnEndedBy: "deadline" | "provider" | undefined; + /** A subprocess backend went silent past the stall allowance this turn (a provider issue). */ + private turnProviderStall = false; /** `--revert` put this turn's work back, so a person cannot accept it as it stands. */ private turnRestored = false; /** The pre-turn working tree, as a commit object nothing else can see. */ @@ -2531,6 +2549,24 @@ export class Engine { return ms > 0 && this.turnStartedAt > 0 ? this.turnStartedAt + ms : undefined; } + /** + * How long a closing summary may wait, as an epoch ms, under a time budget. + * + * The salvage runs after the clock has stopped the model, so the turn's own + * deadline is already behind it; it gets a short grace (deadlineGraceMs) + * past whichever is later, the deadline or now. Without a budget, undefined: + * the idle and stall watchdogs still bound it. + */ + private salvageDeadlineAt(): number | undefined { + const at = this.deadlineAt(); + return at === undefined ? undefined : Math.max(at, Date.now()) + deadlineGraceMs(this.turnDeadlineMs); + } + + /** The stall allowance for a subprocess backend (EngineConfig.backendStallMs). */ + private stallMs(): number { + return this.cfg.backendStallMs ?? backendStallMs(); + } + /** * Whether the independent review would outlast the turn, and so is not run. * Six of eight no-verdict runs on Mercury 2.5 passed their bar and then died @@ -3564,6 +3600,9 @@ export class Engine { stream: false, }, isSelfHosted(this.cfg.baseUrl) ? localSpeed() : undefined), idleMs: idle, + // Under a time budget, bounded by it plus a short grace: a salvage + // waits on the same provider that may just have hung. + deadlineAt: this.salvageDeadlineAt(), }); try { // The salvage is a request like any other, so it speaks whichever @@ -3700,13 +3739,13 @@ export class Engine { if (!texts.length) return; let done: - | { text: string; promptTokens: number; completionTokens: number; error?: string } + | { text: string; promptTokens: number; completionTokens: number; error?: string; stopped?: "deadline" | "stall" } | undefined; /** The last answer that called nothing, in case `done` carries no text. */ let last = ""; this.ccNoTools = true; try { - for await (const ev of cc.send([texts.join("\n\n")])) { + for await (const ev of cc.send([texts.join("\n\n")], { deadlineAt: this.salvageDeadlineAt(), stallMs: this.stallMs() })) { if (ev.kind === "done") done = ev; else if (ev.kind === "assistant" && !ev.toolCalls.length && ev.text) last = ev.text; } @@ -3725,6 +3764,14 @@ export class Engine { this.ccNoTools = false; } + if (done?.stopped) { + // Cut by the clock or a stall: the agent was cancelled and killed, and + // the next turn starts a fresh session. + await this.dropAcpSession(); + log?.append("note", { text: `salvage not written: ${done.error ?? done.stopped}`, ...(done.stopped === "stall" ? { providerIssue: true, providerStall: true } : {}) }); + yield { kind: "info", text: "could not write a closing summary — the work above is all there is" }; + return; + } if (!done || done.error) { // A session that vanished mid-salvage was cancelled, not broken: // `cancel()` ends it, because ending it is the only way to stop the @@ -4070,6 +4117,7 @@ export class Engine { this.turnAllRetired = false; this.turnRevealed = []; this.turnEndedBy = undefined; + this.turnProviderStall = false; this.turnRestored = false; this.refusedThisTurn = false; this.turnReview = undefined; @@ -4217,6 +4265,8 @@ export class Engine { acpSpawn: this.cfg.acpSpawn, // Never longer than the turn has left: a review that outlives the clock ends it without a verdict. ...(this.timeLeftMs() !== undefined ? { timeoutMs: Math.max(1_000, this.timeLeftMs()!) } : {}), + // And every retry and pause inside it, not just one ask. + deadlineAt: this.deadlineAt(), }, }).catch(() => null); this.cfg.journal?.append("review", review ? { confirmed: review.confirmed, votes: review.votes, violations: review.violations.length } : { ran: false }); @@ -4298,6 +4348,7 @@ export class Engine { ...(this.reviewSkipped ? { unreviewed: true } : {}), ...(this.turnEndedBy ? { endedBy: this.turnEndedBy, ...(this.turnEndedBy === "deadline" ? { deadline: true } : {}) } : {}), ...(passedAtEnd ? { passedAtEnd: true } : {}), + ...(this.turnProviderStall ? { providerStall: true } : {}), ...(checksDisagree ? { checksDisagree: failing.map((r) => r.name) } : {}), ...(outcome === "verified" && this.turnRevealed.length ? { revealed: [...this.turnRevealed] } : {}), ...(review ? { review: { confirmed: review.confirmed, votes: review.votes, violations: review.violations } } : {}), @@ -4478,7 +4529,7 @@ export class Engine { ctx: ToolContext, ): AsyncGenerator< EngineEvent, - { msg: Msg; usage: Usage; finishReason?: string; streamed: boolean } | "cancelled" | null + { msg: Msg; usage: Usage; finishReason?: string; streamed: boolean } | "cancelled" | "deadline" | "stall" | null > { this.ccCtx = ctx; this.ccCancelled = false; @@ -4555,10 +4606,15 @@ export class Engine { cachedTokens: number; cumulativeCostUsd: number; error?: string; + stopped?: "deadline" | "stall"; + silentMs?: number; } | undefined; - for await (const ev of cc.send(said)) { + // Every wait on the agent is bounded: by the turn's deadline, and by a + // stall allowance on its silence. Neither existed, and one Grok Build + // prompt turn held a 540 s bench job for an hour. + for await (const ev of cc.send(said, { deadlineAt: this.deadlineAt(), stallMs: this.stallMs() })) { if (ev.kind === "host") { yield ev.event; } else if (ev.kind === "delta") { @@ -4603,6 +4659,32 @@ export class Engine { await this.dropAcpSession(); return null; } + if (done.stopped === "deadline") { + // The clock, not the provider: the step loop closes the turn the way it + // closes any that ran out of time, judging what is on disk. + if (deferred !== null) this.transcript.push({ role: "assistant", content: deferred }); + ctx.log?.append("note", { + text: `time budget reached during step ${step}'s ${this.backendLabel} turn — session/cancel sent, agent process tree ended`, + }); + await this.dropAcpSession(); + return "deadline"; + } + if (done.stopped === "stall") { + // The provider went quiet, which is not the task failing. Journalled the + // way a provider's quota wall is — a provider issue — so a benchmark + // does not count it against the work. + const silent = waited(done.silentMs ?? this.stallMs()); + const text = `provider stall: ${this.backendLabel} sent nothing for ${silent} — its turn was cancelled and the agent process tree ended`; + ctx.log?.append("error", { text, providerIssue: true, providerStall: true, silentMs: done.silentMs ?? null }); + if (deferred !== null) this.transcript.push({ role: "assistant", content: deferred }); + this.turnProviderStall = true; + yield { + kind: "error", + text: `${text}. This is the provider, not the task. Nothing was verified; the work above still happened.`, + }; + await this.dropAcpSession(); + return "stall"; + } if (done.error) { const wall = longQuotaText(done.error); ctx.log?.append("error", { text: wall ?? done.error, ...(wall ? { providerCapped: true } : {}) }); @@ -5815,6 +5897,17 @@ export class Engine { yield { kind: "cancelled", filesWritten: wrote }; return; } + if (got === "deadline") { + deadlineInterrupted = true; + continue; + } + if (got === "stall") { + // Work that happened is judged, as after any provider failure; no + // closing summary is asked of a provider that has just gone silent. + const judged = this.turnWrites.length > 0 ? yield* judgeOnDisk("provider", "") : false; + if (!judged) this.turnEndedBy = "provider"; + return; + } if (!got) return; msg = got.msg; usage = got.usage; @@ -7022,7 +7115,8 @@ export class Engine { cwd: this.cwd, reasoningEffort: this.cfg.review?.reasoningEffort ?? this.cfg.reasoningEffort, fetchFn: this.cfg.fetchFn, - acpSpawn: this.cfg.acpSpawn, + acpSpawn: this.cfg.acpSpawn, + deadlineAt: this.deadlineAt(), }, }).catch(() => null); log?.append("dispute", { @@ -7355,6 +7449,7 @@ export class Engine { fetchFn: this.cfg.fetchFn, acpSpawn: this.cfg.acpSpawn, ...(this.timeLeftMs() !== undefined ? { timeoutMs: Math.max(1_000, this.timeLeftMs()!) } : {}), + deadlineAt: this.deadlineAt(), }, }).catch(() => null); if (review && !review.confirmed && review.violations.length && this.cfg.reviewAdvisory !== true) { diff --git a/src/opencode.ts b/src/opencode.ts index 9fed1a8..34d2800 100644 --- a/src/opencode.ts +++ b/src/opencode.ts @@ -42,6 +42,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { errorText } from "./format.js"; import { isOpencodeUrl, opencodeModelProblem } from "./endpoint.js"; +import { groupSpawn, killTree, trackGroup } from "./proctree.js"; export { OPENCODE_URL, opencodeModelProblem } from "./endpoint.js"; @@ -179,6 +180,8 @@ export type OpencodeAskOptions = { systemPrompt: string; prompt: string; timeoutMs?: number; + /** The run's time budget, as an epoch ms: no attempt waits past it. */ + deadlineAt?: number; /** Injected in tests. Real callers spawn the CLI. */ run?: (cmd: string, args: string[], opts: object) => Promise<{ stdout: string }>; }; @@ -192,12 +195,19 @@ const NO_TOOLS = function runCli(cmd: string, args: string[], opts: { cwd: string; env: NodeJS.ProcessEnv; timeoutMs: number }): Promise<{ stdout: string }> { return new Promise((resolve, reject) => { // stdin must be ignored: `opencode run` waits on an open pipe forever. - const child = spawn(cmd, args, { cwd: opts.cwd, env: opts.env, stdio: ["ignore", "pipe", "pipe"] }); + // Its own process group, so a timeout ends what it started too (src/proctree.ts). + const child = spawn(cmd, args, { cwd: opts.cwd, env: opts.env, stdio: ["ignore", "pipe", "pipe"], ...groupSpawn() }); + trackGroup(child); let stdout = ""; let stderr = ""; child.stdout.on("data", (d: Buffer) => (stdout += d)); child.stderr.on("data", (d: Buffer) => (stderr += d)); - const timer = setTimeout(() => child.kill("SIGKILL"), opts.timeoutMs); + let timedOut = false; + const timer = setTimeout(() => { + // `close` reports a signal only for the leader; say it was the clock. + timedOut = true; + killTree(child); + }, opts.timeoutMs); child.on("error", (e) => { clearTimeout(timer); reject(e); @@ -205,7 +215,7 @@ function runCli(cmd: string, args: string[], opts: { cwd: string; env: NodeJS.Pr child.on("close", (code, signal) => { clearTimeout(timer); // A non-zero exit still carries the error event on stdout; the caller reads it. - if (signal) reject(new Error(`opencode did not answer within ${Math.round(opts.timeoutMs / 1000)}s`)); + if (signal || timedOut) reject(new Error(`opencode did not answer within ${Math.round(opts.timeoutMs / 1000)}s`)); else resolve({ stdout: stdout || (code ? stderr : "") }); }); }); @@ -230,11 +240,14 @@ export async function opencodeAsk(opts: OpencodeAskOptions): Promise(); +let hooked = false; + +/** Spawn options that make the child lead its own process group, where that exists. */ +export function groupSpawn(): { detached: boolean } { + return { detached: process.platform !== "win32" }; +} + +function signalGroup(pid: number, signal: NodeJS.Signals): boolean { + try { + process.kill(-pid, signal); + return true; + } catch { + return false; + } +} + +/** + * Track a child started with `groupSpawn()`. Its group is killed when the + * leader exits (whatever it left running is an orphan nobody will reap) and + * when Maat exits. + */ +export function trackGroup(child: ChildProcess): void { + const pid = child.pid; + if (pid === undefined || process.platform === "win32") return; + groups.add(pid); + if (!hooked) { + hooked = true; + process.on("exit", () => { + for (const g of groups) signalGroup(g, "SIGKILL"); + groups.clear(); + }); + } + child.once("exit", () => { + // The leader is gone; anything still in its group was its, and stays only + // because nobody ended it. + signalGroup(pid, "SIGKILL"); + groups.delete(pid); + }); +} + +/** End a backend child and everything in its process group. */ +export function killTree(child: ChildProcess | undefined): void { + if (!child) return; + const pid = child.pid; + if (pid === undefined || process.platform === "win32" || !groups.has(pid)) { + try { + child.kill(); + } catch { + // Already gone. + } + return; + } + if (!signalGroup(pid, "SIGTERM")) { + try { + child.kill(); + } catch { + // Already gone. + } + } + const t = setTimeout(() => { + if (!groups.has(pid)) return; + signalGroup(pid, "SIGKILL"); + groups.delete(pid); + }, KILL_GRACE_MS); + t.unref?.(); +} diff --git a/src/types.ts b/src/types.ts index ac87a5f..3dc2fa3 100644 --- a/src/types.ts +++ b/src/types.ts @@ -645,6 +645,12 @@ export type EngineEvent = * said the work was done. */ passedAtEnd?: boolean; + /** + * A subprocess backend sent nothing for the stall allowance + * (MAAT_BACKEND_STALL_MS) and its turn was cancelled: a provider issue, + * not a verdict on the work. + */ + providerStall?: boolean; /** * What a passing turn earned (src/tiers.ts): "verified" only when a * check ran the deliverable and asserted a value (or a person wrote the diff --git a/test/acp-deadline.test.ts b/test/acp-deadline.test.ts new file mode 100644 index 0000000..7c68243 --- /dev/null +++ b/test/acp-deadline.test.ts @@ -0,0 +1,271 @@ +/** + * A time budget is Maat's own, whatever the backend does. + * + * Measured 2026-10-07: a bench run given `--for 540s` on Grok Build logged a + * job_end of 3,569,961 ms. One ACP prompt turn hung for an hour and nothing + * bounded the wait for its reply. These pin the fix: every wait on a backend + * is cut at the deadline, the agent is told `session/cancel` and its whole + * process tree is ended, the journal says why, and a backend that goes silent + * is a provider stall — not a failure of the task. + */ +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { chmodSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { after, describe, it } from "node:test"; +import { ACP_AGENTS, backendStallMs, BACKEND_STALL_MS } from "../src/acp.js"; +import { askModel } from "../src/ask.js"; +import { deadlineGraceMs, Engine } from "../src/engine.js"; +import { Journal } from "../src/journal.js"; +import type { EngineEvent } from "../src/types.js"; +import { scriptedAcpAgent } from "./acp-agent.js"; +import { allowAll, drain, workspace } from "./helpers.js"; + +const GROK = ACP_AGENTS.find((a) => a.name === "grok-build")!; +const CLI = join(process.cwd(), "dist-test", "src", "cli.js"); + +const cleanups: (() => void)[] = []; +after(() => cleanups.forEach((c) => c())); +function ws(): string { + const w = workspace(); + cleanups.push(w.cleanup); + return w.dir; +} + +function jobEnd(events: EngineEvent[]) { + const e = events.find((x) => x.kind === "job_end"); + assert.ok(e && e.kind === "job_end"); + return e; +} + +/** + * A `grok` that takes the prompt and never answers — the hang as measured. + * Worst case on purpose: it ignores SIGTERM, and it has a child of its own + * (`sleep 300`) that a kill of one pid would leave behind. + */ +const HUNG_AGENT = `#!/usr/bin/env node +const fs = require("fs"); +const { spawn } = require("child_process"); +const log = process.env.FAKE_ACP_LOG; +const note = (s) => { if (log) fs.appendFileSync(log, s + "\\n"); }; +note("pid " + process.pid); +const kid = spawn("sleep", ["300"], { stdio: "ignore" }); +note("pid " + kid.pid); +process.on("SIGTERM", () => note("sigterm ignored")); +const send = (o) => process.stdout.write(JSON.stringify(o) + "\\n"); +let buf = ""; +process.stdin.on("data", (d) => { + buf += d; + for (let i; (i = buf.indexOf("\\n")) >= 0; ) { + const line = buf.slice(0, i).trim(); + buf = buf.slice(i + 1); + if (!line) continue; + const m = JSON.parse(line); + if (m.method === "initialize") send({ jsonrpc: "2.0", id: m.id, result: { protocolVersion: 1, agentCapabilities: {} } }); + else if (m.method === "session/new") send({ jsonrpc: "2.0", id: m.id, result: { sessionId: "s1" } }); + else if (m.method === "session/cancel") note("cancel"); + else if (m.method === "session/prompt") note("prompt"); + else if (m.id !== undefined) send({ jsonrpc: "2.0", id: m.id, result: {} }); + } +}); +`; + +function alive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +async function allGone(pids: number[], withinMs: number): Promise { + const until = Date.now() + withinMs; + for (;;) { + const left = pids.filter(alive); + if (!left.length || Date.now() > until) return left; + await new Promise((r) => setTimeout(r, 50)); + } +} + +describe("an ACP agent that never answers, under --for 5s", { skip: process.platform === "win32" }, () => { + it("ends the job on time, cancels and kills the agent's process tree, and journals why", async () => { + const root = ws(); + const bin = join(root, "bin"); + const dir = join(root, "project"); + mkdirSync(bin); + mkdirSync(dir); + writeFileSync(join(bin, "grok"), HUNG_AGENT); + chmodSync(join(bin, "grok"), 0o755); + const agentLog = join(root, "agent.log"); + + const t0 = Date.now(); + const { code, stdout } = await new Promise<{ code: number; stdout: string }>((resolve) => { + const child = spawn( + process.execPath, + [CLI, "run", "write a.txt", "--url", "grok-build", "--model", "grok-4.6", "--for", "5s", "--json", "--yes", "--cwd", dir], + { + stdio: ["ignore", "pipe", "ignore"], + env: { ...process.env, PATH: `${bin}:${process.env.PATH ?? ""}`, FAKE_ACP_LOG: agentLog, MOLT_CONFIG_DIR: join(root, "cfg") }, + }, + ); + let out = ""; + child.stdout.on("data", (d: Buffer) => (out += d.toString())); + child.on("exit", (c) => resolve({ code: c ?? -1, stdout: out })); + }); + const took = Date.now() - t0; + + // 5 s of budget, the cancel grace, and a closing summary bounded by its + // 1 s grace — not the hour the hung prompt would have held it. + assert.ok(took < 11_000, `the job took ${took}ms on a 5s budget`); + const end = stdout + .split("\n") + .filter((l) => l.startsWith("{")) + .map((l) => JSON.parse(l) as { kind: string; durationMs?: number; deadline?: boolean; endedBy?: string; outcome?: string }) + .find((e) => e.kind === "job_end"); + assert.ok(end, `no job_end in:\n${stdout}`); + assert.equal(end.deadline, true); + assert.equal(end.endedBy, "deadline"); + assert.notEqual(end.outcome, "verified"); + assert.ok((end.durationMs ?? Infinity) < 9_000, `job_end durationMs ${end.durationMs}`); + assert.notEqual(code, 0); + + // The agent was asked to cancel, and then it and its child were ended — + // SIGTERM ignored or not. + const said = readFileSync(agentLog, "utf8"); + assert.match(said, /^prompt$/m); + assert.match(said, /^cancel$/m); + const pids = [...said.matchAll(/^pid (\d+)$/gm)].map((m) => Number(m[1])); + assert.ok(pids.length >= 2, said); + assert.deepEqual(await allGone(pids, 3_000), [], "agent processes outlived the job"); + + // The journal says the clock ended it, and how. + const logs = join(dir, ".maat", "log"); + const file = readdirSync(logs).find((f) => f.endsWith(".jsonl")); + assert.ok(file); + const rows = Journal.read(join(logs, file)); + assert.equal(rows.filter((r) => r.kind === "deadline").length, 1); + assert.ok( + rows.some((r) => r.kind === "note" && /session\/cancel sent, agent process tree ended/.test(String(r.data.text))), + "the journal does not say the agent was cancelled and killed", + ); + }); +}); + +describe("the deadline inside the engine, on a subprocess backend", () => { + it("cuts a hung prompt turn at the budget and judges the turn as a deadline", async () => { + const dir = ws(); + const agent = scriptedAcpAgent([{ hang: true }, { hang: true }]); + const journal = new Journal(dir, "acp-deadline"); + const engine = new Engine({ + baseUrl: GROK.url, model: "grok-4.6", provider: "grok-build", cwd: dir, bar: null, + acpSpawn: agent.spawnFn, autonomy: "high", turnDeadlineMs: 400, journal, + }); + const t0 = Date.now(); + const events = await drain(engine.run("write a.txt", allowAll)); + assert.ok(Date.now() - t0 < 4_000, `took ${Date.now() - t0}ms on a 400ms budget`); + const e = jobEnd(events); + assert.equal(e.deadline, true); + assert.equal(e.endedBy, "deadline"); + assert.notEqual(e.outcome, "verified"); + assert.ok(!e.providerStall); + const rows = Journal.read(journal.path); + assert.equal(rows.filter((r) => r.kind === "deadline").length, 1); + assert.ok(!rows.some((r) => r.kind === "error"), "running out of time is not an error"); + }); +}); + +describe("a backend that goes silent", () => { + it("is a provider stall: cancelled, journalled as a provider issue, not a task failure", async () => { + const dir = ws(); + const agent = scriptedAcpAgent([{ hang: true }]); + const journal = new Journal(dir, "acp-stall"); + const engine = new Engine({ + baseUrl: GROK.url, model: "grok-4.6", provider: "grok-build", cwd: dir, bar: null, + acpSpawn: agent.spawnFn, autonomy: "high", backendStallMs: 300, journal, + }); + const t0 = Date.now(); + const events = await drain(engine.run("write a.txt", allowAll)); + assert.ok(Date.now() - t0 < 4_000); + const e = jobEnd(events); + assert.equal(e.providerStall, true); + assert.equal(e.endedBy, "provider"); + assert.ok(!e.deadline); + assert.ok(events.some((x) => x.kind === "error" && /provider stall/.test(x.text) && /not the task/.test(x.text))); + const stall = Journal.read(journal.path).find((r) => r.kind === "error" && r.data.providerStall === true); + assert.ok(stall, "no provider-stall row in the journal"); + assert.equal(stall.data.providerIssue, true); + }); + + it("does not count Maat's own tool time as the agent's silence", async () => { + const dir = ws(); + const agent = scriptedAcpAgent([ + { calls: [{ name: "bash", args: { command: "sleep 0.8" } }], text: "Slept." }, + ]); + const engine = new Engine({ + baseUrl: GROK.url, model: "grok-4.6", provider: "grok-build", cwd: dir, bar: null, + acpSpawn: agent.spawnFn, autonomy: "high", backendStallMs: 300, + }); + const events = await drain(engine.run("sleep a bit", allowAll)); + const e = jobEnd(events); + assert.ok(!e.providerStall, "a tool that ran longer than the stall allowance was called a stall"); + assert.ok(events.some((x) => x.kind === "assistant_text" && /Slept/.test(x.text))); + }); + + it("reads MAAT_BACKEND_STALL_MS, five minutes by default, 0 for never", () => { + assert.equal(BACKEND_STALL_MS, 300_000); + assert.equal(backendStallMs(undefined), 300_000); + assert.equal(backendStallMs("1500"), 1500); + assert.equal(backendStallMs("0"), 0); + assert.equal(backendStallMs("soon"), 300_000); + }); +}); + +/** A fetch that answers nothing until it is aborted. */ +const hangingFetch = (async (_url: string, init?: RequestInit) => + new Promise((_, reject) => { + init?.signal?.addEventListener("abort", () => reject(init.signal?.reason ?? new Error("aborted"))); + })) as unknown as typeof fetch; + +describe("the other waits on a provider, under a time budget", () => { + it("an ask (draft, critic, review) is cut at the deadline, not at its own allowance", async () => { + const t0 = Date.now(); + const r = await askModel({ + baseUrl: "http://provider.test/v1", model: "m", system: "s", prompt: "p", + fetchFn: hangingFetch, timeoutMs: 60_000, deadlineAt: Date.now() + 300, + }); + assert.equal(r.ok, false); + assert.ok(Date.now() - t0 < 3_000, `the ask waited ${Date.now() - t0}ms past a 300ms budget`); + }); + + it("an ask with the budget already spent is not made at all", async () => { + let asked = 0; + const r = await askModel({ + baseUrl: "http://provider.test/v1", model: "m", system: "s", prompt: "p", what: "reviewing the claim", + fetchFn: (async () => { + asked += 1; + return new Response("{}"); + }) as unknown as typeof fetch, + deadlineAt: Date.now() - 1, + }); + assert.deepEqual(r, { ok: false, error: "the time budget ran out before reviewing the claim" }); + assert.equal(asked, 0); + }); + + it("the closing summary after the deadline is bounded by a short grace, on HTTP too", async () => { + const dir = ws(); + // The watchdog is off, so only the clock can end either request. + const engine = new Engine({ + baseUrl: "http://provider.test/v1", model: "m", cwd: dir, bar: null, fetchFn: hangingFetch, + stream: false, autonomy: "high", turnDeadlineMs: 300, requestIdleMs: 0, + }); + const t0 = Date.now(); + const events = await drain(engine.run("hi", allowAll)); + const took = Date.now() - t0; + assert.ok(took < 300 + deadlineGraceMs(300) + 2_000, `took ${took}ms`); + assert.equal(jobEnd(events).deadline, true); + assert.equal(deadlineGraceMs(300), 1_000); + assert.equal(deadlineGraceMs(540_000), 30_000); + assert.equal(deadlineGraceMs(60_000), 6_000); + }); +}); From 1f30e1469831fe2597ee9a383e00ad23ad688546 Mon Sep 17 00:00:00 2001 From: Tyler Skelton Date: Wed, 7 Oct 2026 14:15:05 -0700 Subject: [PATCH 2/3] Signals end the agent's process group; one deadline for drafting and the reference Backends lead their own process group, so Ctrl-C, a hangup or a harness's SIGTERM to Maat no longer reached them, and Node runs no exit handlers on a signal nobody listens for: the agent and its helpers were orphaned. proctree now installs SIGINT/SIGTERM/SIGHUP listeners where none exist (as background.ts does), ends every tracked group and exits 130/143/129. - A leader that exits gets the same SIGTERM-then-SIGKILL for its group as killTree, instead of an immediate SIGKILL, and keeps killTree's timer. - The criteria drafter and the reference draft share one deadline taken when the job starts them, instead of each taking its own Date.now(). - Doc comments put back above deadlineGraceMs/criteriaWaitMs and send(). --- src/acp.ts | 7 ---- src/cli.tsx | 17 ++++++--- src/engine.ts | 10 ++--- src/proctree.ts | 78 ++++++++++++++++++++++++++++++--------- test/acp-deadline.test.ts | 49 ++++++++++++++++++++++++ 5 files changed, 125 insertions(+), 36 deletions(-) diff --git a/src/acp.ts b/src/acp.ts index ff32d0c..1ca8c6c 100644 --- a/src/acp.ts +++ b/src/acp.ts @@ -1176,13 +1176,6 @@ export class AcpSession { : { outcome: { outcome: "cancelled" } }; } - /** - * Send messages and read back everything until the model stops. - * - * Returns at `session/prompt`'s reply, which is the same boundary molt's own - * loop uses: the model has stopped calling tools and produced an answer, so - * the bar can run. - */ /** When the agent last said anything, or Maat last finished a tool for it. */ private lastActivity = Date.now(); /** Maat tool calls running for the agent right now. */ diff --git a/src/cli.tsx b/src/cli.tsx index 4698fc0..2689b83 100644 --- a/src/cli.tsx +++ b/src/cli.tsx @@ -1215,13 +1215,14 @@ async function autoDraft( args: Args, soFar?: { draft?: Draft; sealed?: boolean; late?: boolean }, inputs?: { snapshot: DrafterInputs; used: string[] }, + deadlineAt?: number, ): Promise> { // Taken once, now, before the first step: the second try below and every // stage of each draft read this and never the folder the work is changing. const snapshot = inputs?.snapshot ?? drafterSnapshotFor(args); const none: ReturnType = { taskChecks: [], taskNotes: [] }; - // Under --for, no draft, critique or retry waits past the run's budget. - const deadlineAt = args.forMs ? Date.now() + args.forMs : undefined; + // Under --for, no draft, critique or retry waits past the run's budget + // (the job's own deadline, taken once by the caller: runDeadlineAt). // Drafted, then read cold by a critic against the task text: a check that // invents or guesses is dropped (with a task quote), and a draft where // nothing runs the deliverable is asked for once more. See criteria.ts. @@ -1284,7 +1285,7 @@ async function autoDraft( * there is no python3, the project is too large to copy, or no reference * applies. */ -function startReference(args: Args): Promise<{ check: Check; note: Record } | null> | undefined { +function startReference(args: Args, deadlineAt?: number): Promise<{ check: Check; note: Record } | null> | undefined { const here = commandsHere(args.cwd); if (!here.present.includes("python3")) { process.stderr.write("maat: no reference check — python3 is not installed here\n"); @@ -1303,7 +1304,7 @@ function startReference(args: Args): Promise<{ check: Check; note: Record { if (!r.ok) { process.stderr.write(`maat: no reference check — ${r.why}\n`); @@ -1540,7 +1541,11 @@ async function cmdRun(args: Args, ask = false): Promise { // The drafter's inputs, frozen here; the engine journals their hash at turn // start and compares it with what each drafter stage actually used. const drafterInputs = args.autoCriteria && !ask ? { snapshot: drafterSnapshotFor(args), used: [] as string[] } : undefined; - const pendingCriteria = drafterInputs ? autoDraft(engine, args, soFar, drafterInputs) : undefined; + // One deadline for everything this job starts before its turn, taken once: + // the drafter and the reference each used to take their own Date.now(), a + // little later than the job's, and could wait that much past --for. + const runDeadlineAt = args.forMs ? Date.now() + args.forMs : undefined; + const pendingCriteria = drafterInputs ? autoDraft(engine, args, soFar, drafterInputs, runDeadlineAt) : undefined; const draftInputs = drafterInputs ? { sha: drafterInputsHash(drafterInputs.snapshot), used: () => [...drafterInputs.used] } : undefined; @@ -1553,7 +1558,7 @@ async function cmdRun(args: Args, ask = false): Promise { return soFar.draft?.checks.length ? sealDraft(soFar.draft, args) : Promise.resolve({ taskChecks: [], taskNotes: [] }); } : undefined; - const referenceCheck = args.reference && !ask ? startReference(args) : undefined; + const referenceCheck = args.reference && !ask ? startReference(args, runDeadlineAt) : undefined; let undetermined = false; /** The turn's own verdict, from job_end. */ let outcome: string | undefined; diff --git a/src/engine.ts b/src/engine.ts index 38900bb..f20b073 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -1151,11 +1151,6 @@ export function changesSomething(name: string, rawArgs: string): boolean { return cmds.length === 0 || !cmds.every((c) => isReadOnlyCommand(c)); } -/** - * How long a turn with a time budget waits for its drafted checks before - * sealing what is ready: a tenth of the budget, between 45 s and 2 min. No - * budget, no bound — a person at the keyboard can wait. - */ /** * The grace a closing summary gets past the deadline: a tenth of the budget, * between 1 s and 30 s. The summary is a courtesy; a run given 540 s must not @@ -1165,6 +1160,11 @@ export function deadlineGraceMs(budgetMs: number): number { return Math.min(30_000, Math.max(1_000, Math.round(budgetMs / 10))); } +/** + * How long a turn with a time budget waits for its drafted checks before + * sealing what is ready: a tenth of the budget, between 45 s and 2 min. No + * budget, no bound — a person at the keyboard can wait. + */ export function criteriaWaitMs(budgetMs: number): number | undefined { if (!budgetMs) return undefined; return Math.min(120_000, Math.max(45_000, Math.round(budgetMs / 10))); diff --git a/src/proctree.ts b/src/proctree.ts index fa4f581..8c2d5a2 100644 --- a/src/proctree.ts +++ b/src/proctree.ts @@ -13,15 +13,67 @@ * when Maat itself exits is killed on the way out, so a `process.exit` at the * end of a headless run never leaves an agent behind. Windows has no process * groups; there the leader is killed as before. + * + * A detached group is out of the terminal's foreground group, so Ctrl-C and a + * hangup no longer reach the agent with Maat — and Node runs no `exit` + * handlers when it dies of a signal it has no listener for. So the first + * tracked group also installs SIGINT/SIGTERM/SIGHUP listeners (only where + * nobody else listens, as src/background.ts does): they end every group and + * exit with the conventional 128 + signal number, which runs the `exit` + * handlers of everything else on the way out. */ import type { ChildProcess } from "node:child_process"; /** How long a group gets between SIGTERM and SIGKILL. */ export const KILL_GRACE_MS = 1_500; -const groups = new Set(); +/** Tracked group leaders, and the pending SIGKILL of a group being ended. */ +const groups = new Map | null>(); let hooked = false; +const SIGNAL_EXIT: Record<"SIGINT" | "SIGTERM" | "SIGHUP", number> = { SIGHUP: 129, SIGINT: 130, SIGTERM: 143 }; + +function killAllNow(): void { + for (const [g, t] of groups) { + if (t) clearTimeout(t); + signalGroup(g, "SIGKILL"); + } + groups.clear(); +} + +function hookExit(): void { + if (hooked) return; + hooked = true; + process.on("exit", killAllNow); + for (const sig of Object.keys(SIGNAL_EXIT) as (keyof typeof SIGNAL_EXIT)[]) { + if (process.listenerCount(sig) === 0) { + process.once(sig, () => { + // A moment's SIGTERM would be kinder, but the process is going now and + // nothing would be left to send the SIGKILL after it. + for (const g of groups.keys()) signalGroup(g, "SIGTERM"); + killAllNow(); + process.exit(SIGNAL_EXIT[sig]); + }); + } + } +} + +/** + * End a group: SIGTERM now, SIGKILL after KILL_GRACE_MS for whatever did not + * listen. A group already being ended keeps the SIGKILL it has. + */ +function endGroup(pid: number, termSent = false): void { + if (groups.get(pid)) return; + if (!termSent) signalGroup(pid, "SIGTERM"); + const t = setTimeout(() => { + if (!groups.has(pid)) return; + signalGroup(pid, "SIGKILL"); + groups.delete(pid); + }, KILL_GRACE_MS); + t.unref?.(); + groups.set(pid, t); +} + /** Spawn options that make the child lead its own process group, where that exists. */ export function groupSpawn(): { detached: boolean } { return { detached: process.platform !== "win32" }; @@ -44,19 +96,13 @@ function signalGroup(pid: number, signal: NodeJS.Signals): boolean { export function trackGroup(child: ChildProcess): void { const pid = child.pid; if (pid === undefined || process.platform === "win32") return; - groups.add(pid); - if (!hooked) { - hooked = true; - process.on("exit", () => { - for (const g of groups) signalGroup(g, "SIGKILL"); - groups.clear(); - }); - } + groups.set(pid, null); + hookExit(); child.once("exit", () => { // The leader is gone; anything still in its group was its, and stays only - // because nobody ended it. - signalGroup(pid, "SIGKILL"); - groups.delete(pid); + // because nobody ended it. It gets the same SIGTERM-then-SIGKILL as + // killTree (and keeps killTree's timer if that is what ended the leader). + if (groups.has(pid)) endGroup(pid); }); } @@ -72,6 +118,7 @@ export function killTree(child: ChildProcess | undefined): void { } return; } + if (groups.get(pid)) return; if (!signalGroup(pid, "SIGTERM")) { try { child.kill(); @@ -79,10 +126,5 @@ export function killTree(child: ChildProcess | undefined): void { // Already gone. } } - const t = setTimeout(() => { - if (!groups.has(pid)) return; - signalGroup(pid, "SIGKILL"); - groups.delete(pid); - }, KILL_GRACE_MS); - t.unref?.(); + endGroup(pid, true); } diff --git a/test/acp-deadline.test.ts b/test/acp-deadline.test.ts index 7c68243..ab95bd7 100644 --- a/test/acp-deadline.test.ts +++ b/test/acp-deadline.test.ts @@ -152,6 +152,55 @@ describe("an ACP agent that never answers, under --for 5s", { skip: process.plat }); }); +describe("Maat killed by a signal while an ACP agent hangs", { skip: process.platform === "win32" }, () => { + // The agent leads its own process group (src/proctree.ts), so a Ctrl-C or a + // harness's SIGTERM to Maat no longer reaches it — and Node runs no `exit` + // handlers on a signal nobody listens for. Maat has to end the group itself. + for (const [sig, code] of [["SIGTERM", 143], ["SIGINT", 130], ["SIGHUP", 129]] as const) { + it(`${sig}: Maat ends the agent's whole group and exits ${code}`, async () => { + const root = ws(); + const bin = join(root, "bin"); + const dir = join(root, "project"); + mkdirSync(bin); + mkdirSync(dir); + writeFileSync(join(bin, "grok"), HUNG_AGENT); + chmodSync(join(bin, "grok"), 0o755); + const agentLog = join(root, "agent.log"); + const child = spawn( + process.execPath, + [CLI, "run", "write a.txt", "--url", "grok-build", "--model", "grok-4.6", "--json", "--yes", "--cwd", dir], + { + stdio: ["ignore", "ignore", "ignore"], + env: { ...process.env, PATH: `${bin}:${process.env.PATH ?? ""}`, FAKE_ACP_LOG: agentLog, MOLT_CONFIG_DIR: join(root, "cfg") }, + }, + ); + const exited = new Promise<{ code: number | null; signal: NodeJS.Signals | null }>((r) => + child.on("exit", (c, s) => r({ code: c, signal: s })), + ); + // Wait until the agent is holding the prompt, with its own child running. + const until = Date.now() + 20_000; + let said = ""; + while (Date.now() < until) { + try { + said = readFileSync(agentLog, "utf8"); + } catch { + said = ""; + } + if (/^prompt$/m.test(said)) break; + await new Promise((r) => setTimeout(r, 50)); + } + assert.match(said, /^prompt$/m, "the agent never got the prompt"); + const pids = [...said.matchAll(/^pid (\d+)$/gm)].map((m) => Number(m[1])); + assert.ok(pids.length >= 2, said); + + child.kill(sig); + const end = await exited; + assert.deepEqual(end, { code, signal: null }); + assert.deepEqual(await allGone(pids, 3_000), [], `agent processes outlived Maat's ${sig}`); + }); + } +}); + describe("the deadline inside the engine, on a subprocess backend", () => { it("cuts a hung prompt turn at the budget and judges the turn as a deadline", async () => { const dir = ws(); From 6092f60ea33d9193146da65655c0c46898631d7f Mon Sep 17 00:00:00 2001 From: Tyler Skelton Date: Wed, 7 Oct 2026 14:31:18 -0700 Subject: [PATCH 3/3] An interrupted ACP turn stops Maat's tool call too; the agent's own tools are not a stall - A deadline or stall closed the agent but left the bash/write_file it was waiting on running, so the tree went on changing while it was judged. The interrupt now aborts the in-flight tool call (the per-call controller the engine already uses for Ctrl-C), waits up to 5 s for it to end, and refuses any tool call that arrives after it. - The stall clock excused only Maat's own tool time. A builtin the agent announced and has not finished (GROK_OWN_TOOLS running a test suite) is now excused the same way; the deadline still bounds it. - Test helper: a zombie counts as gone. --- src/acp.ts | 43 +++++++++++++++++++++++++++++++++++++-- src/engine.ts | 3 +++ test/acp-agent.ts | 16 ++++++++++++++- test/acp-deadline.test.ts | 43 ++++++++++++++++++++++++++++++++++++--- 4 files changed, 99 insertions(+), 6 deletions(-) diff --git a/src/acp.ts b/src/acp.ts index 1ca8c6c..16de030 100644 --- a/src/acp.ts +++ b/src/acp.ts @@ -827,10 +827,19 @@ export type AcpOptions = { systemPrompt: string; tools: readonly MoltTool[]; runTool: ToolRunner; + /** + * Stop whatever Maat tool call is running for the agent. Called when a + * deadline or a stall ends the agent's turn: the agent is gone, and a + * `bash` it asked for must not go on changing the tree being judged. + */ + abortTools?: () => void; /** Injected in tests, which drive a scripted agent rather than a real one. */ spawnFn?: typeof spawn; }; +/** How long an interrupted turn waits for its aborted tool calls to end. */ +export const TOOL_ABORT_WAIT_MS = 5_000; + /** * An ACP session, alive for as long as molt's is. * @@ -919,10 +928,16 @@ export class AcpSession { // A tool Maat is running is Maat's time, not the agent's silence: a test // suite that takes ten minutes is not a stalled provider. const runTool: ToolRunner = async (...a) => { + // The agent was interrupted; nothing it asks for now may run. + if (this.interrupted) return "[molt: the turn was stopped. No more tools.]"; this.busy += 1; + const run = this.opts.runTool(...a); + const settled = run.then(() => {}, () => {}); + this.running.add(settled); try { - return await this.opts.runTool(...a); + return await run; } finally { + this.running.delete(settled); this.busy -= 1; this.touch(); } @@ -1180,6 +1195,10 @@ export class AcpSession { private lastActivity = Date.now(); /** Maat tool calls running for the agent right now. */ private busy = 0; + /** Those calls, settled or not, so an interrupt can wait for them to end. */ + private running = new Set>(); + /** Set by an interrupt: no tool call starts after it. */ + private interrupted = false; private touch(): void { this.lastActivity = Date.now(); @@ -1206,7 +1225,12 @@ export class AcpSession { if (limits.deadlineAt !== undefined && now >= limits.deadlineAt) { return { ok: false, stop: "deadline", silentMs: now - this.lastActivity }; } - const stallAt = stallMs ? (this.busy > 0 ? now + stallMs : this.lastActivity + stallMs) : Infinity; + // Silence while a tool runs is not a stall: Maat's own tools (busy), or + // the agent's own builtins it announced and has not finished (inFlight; + // GROK_OWN_TOOLS runs a test suite that way with no ACP traffic). The + // deadline still bounds both. + const working = this.busy > 0 || this.inFlight.size > 0; + const stallAt = stallMs ? (working ? now + stallMs : this.lastActivity + stallMs) : Infinity; if (now >= stallAt) return { ok: false, stop: "stall", silentMs: now - this.lastActivity }; const wake = Math.min(limits.deadlineAt ?? Infinity, stallAt) - now; let timer: ReturnType | undefined; @@ -1229,6 +1253,7 @@ export class AcpSession { * past hearing it — is exactly the one that must not outlive the job. */ private async interrupt(answered?: Promise): Promise { + this.interrupted = true; if (this.conn && this.sessionId) { try { this.conn.notify("session/cancel", { sessionId: this.sessionId }); @@ -1247,6 +1272,20 @@ export class AcpSession { } } await this.close(); + // The agent is gone, but a Maat tool call it started (a build, a script + // that rewrites files) would go on changing the tree the judge is about + // to read. Stop it, and wait for it to end, within reason. + if (this.running.size) { + this.opts.abortTools?.(); + let timer: ReturnType | undefined; + await Promise.race([ + Promise.allSettled([...this.running]), + new Promise((r) => { + timer = setTimeout(r, TOOL_ABORT_WAIT_MS); + }), + ]); + clearTimeout(timer); + } } /** The `done` for a wait Maat cut short. The session is closed by then. */ diff --git a/src/engine.ts b/src/engine.ts index f20b073..ae62795 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -4507,6 +4507,9 @@ export class Engine { systemPrompt: system, tools: TOOLS, runTool, + // The per-call controller invokeTool sets: a deadline or a stall that + // ends the agent's turn ends the command it was waiting on too. + abortTools: () => this.running?.abort(), ...(this.cfg.acpSpawn ? { spawnFn: this.cfg.acpSpawn } : {}), }); } diff --git a/test/acp-agent.ts b/test/acp-agent.ts index 8964f00..947f4e9 100644 --- a/test/acp-agent.ts +++ b/test/acp-agent.ts @@ -50,6 +50,11 @@ export type ScriptedAcpTurn = { authError?: boolean; /** Accept the prompt and never answer it. */ hang?: boolean; + /** + * Run one of its own builtins for this long first: announced, silent while + * it runs (as GROK_OWN_TOOLS runs a test suite), then completed. + */ + ownToolMs?: number; }; export type ScriptedAgent = { @@ -236,6 +241,14 @@ export function scriptedAcpAgent(turns: ScriptedAcpTurn[], models: AgentModels = turn += 1; if (script.hang) return; + if (script.ownToolMs) { + const upd = (update: Record) => + write({ jsonrpc: "2.0", method: "session/update", params: { sessionId: "s1", update } }); + upd({ sessionUpdate: "tool_call", toolCallId: "own_1", title: "run_terminal_cmd", toolName: "run_terminal_cmd" }); + await new Promise((r) => setTimeout(r, script.ownToolMs)); + upd({ sessionUpdate: "tool_call_update", toolCallId: "own_1", status: "completed" }); + } + for (const b of script.autoTools ?? []) { // Announced, then completed, with no permission request in // between. molt cannot stop this one; it can only say so. @@ -353,7 +366,8 @@ export function scriptedAcpAgent(turns: ScriptedAcpTurn[], models: AgentModels = }); if ((await ask(wire)) !== "yes") continue; if (endpoint) { - await mcpCall(endpoint, "tools/call", { name: script.wirePrefix ? c.name : wire, arguments: c.args }, mcpId++); + // An agent whose tool server went away (an interrupted turn) gets an error back, not a crash. + await mcpCall(endpoint, "tools/call", { name: script.wirePrefix ? c.name : wire, arguments: c.args }, mcpId++).catch(() => ({})); } } diff --git a/test/acp-deadline.test.ts b/test/acp-deadline.test.ts index ab95bd7..e2581a7 100644 --- a/test/acp-deadline.test.ts +++ b/test/acp-deadline.test.ts @@ -9,8 +9,8 @@ * is a provider stall — not a failure of the task. */ import assert from "node:assert/strict"; -import { spawn } from "node:child_process"; -import { chmodSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from "node:fs"; +import { execFileSync, spawn } from "node:child_process"; +import { chmodSync, existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { after, describe, it } from "node:test"; import { ACP_AGENTS, backendStallMs, BACKEND_STALL_MS } from "../src/acp.js"; @@ -73,7 +73,13 @@ process.stdin.on("data", (d) => { function alive(pid: number): boolean { try { process.kill(pid, 0); - return true; + } catch { + return false; + } + // kill(pid, 0) succeeds on a zombie, which is dead but not yet reaped (a + // container with no init to reap the orphaned `sleep`). + try { + return !execFileSync("ps", ["-o", "stat=", "-p", String(pid)]).toString().trim().startsWith("Z"); } catch { return false; } @@ -224,6 +230,24 @@ describe("the deadline inside the engine, on a subprocess backend", () => { }); }); +describe("a Maat tool call running when the deadline ends the agent's turn", { skip: process.platform === "win32" }, () => { + it("is stopped before the tree is judged", async () => { + const dir = ws(); + const agent = scriptedAcpAgent([{ calls: [{ name: "bash", args: { command: "sleep 2 && echo late > late.txt" } }], text: "Done." }]); + const engine = new Engine({ + baseUrl: GROK.url, model: "grok-4.6", provider: "grok-build", cwd: dir, bar: null, + acpSpawn: agent.spawnFn, autonomy: "high", turnDeadlineMs: 500, + }); + const t0 = Date.now(); + const events = await drain(engine.run("write late.txt", allowAll)); + assert.ok(Date.now() - t0 < 2_000, `the turn waited out the command: ${Date.now() - t0}ms`); + assert.equal(jobEnd(events).endedBy, "deadline"); + // Long enough for the command to have finished, had it not been stopped. + await new Promise((r) => setTimeout(r, 2_500)); + assert.equal(existsSync(join(dir, "late.txt")), false, "the command went on writing after the deadline"); + }); +}); + describe("a backend that goes silent", () => { it("is a provider stall: cancelled, journalled as a provider issue, not a task failure", async () => { const dir = ws(); @@ -261,6 +285,19 @@ describe("a backend that goes silent", () => { assert.ok(events.some((x) => x.kind === "assistant_text" && /Slept/.test(x.text))); }); + it("does not count the agent's own running tool as silence either", async () => { + const dir = ws(); + const agent = scriptedAcpAgent([{ ownToolMs: 900, text: "Tests ran." }]); + const engine = new Engine({ + baseUrl: GROK.url, model: "grok-4.6", provider: "grok-build", cwd: dir, bar: null, + acpSpawn: agent.spawnFn, autonomy: "high", backendStallMs: 300, + }); + const events = await drain(engine.run("run the tests", allowAll)); + const e = jobEnd(events); + assert.ok(!e.providerStall, "the agent's own tool running past the stall allowance was called a stall"); + assert.ok(events.some((x) => x.kind === "assistant_text" && /Tests ran/.test(x.text))); + }); + it("reads MAAT_BACKEND_STALL_MS, five minutes by default, 0 for never", () => { assert.equal(BACKEND_STALL_MS, 300_000); assert.equal(backendStallMs(undefined), 300_000);