diff --git a/test/acp-deadline.test.ts b/test/acp-deadline.test.ts index e2581a7..215d37f 100644 --- a/test/acp-deadline.test.ts +++ b/test/acp-deadline.test.ts @@ -13,7 +13,7 @@ import { execFileSync, spawn } from "node:child_process"; import { chmodSync, existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { after, describe, it } from "node:test"; -import { ACP_AGENTS, backendStallMs, BACKEND_STALL_MS } from "../src/acp.js"; +import { ACP_AGENTS, backendStallMs, BACKEND_STALL_MS, CANCEL_GRACE_MS } from "../src/acp.js"; import { askModel } from "../src/ask.js"; import { deadlineGraceMs, Engine } from "../src/engine.js"; import { Journal } from "../src/journal.js"; @@ -105,8 +105,26 @@ describe("an ACP agent that never answers, under --for 5s", { skip: process.plat chmodSync(join(bin, "grok"), 0o755); const agentLog = join(root, "agent.log"); - const t0 = Date.now(); - const { code, stdout } = await new Promise<{ code: number; stdout: string }>((resolve) => { + // The clock is started when the agent has the prompt, not when the CLI is + // spawned. `--for` is a ceiling on the turn (the turn's own clock is + // job_end's durationMs, checked below); the node start-up and set-up + // before the turn are outside it, and on a loaded machine they are not + // small. Measured 2026-10-08 with 150 CPU burners on a 10-core Mac: + // durationMs 7007 on every run, but 11.6–12.5 s from spawn to exit — about + // 3 s of module loading before the journal opened and 1.7 s of set-up + // before the turn began. That is the 11.06 s CI failure on #54 and the + // 11.3 s on #51/#52: a wall clock that included start-up, not a deadline + // missed. From the prompt on, every wait is a timer. + let promptAt = 0; + const watch = setInterval(() => { + if (promptAt) return; + try { + if (/^prompt$/m.test(readFileSync(agentLog, "utf8"))) promptAt = Date.now(); + } catch { + // Not written yet. + } + }, 20); + const { code, stdout, exitAt } = await new Promise<{ code: number; stdout: string; exitAt: number }>((resolve) => { const child = spawn( process.execPath, [CLI, "run", "write a.txt", "--url", "grok-build", "--model", "grok-4.6", "--for", "5s", "--json", "--yes", "--cwd", dir], @@ -117,13 +135,9 @@ describe("an ACP agent that never answers, under --for 5s", { skip: process.plat ); let out = ""; child.stdout.on("data", (d: Buffer) => (out += d.toString())); - child.on("exit", (c) => resolve({ code: c ?? -1, stdout: out })); + child.on("exit", (c) => resolve({ code: c ?? -1, stdout: out, exitAt: Date.now() })); }); - const took = Date.now() - t0; - - // 5 s of budget, the cancel grace, and a closing summary bounded by its - // 1 s grace — not the hour the hung prompt would have held it. - assert.ok(took < 11_000, `the job took ${took}ms on a 5s budget`); + clearInterval(watch); const end = stdout .split("\n") .filter((l) => l.startsWith("{")) @@ -133,7 +147,18 @@ describe("an ACP agent that never answers, under --for 5s", { skip: process.plat assert.equal(end.deadline, true); assert.equal(end.endedBy, "deadline"); assert.notEqual(end.outcome, "verified"); - assert.ok((end.durationMs ?? Infinity) < 9_000, `job_end durationMs ${end.durationMs}`); + // 5 s of budget, the cancel grace, a closing summary bounded by its grace, + // and the cancel grace again when that is cut — not the hour the hung + // prompt would have held it. 7 s by the constants; the rest is slack for + // timers firing late on a busy machine. + const tail = 2 * CANCEL_GRACE_MS + deadlineGraceMs(5_000); + assert.ok((end.durationMs ?? Infinity) < 5_000 + tail + 2_000, `job_end durationMs ${end.durationMs} on a 5s budget`); + // And nothing holds the process open past the turn: from the prompt to the + // exit is within the same bound. A kill that waited on the hung agent, or + // a cancel awaited without a cap, would show here. + assert.ok(promptAt > 0, "never saw the agent take the prompt"); + const sincePrompt = exitAt - promptAt; + assert.ok(sincePrompt < 5_000 + tail + 2_000, `the job ran ${sincePrompt}ms past the prompt on a 5s budget (durationMs ${end.durationMs})`); assert.notEqual(code, 0); // The agent was asked to cancel, and then it and its child were ended — diff --git a/test/acp-seal-gate.test.ts b/test/acp-seal-gate.test.ts index 19e846e..3f30cf0 100644 --- a/test/acp-seal-gate.test.ts +++ b/test/acp-seal-gate.test.ts @@ -14,9 +14,9 @@ import { join } from "node:path"; import { describe, it } from "node:test"; import { ACP_AGENTS } from "../src/acp.js"; import { Engine } from "../src/engine.js"; -import type { Check } from "../src/types.js"; +import type { Check, EngineEvent } from "../src/types.js"; import { scriptedAcpAgent } from "./acp-agent.js"; -import { allowAll, drain, workspace } from "./helpers.js"; +import { allowAll, workspace } from "./helpers.js"; const GROK = ACP_AGENTS.find((a) => a.name === "grok-build")!; @@ -36,19 +36,32 @@ describe("seal gate on a subprocess backend", () => { }, ]); const engine = new Engine({ baseUrl: GROK.url, model: "grok-4.6", provider: "grok-build", cwd: ws.dir, bar: null, acpSpawn: agent.spawnFn, autonomy: "high" }); + // The drafter finishes when the gate is holding the write, not at a fixed + // 600 ms. On a loaded machine (150 CPU burners, 2026-10-08) the scripted + // agent's first call came 700–1000 ms in, after a 600 ms drafter had + // already finished: nothing was left to gate, the product rightly did + // not wait, and "the wait was announced" failed 15 runs in 20. The + // fallback only bounds a gate that never holds; that still fails below. let fileWhenSealed: boolean | undefined; - const pending = new Promise<{ taskChecks: Check[]; taskNotes: string[] }>((resolve) => - setTimeout(() => { - fileWhenSealed = existsSync(join(ws.dir, "out.txt")); - resolve({ taskChecks: [mk("greeting", "grep -qx hello out.txt")], taskNotes: [] }); - }, 600), - ); - const ev = await drain( - engine.run("write hello to out.txt", allowAll, { - pendingCriteria: pending, - criteriaSoFar: async () => ({ taskChecks: [], taskNotes: [] }), - }), - ); + let gateHeld!: () => void; + let fallback: ReturnType | undefined; + const held = new Promise((r) => { + gateHeld = r; + fallback = setTimeout(r, 10_000); + }); + const pending = held.then(() => { + clearTimeout(fallback); + fileWhenSealed = existsSync(join(ws.dir, "out.txt")); + return { taskChecks: [mk("greeting", "grep -qx hello out.txt")], taskNotes: [] as string[] }; + }); + const ev: EngineEvent[] = []; + for await (const e of engine.run("write hello to out.txt", allowAll, { + pendingCriteria: pending, + criteriaSoFar: async () => ({ taskChecks: [], taskNotes: [] }), + })) { + ev.push(e); + if (e.kind === "info" && /waiting for this task's checks to be sealed/.test(e.text)) gateHeld(); + } assert.equal(fileWhenSealed, false, "the drafter finished before the file was written"); const wait = ev.findIndex((e) => e.kind === "info" && /waiting for this task's checks to be sealed/.test(e.text)); const write = ev.findIndex((e) => e.kind === "tool" && e.name === "write_file"); diff --git a/test/discriminating-checks.test.ts b/test/discriminating-checks.test.ts index a834582..2fdfdd3 100644 --- a/test/discriminating-checks.test.ts +++ b/test/discriminating-checks.test.ts @@ -26,7 +26,7 @@ import { Journal } from "../src/journal.js"; import { Receipts } from "../src/receipts.js"; import { preWorkCopy } from "../src/scratch.js"; import { claimLabel, noteCoverage, tierOf, UNTESTED_CLAIM } from "../src/tiers.js"; -import type { Check, CheckAuthor } from "../src/types.js"; +import type { Check, CheckAuthor, EngineEvent } from "../src/types.js"; import { allowAll, drain, scriptedProvider, workspace, type ScriptedTurn } from "./helpers.js"; const JUDGE: CheckAuthor = { kind: "judge", model: "qwen3-coder-30b-a3b" }; @@ -213,7 +213,6 @@ describe("parseArgs --require-discriminating", () => { }); describe("late checks and the copy taken before the work", () => { - const later = (v: T, ms: number) => new Promise((r) => setTimeout(() => r(v), ms)); const nothing = async () => ({ taskChecks: [] as Check[], taskNotes: [] as string[] }); const mk = (name: string, run: string): Check => ({ name, kind: "command", run, timeoutMs: 5_000, expectExit: 0, tags: ["task", "value", "exact"], hidden: true, author: JUDGE }) as Check; @@ -226,7 +225,26 @@ describe("late checks and the copy taken before the work", () => { requireDiscriminating: true, }); engine.setTurnDeadline(600_000); - const events = await drain(engine.run(TASK, allowAll, { pendingCriteria: later({ taskChecks: checks, taskNotes: [] }, 300), criteriaSoFar: nothing, criteriaWaitMs: 100 })); + // The draft lands once the engine has cut its wait with none ready, so the + // checks really are late. On a 300 ms timer started before the run, a + // start-up slowed by load (150 CPU burners, 2026-10-08) let the draft be + // ready at the 100 ms cut: it was sealed there, rightly, no copy was + // tried, and these cases failed 1 run in 20. The fallback only bounds an + // engine that never cuts. + let cut!: () => void; + let fallback: ReturnType | undefined; + const pendingCriteria = new Promise((r) => { + cut = r; + fallback = setTimeout(r, 10_000); + }).then(() => { + clearTimeout(fallback); + return { taskChecks: checks, taskNotes: [] as string[] }; + }); + const events: EngineEvent[] = []; + for await (const e of engine.run(TASK, allowAll, { pendingCriteria, criteriaSoFar: nothing, criteriaWaitMs: 100 })) { + events.push(e); + if (e.kind === "info" && /none was ready; not sealing an empty set/.test(e.text)) cut(); + } const end = events.find((e) => e.kind === "job_end"); assert.ok(end && end.kind === "job_end"); return { end, tried: Journal.read(journal.path).filter((e) => e.kind === "note" && e.data.kind === "pre-work-try"), copies: Journal.read(journal.path).filter((e) => e.kind === "note" && e.data.kind === "pre-work-copy") }; diff --git a/test/ended-with-verdict.test.ts b/test/ended-with-verdict.test.ts index 7c03de1..5a4c5fb 100644 --- a/test/ended-with-verdict.test.ts +++ b/test/ended-with-verdict.test.ts @@ -7,10 +7,10 @@ import assert from "node:assert/strict"; import { readdirSync, readFileSync } from "node:fs"; import { join } from "node:path"; import { describe, it } from "node:test"; -import { Engine } from "../src/engine.js"; +import { Engine, type RunOptions } from "../src/engine.js"; import { Receipts } from "../src/receipts.js"; import { LatencyLearner } from "../src/watchdog.js"; -import type { Check } from "../src/types.js"; +import type { Check, EngineEvent } from "../src/types.js"; import { allowAll, drain, scriptedProvider, workspace } from "./helpers.js"; const greet = (hidden = true): Check => ({ @@ -31,6 +31,26 @@ const slowWrite = (content: string) => [ { text: "never asked for" }, ]; +/** + * Run the turn with its 400 ms budget armed once out.txt is being written. + * + * Armed at the start, the budget raced the engine's start-up: under 150 CPU + * burners (2026-10-08) it ran out before the scripted reply was read ("time + * budget reached (414ms of 400ms) — no more tool calls"), nothing was + * written, and the bar rightly failed the empty tree; `passedAtEnd` failed + * 14 runs in 20. The budget is read live (`deadlineAt`), so arming it at the + * write makes the clock stop the turn after the work, under any load: during + * the 0.7 s command if the write came early, at once if it came late. + */ +async function stoppedAfterWrite(engine: Engine, opts?: RunOptions) { + const events: EngineEvent[] = []; + for await (const e of engine.run("write out.txt", allowAll, opts)) { + events.push(e); + if (e.kind === "tool" && e.name === "write_file") engine.setTurnDeadline(400); + } + return events; +} + describe("a turn stopped by the clock", () => { // Mercury 2.5: three half-finished trees passed weak drafted checks at the // deadline and were "verified"; the grader failed all three. @@ -40,9 +60,9 @@ describe("a turn stopped by the clock", () => { const p = scriptedProvider(slowWrite("hello\n")); const engine = new Engine({ baseUrl: "http://provider.test/v1", model: "m", cwd: ws.dir, fetchFn: p.fetchFn, bar: null, stream: false, - autonomy: "high", turnDeadlineMs: 400, receipts: new Receipts(ws.dir), + autonomy: "high", receipts: new Receipts(ws.dir), }); - const events = await drain(engine.run("write out.txt", allowAll, { taskChecks: [greet()] })); + const events = await stoppedAfterWrite(engine, { taskChecks: [greet()] }); const e = end(events); assert.equal(e.outcome, "unverified"); assert.equal(e.passedAtEnd, true); @@ -65,10 +85,10 @@ describe("a turn stopped by the clock", () => { const p = scriptedProvider(slowWrite("goodbye\n")); const engine = new Engine({ baseUrl: "http://provider.test/v1", model: "m", cwd: ws.dir, fetchFn: p.fetchFn, bar: null, stream: false, - autonomy: "high", turnDeadlineMs: 400, + autonomy: "high", }); // A person's check, so the refusal is not softened to "unverified". - const events = await drain(engine.run("write out.txt", allowAll, { taskChecks: [greet(false)] })); + const events = await stoppedAfterWrite(engine, { taskChecks: [greet(false)] }); const e = end(events); assert.equal(e.outcome, "not proven"); assert.equal(e.deadline, true); @@ -83,9 +103,9 @@ describe("a turn stopped by the clock", () => { const p = scriptedProvider([...slowWrite("hello\n"), { text: "Wrote out.txt; ran out of time." }]); const engine = new Engine({ baseUrl: "http://provider.test/v1", model: "m", cwd: ws.dir, fetchFn: p.fetchFn, bar: null, stream: false, - autonomy: "high", turnDeadlineMs: 400, + autonomy: "high", }); - const e = end(await drain(engine.run("write out.txt", allowAll))); + const e = end(await stoppedAfterWrite(engine)); assert.equal(e.outcome, "unverified"); assert.equal(e.deadline, true); } finally { diff --git a/test/late-criteria.test.ts b/test/late-criteria.test.ts index 81618f4..a43c340 100644 --- a/test/late-criteria.test.ts +++ b/test/late-criteria.test.ts @@ -34,17 +34,54 @@ function setup(dir: string, id: string) { } const notes = (j: Journal, kind: string) => Journal.read(j.path).filter((e) => e.kind === "note" && e.data.kind === kind); +/** + * A draft that finishes once the engine has cut its wait with none ready. + * + * These cases used to finish the draft on a wall-clock timer (300-400 ms) + * started before `engine.run`, against a 100 ms cut. Under 150 CPU burners + * (2026-10-08) the engine's start-up outran the timer: the journal showed the + * draft sealed 7-38 ms into the turn, ready at the cut, so the product rightly + * sealed it there, and "none was ready" failed 12 runs in 20. Finishing on the + * cut's own announcement puts the draft after the cut under any load. The + * fallback only bounds an engine that never cuts; that still fails. + */ +function draftAfterCut(checks: Check[]) { + let cut!: () => void; + let fallback: ReturnType | undefined; + const pendingCriteria = new Promise((r) => { + cut = r; + fallback = setTimeout(r, 10_000); + }).then(() => { + clearTimeout(fallback); + return { taskChecks: checks, taskNotes: [] as string[] }; + }); + const watch = (e: EngineEvent) => { + if (e.kind === "info" && /none was ready; not sealing an empty set/.test(e.text)) cut(); + }; + return { pendingCriteria, watch }; +} +async function drainWatching(run: AsyncGenerator, watch: (e: EngineEvent) => void): Promise { + const ev: EngineEvent[] = []; + for await (const e of run) { + ev.push(e); + watch(e); + } + return ev; +} + describe("zero checks at the time budget's cut", () => { it("are not sealed empty: the draft joins at the first claim and judges it, the input hash journalled twice", async () => { const ws = workspace(); try { const { engine, journal } = setup(ws.dir, "late-1"); - const ev = await drain( + const draft = draftAfterCut([mk("greeting", "grep -qx hello out.txt")]); + const ev = await drainWatching( engine.run(TASK, allowAll, { - pendingCriteria: later({ taskChecks: [mk("greeting", "grep -qx hello out.txt")], taskNotes: [] }, 400), + pendingCriteria: draft.pendingCriteria, criteriaSoFar: nothing, criteriaWaitMs: 100, }), + draft.watch, ); assert.ok(ev.some((e) => e.kind === "info" && /none was ready; not sealing an empty set/.test(e.text))); assert.ok(ev.some((e) => e.kind === "info" && /joined this turn's checks/.test(e.text))); @@ -67,13 +104,16 @@ describe("zero checks at the time budget's cut", () => { const ws = workspace(); try { const { engine } = setup(ws.dir, "late-2"); - const ev = await drain( + const draft = draftAfterCut([mk("farewell", "grep -qx goodbye out.txt")]); + const ev = await drainWatching( engine.run(TASK, allowAll, { - pendingCriteria: later({ taskChecks: [mk("farewell", "grep -qx goodbye out.txt")], taskNotes: [] }, 300), + pendingCriteria: draft.pendingCriteria, criteriaSoFar: nothing, criteriaWaitMs: 100, }), + draft.watch, ); + assert.ok(ev.some((e) => e.kind === "info" && /joined this turn's checks/.test(e.text)), "the late check joined"); const end = ev.find((e) => e.kind === "job_end"); assert.ok(end && end.kind === "job_end" && end.outcome !== "verified"); assert.ok(ev.some((e) => e.kind === "proof_start")); @@ -89,7 +129,9 @@ describe("zero checks at the time budget's cut", () => { const started = Date.now(); const ev = await drain( engine.run(TASK, allowAll, { - pendingCriteria: later({ taskChecks: [mk("greeting", "grep -q hello out.txt")], taskNotes: [] }, 2_500), + // Never arrives: a 2.5 s timer started before the run could beat a + // start-up slowed by load, and then it was not late at all. + pendingCriteria: new Promise<{ taskChecks: Check[]; taskNotes: string[] }>(() => {}), criteriaSoFar: nothing, criteriaWaitMs: 100, lateCriteriaWaitMs: 50, @@ -133,15 +175,22 @@ describe("a claim waits for late drafts until the time budget's margin", () => { const realTimeout = setTimeout; const slow = (name: string, run: string): Check => ({ ...mk(name, run), timeoutMs: 10_000_000 }); /** Run a turn on a virtual clock: one simulated second per real 2 ms. */ - async function onVirtualClock(start: (at: (v: T, ms: number) => Promise) => AsyncGenerator): Promise<{ ev: EngineEvent[]; simMs: number }> { + async function onVirtualClock( + start: (at: (v: T, ms: number) => Promise) => AsyncGenerator, + ): Promise<{ ev: EngineEvent[]; simMs: number; t0: number; turnStart: number }> { mock.timers.enable({ apis: ["setTimeout", "Date"], now: Date.now() }); const t0 = Date.now(); try { const ev: EngineEvent[] = []; + let turnStart = NaN; let done = false; let err: unknown; const run = (async () => { - for await (const e of start((v, ms) => new Promise((r) => setTimeout(() => r(v), ms)))) ev.push(e); + for await (const e of start((v, ms) => new Promise((r) => setTimeout(() => r(v), ms)))) { + ev.push(e); + // The turn's own start on the simulated clock, which its deadline counts from. + if (e.kind === "job_end") turnStart = Date.now() - e.durationMs; + } })().then( () => { done = true; }, (e: unknown) => { err = e; done = true; }, @@ -152,7 +201,7 @@ describe("a claim waits for late drafts until the time budget's margin", () => { } await run; if (err) throw err; - return { ev, simMs: Date.now() - t0 }; + return { ev, simMs: Date.now() - t0, t0, turnStart }; } finally { mock.timers.reset(); } @@ -179,7 +228,7 @@ describe("a claim waits for late drafts until the time budget's margin", () => { const ws = workspace(); try { const { engine, journal } = setup(ws.dir, "late-wait-1"); - const { ev, simMs } = await onVirtualClock((at) => + const { ev, simMs, t0 } = await onVirtualClock((at) => engine.run(TASK, allowAll, { pendingCriteria: at({ taskChecks: [slow("greeting", "grep -qx hello out.txt")], taskNotes: [] }, 200_000), criteriaSoFar: nothing, @@ -187,7 +236,11 @@ describe("a claim waits for late drafts until the time budget's margin", () => { draftInputs: sealed, }), ); - assert.ok(infos(ev).some((t) => /^waited (19\d|20\d)s for the drafted checks \(waited to the time budget's margin.*1 arrived and joined/.test(t)), infos(ev).join("\n")); + // How long the claim waited depends on when it began, and the simulated + // clock runs on while the engine's real work does: under load the claim + // began 11 s in and "waited 189s" failed a 19x-20x s pattern. What the + // product owes is to be still waiting when the draft lands at 200 s. + assert.ok(infos(ev).some((t) => /^waited \d+s for the drafted checks \(waited to the time budget's margin.*1 arrived and joined/.test(t)), infos(ev).join("\n")); const proof = ev.find((e) => e.kind === "proof_result"); assert.ok(proof && proof.kind === "proof_result" && proof.result.results.some((r: { name: string }) => r.name === "task:greeting")); assert.equal(outcome(ev), "verified"); @@ -195,7 +248,10 @@ describe("a claim waits for late drafts until the time budget's margin", () => { const join = notes(journal, "late-checks"); assert.equal(join.length, 1); assert.equal(join[0]!.data.arrived, true); - assert.ok(Number(join[0]!.data.waitedMs) >= 195_000); + // Joined once it landed (the note follows the join's own pre-work try, + // real work on a simulated clock), and well before the 510 s margin. + const joinedAt = Date.parse(String(join[0]!.iso)) - t0; + assert.ok(joinedAt >= 200_000 && joinedAt < 510_000, `joined ${joinedAt} ms in, for a draft that landed at 200 s`); assert.equal(notes(journal, "draft-inputs")[0]!.data.inputsSha, "frozen-inputs"); assert.equal(join[0]!.data.inputsSha, "frozen-inputs"); } finally { @@ -227,7 +283,7 @@ describe("a claim waits for late drafts until the time budget's margin", () => { const ws = workspace(); try { const { engine, journal } = setup(ws.dir, "late-wait-3"); - const { ev, simMs } = await onVirtualClock(() => + const { ev, simMs, turnStart } = await onVirtualClock(() => engine.run(TASK, allowAll, { pendingCriteria: new Promise(() => {}), criteriaSoFar: nothing, @@ -237,9 +293,12 @@ describe("a claim waits for late drafts until the time budget's margin", () => { ); const said = infos(ev).find((t) => /did not arrive/.test(t)); assert.ok(said, infos(ev).join("\n")); - // 600 s budget, 90 s margin: about 510 s waited, then judged in what is left. - const waited = Number(/^waited (\d+)s/.exec(said!)?.[1]); - assert.ok(waited >= 500 && waited <= 512, said); + // 600 s budget, 90 s margin: the wait ends 510 s after the turn began, + // then the claim is judged in what is left. Counted from the turn's + // start, not from the claim, which load can push tens of seconds in. + const gaveUp = notes(journal, "late-checks")[0]!; + const endedAt = Date.parse(String(gaveUp.iso)) - turnStart; + assert.ok(endedAt >= 510_000 && endedAt <= 513_000, `${said} (gave up ${endedAt} ms into the turn)`); assert.match(said!, /judged without them/); assert.notEqual(outcome(ev), "verified"); assert.ok(ev.some((e) => e.kind === "job_end"));