From 3534fc23463209f5370a27e88bdc7eadd32045e1 Mon Sep 17 00:00:00 2001 From: Tyler Skelton Date: Thu, 8 Oct 2026 05:42:33 -0700 Subject: [PATCH 1/5] Time the ACP deadline test from the prompt, not from the CLI spawn "ends the job on time" failed on macOS CI for #54 (11055ms) and on the box for #51/#52 (11.3s) when suites ran in parallel. The product was not late: under 150 CPU burners on a 10-core Mac, job_end's durationMs was 7007ms on every run (5s budget + 500ms cancel grace + 1s salvage grace + 500ms cancel grace), while spawn-to-exit was 11.2-12.3s. The extra ~5s was CPU-starved start-up outside the turn's clock: ~3s of node module loading before the journal opened and ~1.7s of set-up before the turn began (120ms unloaded). The test measured the whole process lifetime against an 11s bound. It now checks the turn's own clock (durationMs) and the time from the agent taking the prompt to the process exiting, both against 5s + the documented graces (2 x CANCEL_GRACE_MS + deadlineGraceMs) + 2s slack. A 3s stall injected into the interrupt path still fails it (durationMs 13021). Under 150 CPU burners: 0/20 passes before, 20/20 after. --- test/acp-deadline.test.ts | 45 ++++++++++++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 10 deletions(-) diff --git a/test/acp-deadline.test.ts b/test/acp-deadline.test.ts index e2581a7..215d37f 100644 --- a/test/acp-deadline.test.ts +++ b/test/acp-deadline.test.ts @@ -13,7 +13,7 @@ import { execFileSync, spawn } from "node:child_process"; import { chmodSync, existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { after, describe, it } from "node:test"; -import { ACP_AGENTS, backendStallMs, BACKEND_STALL_MS } from "../src/acp.js"; +import { ACP_AGENTS, backendStallMs, BACKEND_STALL_MS, CANCEL_GRACE_MS } from "../src/acp.js"; import { askModel } from "../src/ask.js"; import { deadlineGraceMs, Engine } from "../src/engine.js"; import { Journal } from "../src/journal.js"; @@ -105,8 +105,26 @@ describe("an ACP agent that never answers, under --for 5s", { skip: process.plat chmodSync(join(bin, "grok"), 0o755); const agentLog = join(root, "agent.log"); - const t0 = Date.now(); - const { code, stdout } = await new Promise<{ code: number; stdout: string }>((resolve) => { + // The clock is started when the agent has the prompt, not when the CLI is + // spawned. `--for` is a ceiling on the turn (the turn's own clock is + // job_end's durationMs, checked below); the node start-up and set-up + // before the turn are outside it, and on a loaded machine they are not + // small. Measured 2026-10-08 with 150 CPU burners on a 10-core Mac: + // durationMs 7007 on every run, but 11.6–12.5 s from spawn to exit — about + // 3 s of module loading before the journal opened and 1.7 s of set-up + // before the turn began. That is the 11.06 s CI failure on #54 and the + // 11.3 s on #51/#52: a wall clock that included start-up, not a deadline + // missed. From the prompt on, every wait is a timer. + let promptAt = 0; + const watch = setInterval(() => { + if (promptAt) return; + try { + if (/^prompt$/m.test(readFileSync(agentLog, "utf8"))) promptAt = Date.now(); + } catch { + // Not written yet. + } + }, 20); + const { code, stdout, exitAt } = await new Promise<{ code: number; stdout: string; exitAt: number }>((resolve) => { const child = spawn( process.execPath, [CLI, "run", "write a.txt", "--url", "grok-build", "--model", "grok-4.6", "--for", "5s", "--json", "--yes", "--cwd", dir], @@ -117,13 +135,9 @@ describe("an ACP agent that never answers, under --for 5s", { skip: process.plat ); let out = ""; child.stdout.on("data", (d: Buffer) => (out += d.toString())); - child.on("exit", (c) => resolve({ code: c ?? -1, stdout: out })); + child.on("exit", (c) => resolve({ code: c ?? -1, stdout: out, exitAt: Date.now() })); }); - const took = Date.now() - t0; - - // 5 s of budget, the cancel grace, and a closing summary bounded by its - // 1 s grace — not the hour the hung prompt would have held it. - assert.ok(took < 11_000, `the job took ${took}ms on a 5s budget`); + clearInterval(watch); const end = stdout .split("\n") .filter((l) => l.startsWith("{")) @@ -133,7 +147,18 @@ describe("an ACP agent that never answers, under --for 5s", { skip: process.plat assert.equal(end.deadline, true); assert.equal(end.endedBy, "deadline"); assert.notEqual(end.outcome, "verified"); - assert.ok((end.durationMs ?? Infinity) < 9_000, `job_end durationMs ${end.durationMs}`); + // 5 s of budget, the cancel grace, a closing summary bounded by its grace, + // and the cancel grace again when that is cut — not the hour the hung + // prompt would have held it. 7 s by the constants; the rest is slack for + // timers firing late on a busy machine. + const tail = 2 * CANCEL_GRACE_MS + deadlineGraceMs(5_000); + assert.ok((end.durationMs ?? Infinity) < 5_000 + tail + 2_000, `job_end durationMs ${end.durationMs} on a 5s budget`); + // And nothing holds the process open past the turn: from the prompt to the + // exit is within the same bound. A kill that waited on the hung agent, or + // a cancel awaited without a cap, would show here. + assert.ok(promptAt > 0, "never saw the agent take the prompt"); + const sincePrompt = exitAt - promptAt; + assert.ok(sincePrompt < 5_000 + tail + 2_000, `the job ran ${sincePrompt}ms past the prompt on a 5s budget (durationMs ${end.durationMs})`); assert.notEqual(code, 0); // The agent was asked to cancel, and then it and its child were ended — From 0234434a313f1a3d8274d6594e81872584b1872f Mon Sep 17 00:00:00 2001 From: Tyler Skelton Date: Thu, 8 Oct 2026 06:15:56 -0700 Subject: [PATCH 2/5] Release the seal-gate test's drafter when the gate holds, not at 600ms "holds the first write until the checks are sealed" failed 15 of 20 runs under 150 CPU burners with "the wait was announced". Traced with a journal: the scripted ACP agent's first call came 500-860ms into the run, after the 600ms drafter timer had fired, so the checks were sealed before any write and the gate correctly had nothing to hold. The product was right; the stimulus was on a wall clock that ran during start-up. The drafter now finishes when the engine announces it is holding the write (a 10s fallback only bounds a gate that never holds). With the per-call gate switched off in the product the test still fails, in 100ms: "the drafter finished before the file was written". --- test/acp-seal-gate.test.ts | 41 +++++++++++++++++++++++++------------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/test/acp-seal-gate.test.ts b/test/acp-seal-gate.test.ts index 19e846e..3f30cf0 100644 --- a/test/acp-seal-gate.test.ts +++ b/test/acp-seal-gate.test.ts @@ -14,9 +14,9 @@ import { join } from "node:path"; import { describe, it } from "node:test"; import { ACP_AGENTS } from "../src/acp.js"; import { Engine } from "../src/engine.js"; -import type { Check } from "../src/types.js"; +import type { Check, EngineEvent } from "../src/types.js"; import { scriptedAcpAgent } from "./acp-agent.js"; -import { allowAll, drain, workspace } from "./helpers.js"; +import { allowAll, workspace } from "./helpers.js"; const GROK = ACP_AGENTS.find((a) => a.name === "grok-build")!; @@ -36,19 +36,32 @@ describe("seal gate on a subprocess backend", () => { }, ]); const engine = new Engine({ baseUrl: GROK.url, model: "grok-4.6", provider: "grok-build", cwd: ws.dir, bar: null, acpSpawn: agent.spawnFn, autonomy: "high" }); + // The drafter finishes when the gate is holding the write, not at a fixed + // 600 ms. On a loaded machine (150 CPU burners, 2026-10-08) the scripted + // agent's first call came 700–1000 ms in, after a 600 ms drafter had + // already finished: nothing was left to gate, the product rightly did + // not wait, and "the wait was announced" failed 15 runs in 20. The + // fallback only bounds a gate that never holds; that still fails below. let fileWhenSealed: boolean | undefined; - const pending = new Promise<{ taskChecks: Check[]; taskNotes: string[] }>((resolve) => - setTimeout(() => { - fileWhenSealed = existsSync(join(ws.dir, "out.txt")); - resolve({ taskChecks: [mk("greeting", "grep -qx hello out.txt")], taskNotes: [] }); - }, 600), - ); - const ev = await drain( - engine.run("write hello to out.txt", allowAll, { - pendingCriteria: pending, - criteriaSoFar: async () => ({ taskChecks: [], taskNotes: [] }), - }), - ); + let gateHeld!: () => void; + let fallback: ReturnType | undefined; + const held = new Promise((r) => { + gateHeld = r; + fallback = setTimeout(r, 10_000); + }); + const pending = held.then(() => { + clearTimeout(fallback); + fileWhenSealed = existsSync(join(ws.dir, "out.txt")); + return { taskChecks: [mk("greeting", "grep -qx hello out.txt")], taskNotes: [] as string[] }; + }); + const ev: EngineEvent[] = []; + for await (const e of engine.run("write hello to out.txt", allowAll, { + pendingCriteria: pending, + criteriaSoFar: async () => ({ taskChecks: [], taskNotes: [] }), + })) { + ev.push(e); + if (e.kind === "info" && /waiting for this task's checks to be sealed/.test(e.text)) gateHeld(); + } assert.equal(fileWhenSealed, false, "the drafter finished before the file was written"); const wait = ev.findIndex((e) => e.kind === "info" && /waiting for this task's checks to be sealed/.test(e.text)); const write = ev.findIndex((e) => e.kind === "tool" && e.name === "write_file"); From a9eab8458fa98eaa5f4ac3598df5e73f729535b2 Mon Sep 17 00:00:00 2001 From: Tyler Skelton Date: Thu, 8 Oct 2026 06:17:09 -0700 Subject: [PATCH 3/5] Arm the clock-stop tests' 400ms budget at the write, not at turn start "runs the sealed bar on the tree ... the model never said done" failed 14 of 20 runs under 150 CPU burners (passedAtEnd undefined). In every failing run the 400ms budget ran out before the scripted provider's first reply was read: "time budget reached (414ms of 400ms) - no more tool calls". Nothing was written, so the bar rightly failed the empty tree. The deadline itself was honoured (seen at 414ms, job_end 526-777ms with the bar). The three "a turn stopped by the clock" cases now arm the same 400ms budget when write_file is seen (the budget is read live), so the clock stops the turn after the work exists under any load. With the product's between-steps deadline stop removed, all three fail. --- test/ended-with-verdict.test.ts | 36 +++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/test/ended-with-verdict.test.ts b/test/ended-with-verdict.test.ts index 7c03de1..5a4c5fb 100644 --- a/test/ended-with-verdict.test.ts +++ b/test/ended-with-verdict.test.ts @@ -7,10 +7,10 @@ import assert from "node:assert/strict"; import { readdirSync, readFileSync } from "node:fs"; import { join } from "node:path"; import { describe, it } from "node:test"; -import { Engine } from "../src/engine.js"; +import { Engine, type RunOptions } from "../src/engine.js"; import { Receipts } from "../src/receipts.js"; import { LatencyLearner } from "../src/watchdog.js"; -import type { Check } from "../src/types.js"; +import type { Check, EngineEvent } from "../src/types.js"; import { allowAll, drain, scriptedProvider, workspace } from "./helpers.js"; const greet = (hidden = true): Check => ({ @@ -31,6 +31,26 @@ const slowWrite = (content: string) => [ { text: "never asked for" }, ]; +/** + * Run the turn with its 400 ms budget armed once out.txt is being written. + * + * Armed at the start, the budget raced the engine's start-up: under 150 CPU + * burners (2026-10-08) it ran out before the scripted reply was read ("time + * budget reached (414ms of 400ms) — no more tool calls"), nothing was + * written, and the bar rightly failed the empty tree; `passedAtEnd` failed + * 14 runs in 20. The budget is read live (`deadlineAt`), so arming it at the + * write makes the clock stop the turn after the work, under any load: during + * the 0.7 s command if the write came early, at once if it came late. + */ +async function stoppedAfterWrite(engine: Engine, opts?: RunOptions) { + const events: EngineEvent[] = []; + for await (const e of engine.run("write out.txt", allowAll, opts)) { + events.push(e); + if (e.kind === "tool" && e.name === "write_file") engine.setTurnDeadline(400); + } + return events; +} + describe("a turn stopped by the clock", () => { // Mercury 2.5: three half-finished trees passed weak drafted checks at the // deadline and were "verified"; the grader failed all three. @@ -40,9 +60,9 @@ describe("a turn stopped by the clock", () => { const p = scriptedProvider(slowWrite("hello\n")); const engine = new Engine({ baseUrl: "http://provider.test/v1", model: "m", cwd: ws.dir, fetchFn: p.fetchFn, bar: null, stream: false, - autonomy: "high", turnDeadlineMs: 400, receipts: new Receipts(ws.dir), + autonomy: "high", receipts: new Receipts(ws.dir), }); - const events = await drain(engine.run("write out.txt", allowAll, { taskChecks: [greet()] })); + const events = await stoppedAfterWrite(engine, { taskChecks: [greet()] }); const e = end(events); assert.equal(e.outcome, "unverified"); assert.equal(e.passedAtEnd, true); @@ -65,10 +85,10 @@ describe("a turn stopped by the clock", () => { const p = scriptedProvider(slowWrite("goodbye\n")); const engine = new Engine({ baseUrl: "http://provider.test/v1", model: "m", cwd: ws.dir, fetchFn: p.fetchFn, bar: null, stream: false, - autonomy: "high", turnDeadlineMs: 400, + autonomy: "high", }); // A person's check, so the refusal is not softened to "unverified". - const events = await drain(engine.run("write out.txt", allowAll, { taskChecks: [greet(false)] })); + const events = await stoppedAfterWrite(engine, { taskChecks: [greet(false)] }); const e = end(events); assert.equal(e.outcome, "not proven"); assert.equal(e.deadline, true); @@ -83,9 +103,9 @@ describe("a turn stopped by the clock", () => { const p = scriptedProvider([...slowWrite("hello\n"), { text: "Wrote out.txt; ran out of time." }]); const engine = new Engine({ baseUrl: "http://provider.test/v1", model: "m", cwd: ws.dir, fetchFn: p.fetchFn, bar: null, stream: false, - autonomy: "high", turnDeadlineMs: 400, + autonomy: "high", }); - const e = end(await drain(engine.run("write out.txt", allowAll))); + const e = end(await stoppedAfterWrite(engine)); assert.equal(e.outcome, "unverified"); assert.equal(e.deadline, true); } finally { From 029493610597f39df595470f0ea591bc68470c8b Mon Sep 17 00:00:00 2001 From: Tyler Skelton Date: Thu, 8 Oct 2026 06:18:47 -0700 Subject: [PATCH 4/5] Time late-criteria's drafts by the engine's phases, not by start-up Under 150 CPU burners, "are not sealed empty" failed 12 of 20 runs and "drafts arriving at 200 s" 1 of 20. Both were the product on time and the test's clock in the wrong place: - The draft finished on a 300-400ms timer started before engine.run, against a 100ms cut. The journal showed the draft sealed 7-38ms into the turn: start-up had outrun the timer, the draft was ready at the cut, and sealing it there is right. The draft now finishes on the cut's own "none was ready" announcement, and the never-late case uses a draft that never arrives. - On the virtual clock (1 simulated second per 2ms real), the claim began 11 simulated seconds in under load, so it waited 189s for a draft that landed at 200s, which is correct, and a 19x-20x pattern failed. The test now checks that the draft joined after it landed and before the 510s margin, and that a draft that never arrives is given up at 510s from the turn's start (from job_end's durationMs), not a wait counted from wherever the claim began. With the late join switched off, the first two fail; with the claim held to the old 120s rule, both virtual-clock cases fail. --- test/late-criteria.test.ts | 89 +++++++++++++++++++++++++++++++------- 1 file changed, 74 insertions(+), 15 deletions(-) diff --git a/test/late-criteria.test.ts b/test/late-criteria.test.ts index 81618f4..a43c340 100644 --- a/test/late-criteria.test.ts +++ b/test/late-criteria.test.ts @@ -34,17 +34,54 @@ function setup(dir: string, id: string) { } const notes = (j: Journal, kind: string) => Journal.read(j.path).filter((e) => e.kind === "note" && e.data.kind === kind); +/** + * A draft that finishes once the engine has cut its wait with none ready. + * + * These cases used to finish the draft on a wall-clock timer (300-400 ms) + * started before `engine.run`, against a 100 ms cut. Under 150 CPU burners + * (2026-10-08) the engine's start-up outran the timer: the journal showed the + * draft sealed 7-38 ms into the turn, ready at the cut, so the product rightly + * sealed it there, and "none was ready" failed 12 runs in 20. Finishing on the + * cut's own announcement puts the draft after the cut under any load. The + * fallback only bounds an engine that never cuts; that still fails. + */ +function draftAfterCut(checks: Check[]) { + let cut!: () => void; + let fallback: ReturnType | undefined; + const pendingCriteria = new Promise((r) => { + cut = r; + fallback = setTimeout(r, 10_000); + }).then(() => { + clearTimeout(fallback); + return { taskChecks: checks, taskNotes: [] as string[] }; + }); + const watch = (e: EngineEvent) => { + if (e.kind === "info" && /none was ready; not sealing an empty set/.test(e.text)) cut(); + }; + return { pendingCriteria, watch }; +} +async function drainWatching(run: AsyncGenerator, watch: (e: EngineEvent) => void): Promise { + const ev: EngineEvent[] = []; + for await (const e of run) { + ev.push(e); + watch(e); + } + return ev; +} + describe("zero checks at the time budget's cut", () => { it("are not sealed empty: the draft joins at the first claim and judges it, the input hash journalled twice", async () => { const ws = workspace(); try { const { engine, journal } = setup(ws.dir, "late-1"); - const ev = await drain( + const draft = draftAfterCut([mk("greeting", "grep -qx hello out.txt")]); + const ev = await drainWatching( engine.run(TASK, allowAll, { - pendingCriteria: later({ taskChecks: [mk("greeting", "grep -qx hello out.txt")], taskNotes: [] }, 400), + pendingCriteria: draft.pendingCriteria, criteriaSoFar: nothing, criteriaWaitMs: 100, }), + draft.watch, ); assert.ok(ev.some((e) => e.kind === "info" && /none was ready; not sealing an empty set/.test(e.text))); assert.ok(ev.some((e) => e.kind === "info" && /joined this turn's checks/.test(e.text))); @@ -67,13 +104,16 @@ describe("zero checks at the time budget's cut", () => { const ws = workspace(); try { const { engine } = setup(ws.dir, "late-2"); - const ev = await drain( + const draft = draftAfterCut([mk("farewell", "grep -qx goodbye out.txt")]); + const ev = await drainWatching( engine.run(TASK, allowAll, { - pendingCriteria: later({ taskChecks: [mk("farewell", "grep -qx goodbye out.txt")], taskNotes: [] }, 300), + pendingCriteria: draft.pendingCriteria, criteriaSoFar: nothing, criteriaWaitMs: 100, }), + draft.watch, ); + assert.ok(ev.some((e) => e.kind === "info" && /joined this turn's checks/.test(e.text)), "the late check joined"); const end = ev.find((e) => e.kind === "job_end"); assert.ok(end && end.kind === "job_end" && end.outcome !== "verified"); assert.ok(ev.some((e) => e.kind === "proof_start")); @@ -89,7 +129,9 @@ describe("zero checks at the time budget's cut", () => { const started = Date.now(); const ev = await drain( engine.run(TASK, allowAll, { - pendingCriteria: later({ taskChecks: [mk("greeting", "grep -q hello out.txt")], taskNotes: [] }, 2_500), + // Never arrives: a 2.5 s timer started before the run could beat a + // start-up slowed by load, and then it was not late at all. + pendingCriteria: new Promise<{ taskChecks: Check[]; taskNotes: string[] }>(() => {}), criteriaSoFar: nothing, criteriaWaitMs: 100, lateCriteriaWaitMs: 50, @@ -133,15 +175,22 @@ describe("a claim waits for late drafts until the time budget's margin", () => { const realTimeout = setTimeout; const slow = (name: string, run: string): Check => ({ ...mk(name, run), timeoutMs: 10_000_000 }); /** Run a turn on a virtual clock: one simulated second per real 2 ms. */ - async function onVirtualClock(start: (at: (v: T, ms: number) => Promise) => AsyncGenerator): Promise<{ ev: EngineEvent[]; simMs: number }> { + async function onVirtualClock( + start: (at: (v: T, ms: number) => Promise) => AsyncGenerator, + ): Promise<{ ev: EngineEvent[]; simMs: number; t0: number; turnStart: number }> { mock.timers.enable({ apis: ["setTimeout", "Date"], now: Date.now() }); const t0 = Date.now(); try { const ev: EngineEvent[] = []; + let turnStart = NaN; let done = false; let err: unknown; const run = (async () => { - for await (const e of start((v, ms) => new Promise((r) => setTimeout(() => r(v), ms)))) ev.push(e); + for await (const e of start((v, ms) => new Promise((r) => setTimeout(() => r(v), ms)))) { + ev.push(e); + // The turn's own start on the simulated clock, which its deadline counts from. + if (e.kind === "job_end") turnStart = Date.now() - e.durationMs; + } })().then( () => { done = true; }, (e: unknown) => { err = e; done = true; }, @@ -152,7 +201,7 @@ describe("a claim waits for late drafts until the time budget's margin", () => { } await run; if (err) throw err; - return { ev, simMs: Date.now() - t0 }; + return { ev, simMs: Date.now() - t0, t0, turnStart }; } finally { mock.timers.reset(); } @@ -179,7 +228,7 @@ describe("a claim waits for late drafts until the time budget's margin", () => { const ws = workspace(); try { const { engine, journal } = setup(ws.dir, "late-wait-1"); - const { ev, simMs } = await onVirtualClock((at) => + const { ev, simMs, t0 } = await onVirtualClock((at) => engine.run(TASK, allowAll, { pendingCriteria: at({ taskChecks: [slow("greeting", "grep -qx hello out.txt")], taskNotes: [] }, 200_000), criteriaSoFar: nothing, @@ -187,7 +236,11 @@ describe("a claim waits for late drafts until the time budget's margin", () => { draftInputs: sealed, }), ); - assert.ok(infos(ev).some((t) => /^waited (19\d|20\d)s for the drafted checks \(waited to the time budget's margin.*1 arrived and joined/.test(t)), infos(ev).join("\n")); + // How long the claim waited depends on when it began, and the simulated + // clock runs on while the engine's real work does: under load the claim + // began 11 s in and "waited 189s" failed a 19x-20x s pattern. What the + // product owes is to be still waiting when the draft lands at 200 s. + assert.ok(infos(ev).some((t) => /^waited \d+s for the drafted checks \(waited to the time budget's margin.*1 arrived and joined/.test(t)), infos(ev).join("\n")); const proof = ev.find((e) => e.kind === "proof_result"); assert.ok(proof && proof.kind === "proof_result" && proof.result.results.some((r: { name: string }) => r.name === "task:greeting")); assert.equal(outcome(ev), "verified"); @@ -195,7 +248,10 @@ describe("a claim waits for late drafts until the time budget's margin", () => { const join = notes(journal, "late-checks"); assert.equal(join.length, 1); assert.equal(join[0]!.data.arrived, true); - assert.ok(Number(join[0]!.data.waitedMs) >= 195_000); + // Joined once it landed (the note follows the join's own pre-work try, + // real work on a simulated clock), and well before the 510 s margin. + const joinedAt = Date.parse(String(join[0]!.iso)) - t0; + assert.ok(joinedAt >= 200_000 && joinedAt < 510_000, `joined ${joinedAt} ms in, for a draft that landed at 200 s`); assert.equal(notes(journal, "draft-inputs")[0]!.data.inputsSha, "frozen-inputs"); assert.equal(join[0]!.data.inputsSha, "frozen-inputs"); } finally { @@ -227,7 +283,7 @@ describe("a claim waits for late drafts until the time budget's margin", () => { const ws = workspace(); try { const { engine, journal } = setup(ws.dir, "late-wait-3"); - const { ev, simMs } = await onVirtualClock(() => + const { ev, simMs, turnStart } = await onVirtualClock(() => engine.run(TASK, allowAll, { pendingCriteria: new Promise(() => {}), criteriaSoFar: nothing, @@ -237,9 +293,12 @@ describe("a claim waits for late drafts until the time budget's margin", () => { ); const said = infos(ev).find((t) => /did not arrive/.test(t)); assert.ok(said, infos(ev).join("\n")); - // 600 s budget, 90 s margin: about 510 s waited, then judged in what is left. - const waited = Number(/^waited (\d+)s/.exec(said!)?.[1]); - assert.ok(waited >= 500 && waited <= 512, said); + // 600 s budget, 90 s margin: the wait ends 510 s after the turn began, + // then the claim is judged in what is left. Counted from the turn's + // start, not from the claim, which load can push tens of seconds in. + const gaveUp = notes(journal, "late-checks")[0]!; + const endedAt = Date.parse(String(gaveUp.iso)) - turnStart; + assert.ok(endedAt >= 510_000 && endedAt <= 513_000, `${said} (gave up ${endedAt} ms into the turn)`); assert.match(said!, /judged without them/); assert.notEqual(outcome(ev), "verified"); assert.ok(ev.some((e) => e.kind === "job_end")); From fd056e7aa3cef6caab4588f919602657a34bd9bd Mon Sep 17 00:00:00 2001 From: Tyler Skelton Date: Thu, 8 Oct 2026 06:19:10 -0700 Subject: [PATCH 5/5] Land discriminating-checks' late drafts after the cut, not on a timer The three "late checks and the copy taken before the work" cases finished their draft on a 300ms timer started before engine.run, against a 100ms cut. Under 150 CPU burners the journals of 4 of 24 runs showed the draft sealed at the cut (62-102ms into the turn): start-up had outrun the timer, the checks were not late, no pre-work copy was tried, and "the change to the copy is journalled" / "task:abs untried" failed (1 of 20 runs each). Sealing a ready draft at the cut is right. The draft now lands on the cut's own "none was ready" announcement. With the product's late join switched off, all three cases fail. --- test/discriminating-checks.test.ts | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/test/discriminating-checks.test.ts b/test/discriminating-checks.test.ts index a834582..2fdfdd3 100644 --- a/test/discriminating-checks.test.ts +++ b/test/discriminating-checks.test.ts @@ -26,7 +26,7 @@ import { Journal } from "../src/journal.js"; import { Receipts } from "../src/receipts.js"; import { preWorkCopy } from "../src/scratch.js"; import { claimLabel, noteCoverage, tierOf, UNTESTED_CLAIM } from "../src/tiers.js"; -import type { Check, CheckAuthor } from "../src/types.js"; +import type { Check, CheckAuthor, EngineEvent } from "../src/types.js"; import { allowAll, drain, scriptedProvider, workspace, type ScriptedTurn } from "./helpers.js"; const JUDGE: CheckAuthor = { kind: "judge", model: "qwen3-coder-30b-a3b" }; @@ -213,7 +213,6 @@ describe("parseArgs --require-discriminating", () => { }); describe("late checks and the copy taken before the work", () => { - const later = (v: T, ms: number) => new Promise((r) => setTimeout(() => r(v), ms)); const nothing = async () => ({ taskChecks: [] as Check[], taskNotes: [] as string[] }); const mk = (name: string, run: string): Check => ({ name, kind: "command", run, timeoutMs: 5_000, expectExit: 0, tags: ["task", "value", "exact"], hidden: true, author: JUDGE }) as Check; @@ -226,7 +225,26 @@ describe("late checks and the copy taken before the work", () => { requireDiscriminating: true, }); engine.setTurnDeadline(600_000); - const events = await drain(engine.run(TASK, allowAll, { pendingCriteria: later({ taskChecks: checks, taskNotes: [] }, 300), criteriaSoFar: nothing, criteriaWaitMs: 100 })); + // The draft lands once the engine has cut its wait with none ready, so the + // checks really are late. On a 300 ms timer started before the run, a + // start-up slowed by load (150 CPU burners, 2026-10-08) let the draft be + // ready at the 100 ms cut: it was sealed there, rightly, no copy was + // tried, and these cases failed 1 run in 20. The fallback only bounds an + // engine that never cuts. + let cut!: () => void; + let fallback: ReturnType | undefined; + const pendingCriteria = new Promise((r) => { + cut = r; + fallback = setTimeout(r, 10_000); + }).then(() => { + clearTimeout(fallback); + return { taskChecks: checks, taskNotes: [] as string[] }; + }); + const events: EngineEvent[] = []; + for await (const e of engine.run(TASK, allowAll, { pendingCriteria, criteriaSoFar: nothing, criteriaWaitMs: 100 })) { + events.push(e); + if (e.kind === "info" && /none was ready; not sealing an empty set/.test(e.text)) cut(); + } const end = events.find((e) => e.kind === "job_end"); assert.ok(end && end.kind === "job_end"); return { end, tried: Journal.read(journal.path).filter((e) => e.kind === "note" && e.data.kind === "pre-work-try"), copies: Journal.read(journal.path).filter((e) => e.kind === "note" && e.data.kind === "pre-work-copy") };