Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 35 additions & 10 deletions test/acp-deadline.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ import { execFileSync, spawn } from "node:child_process";
import { chmodSync, existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import { after, describe, it } from "node:test";
import { ACP_AGENTS, backendStallMs, BACKEND_STALL_MS } from "../src/acp.js";
import { ACP_AGENTS, backendStallMs, BACKEND_STALL_MS, CANCEL_GRACE_MS } from "../src/acp.js";
import { askModel } from "../src/ask.js";
import { deadlineGraceMs, Engine } from "../src/engine.js";
import { Journal } from "../src/journal.js";
Expand Down Expand Up @@ -105,8 +105,26 @@ describe("an ACP agent that never answers, under --for 5s", { skip: process.plat
chmodSync(join(bin, "grok"), 0o755);
const agentLog = join(root, "agent.log");

const t0 = Date.now();
const { code, stdout } = await new Promise<{ code: number; stdout: string }>((resolve) => {
// The clock is started when the agent has the prompt, not when the CLI is
// spawned. `--for` is a ceiling on the turn (the turn's own clock is
// job_end's durationMs, checked below); the node start-up and set-up
// before the turn are outside it, and on a loaded machine they are not
// small. Measured 2026-10-08 with 150 CPU burners on a 10-core Mac:
// durationMs 7007 on every run, but 11.6–12.5 s from spawn to exit — about
// 3 s of module loading before the journal opened and 1.7 s of set-up
// before the turn began. That is the 11.06 s CI failure on #54 and the
// 11.3 s on #51/#52: a wall clock that included start-up, not a deadline
// missed. From the prompt on, every wait is a timer.
let promptAt = 0;
const watch = setInterval(() => {
if (promptAt) return;
try {
if (/^prompt$/m.test(readFileSync(agentLog, "utf8"))) promptAt = Date.now();
} catch {
// Not written yet.
}
}, 20);
const { code, stdout, exitAt } = await new Promise<{ code: number; stdout: string; exitAt: number }>((resolve) => {
const child = spawn(
process.execPath,
[CLI, "run", "write a.txt", "--url", "grok-build", "--model", "grok-4.6", "--for", "5s", "--json", "--yes", "--cwd", dir],
Expand All @@ -117,13 +135,9 @@ describe("an ACP agent that never answers, under --for 5s", { skip: process.plat
);
let out = "";
child.stdout.on("data", (d: Buffer) => (out += d.toString()));
child.on("exit", (c) => resolve({ code: c ?? -1, stdout: out }));
child.on("exit", (c) => resolve({ code: c ?? -1, stdout: out, exitAt: Date.now() }));
});
const took = Date.now() - t0;

// 5 s of budget, the cancel grace, and a closing summary bounded by its
// 1 s grace — not the hour the hung prompt would have held it.
assert.ok(took < 11_000, `the job took ${took}ms on a 5s budget`);
clearInterval(watch);
const end = stdout
.split("\n")
.filter((l) => l.startsWith("{"))
Expand All @@ -133,7 +147,18 @@ describe("an ACP agent that never answers, under --for 5s", { skip: process.plat
assert.equal(end.deadline, true);
assert.equal(end.endedBy, "deadline");
assert.notEqual(end.outcome, "verified");
assert.ok((end.durationMs ?? Infinity) < 9_000, `job_end durationMs ${end.durationMs}`);
// 5 s of budget, the cancel grace, a closing summary bounded by its grace,
// and the cancel grace again when that is cut — not the hour the hung
// prompt would have held it. 7 s by the constants; the rest is slack for
// timers firing late on a busy machine.
const tail = 2 * CANCEL_GRACE_MS + deadlineGraceMs(5_000);
assert.ok((end.durationMs ?? Infinity) < 5_000 + tail + 2_000, `job_end durationMs ${end.durationMs} on a 5s budget`);
// And nothing holds the process open past the turn: from the prompt to the
// exit is within the same bound. A kill that waited on the hung agent, or
// a cancel awaited without a cap, would show here.
assert.ok(promptAt > 0, "never saw the agent take the prompt");
const sincePrompt = exitAt - promptAt;
assert.ok(sincePrompt < 5_000 + tail + 2_000, `the job ran ${sincePrompt}ms past the prompt on a 5s budget (durationMs ${end.durationMs})`);
assert.notEqual(code, 0);

// The agent was asked to cancel, and then it and its child were ended —
Expand Down
41 changes: 27 additions & 14 deletions test/acp-seal-gate.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,9 @@ import { join } from "node:path";
import { describe, it } from "node:test";
import { ACP_AGENTS } from "../src/acp.js";
import { Engine } from "../src/engine.js";
import type { Check } from "../src/types.js";
import type { Check, EngineEvent } from "../src/types.js";
import { scriptedAcpAgent } from "./acp-agent.js";
import { allowAll, drain, workspace } from "./helpers.js";
import { allowAll, workspace } from "./helpers.js";

const GROK = ACP_AGENTS.find((a) => a.name === "grok-build")!;

Expand All @@ -36,19 +36,32 @@ describe("seal gate on a subprocess backend", () => {
},
]);
const engine = new Engine({ baseUrl: GROK.url, model: "grok-4.6", provider: "grok-build", cwd: ws.dir, bar: null, acpSpawn: agent.spawnFn, autonomy: "high" });
// The drafter finishes when the gate is holding the write, not at a fixed
// 600 ms. On a loaded machine (150 CPU burners, 2026-10-08) the scripted
// agent's first call came 700–1000 ms in, after a 600 ms drafter had
// already finished: nothing was left to gate, the product rightly did
// not wait, and "the wait was announced" failed 15 runs in 20. The
// fallback only bounds a gate that never holds; that still fails below.
let fileWhenSealed: boolean | undefined;
const pending = new Promise<{ taskChecks: Check[]; taskNotes: string[] }>((resolve) =>
setTimeout(() => {
fileWhenSealed = existsSync(join(ws.dir, "out.txt"));
resolve({ taskChecks: [mk("greeting", "grep -qx hello out.txt")], taskNotes: [] });
}, 600),
);
const ev = await drain(
engine.run("write hello to out.txt", allowAll, {
pendingCriteria: pending,
criteriaSoFar: async () => ({ taskChecks: [], taskNotes: [] }),
}),
);
let gateHeld!: () => void;
let fallback: ReturnType<typeof setTimeout> | undefined;
const held = new Promise<void>((r) => {
gateHeld = r;
fallback = setTimeout(r, 10_000);
});
const pending = held.then(() => {
clearTimeout(fallback);
fileWhenSealed = existsSync(join(ws.dir, "out.txt"));
return { taskChecks: [mk("greeting", "grep -qx hello out.txt")], taskNotes: [] as string[] };
});
const ev: EngineEvent[] = [];
for await (const e of engine.run("write hello to out.txt", allowAll, {
pendingCriteria: pending,
criteriaSoFar: async () => ({ taskChecks: [], taskNotes: [] }),
})) {
ev.push(e);
if (e.kind === "info" && /waiting for this task's checks to be sealed/.test(e.text)) gateHeld();
}
assert.equal(fileWhenSealed, false, "the drafter finished before the file was written");
const wait = ev.findIndex((e) => e.kind === "info" && /waiting for this task's checks to be sealed/.test(e.text));
const write = ev.findIndex((e) => e.kind === "tool" && e.name === "write_file");
Expand Down
24 changes: 21 additions & 3 deletions test/discriminating-checks.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ import { Journal } from "../src/journal.js";
import { Receipts } from "../src/receipts.js";
import { preWorkCopy } from "../src/scratch.js";
import { claimLabel, noteCoverage, tierOf, UNTESTED_CLAIM } from "../src/tiers.js";
import type { Check, CheckAuthor } from "../src/types.js";
import type { Check, CheckAuthor, EngineEvent } from "../src/types.js";
import { allowAll, drain, scriptedProvider, workspace, type ScriptedTurn } from "./helpers.js";

const JUDGE: CheckAuthor = { kind: "judge", model: "qwen3-coder-30b-a3b" };
Expand Down Expand Up @@ -213,7 +213,6 @@ describe("parseArgs --require-discriminating", () => {
});

describe("late checks and the copy taken before the work", () => {
const later = <T>(v: T, ms: number) => new Promise<T>((r) => setTimeout(() => r(v), ms));
const nothing = async () => ({ taskChecks: [] as Check[], taskNotes: [] as string[] });
const mk = (name: string, run: string): Check => ({ name, kind: "command", run, timeoutMs: 5_000, expectExit: 0, tags: ["task", "value", "exact"], hidden: true, author: JUDGE }) as Check;

Expand All @@ -226,7 +225,26 @@ describe("late checks and the copy taken before the work", () => {
requireDiscriminating: true,
});
engine.setTurnDeadline(600_000);
const events = await drain(engine.run(TASK, allowAll, { pendingCriteria: later({ taskChecks: checks, taskNotes: [] }, 300), criteriaSoFar: nothing, criteriaWaitMs: 100 }));
// The draft lands once the engine has cut its wait with none ready, so the
// checks really are late. On a 300 ms timer started before the run, a
// start-up slowed by load (150 CPU burners, 2026-10-08) let the draft be
// ready at the 100 ms cut: it was sealed there, rightly, no copy was
// tried, and these cases failed 1 run in 20. The fallback only bounds an
// engine that never cuts.
let cut!: () => void;
let fallback: ReturnType<typeof setTimeout> | undefined;
const pendingCriteria = new Promise<void>((r) => {
cut = r;
fallback = setTimeout(r, 10_000);
}).then(() => {
clearTimeout(fallback);
return { taskChecks: checks, taskNotes: [] as string[] };
});
const events: EngineEvent[] = [];
for await (const e of engine.run(TASK, allowAll, { pendingCriteria, criteriaSoFar: nothing, criteriaWaitMs: 100 })) {
events.push(e);
if (e.kind === "info" && /none was ready; not sealing an empty set/.test(e.text)) cut();
}
const end = events.find((e) => e.kind === "job_end");
assert.ok(end && end.kind === "job_end");
return { end, tried: Journal.read(journal.path).filter((e) => e.kind === "note" && e.data.kind === "pre-work-try"), copies: Journal.read(journal.path).filter((e) => e.kind === "note" && e.data.kind === "pre-work-copy") };
Expand Down
36 changes: 28 additions & 8 deletions test/ended-with-verdict.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,10 @@ import assert from "node:assert/strict";
import { readdirSync, readFileSync } from "node:fs";
import { join } from "node:path";
import { describe, it } from "node:test";
import { Engine } from "../src/engine.js";
import { Engine, type RunOptions } from "../src/engine.js";
import { Receipts } from "../src/receipts.js";
import { LatencyLearner } from "../src/watchdog.js";
import type { Check } from "../src/types.js";
import type { Check, EngineEvent } from "../src/types.js";
import { allowAll, drain, scriptedProvider, workspace } from "./helpers.js";

const greet = (hidden = true): Check => ({
Expand All @@ -31,6 +31,26 @@ const slowWrite = (content: string) => [
{ text: "never asked for" },
];

/**
* Run the turn with its 400 ms budget armed once out.txt is being written.
*
* Armed at the start, the budget raced the engine's start-up: under 150 CPU
* burners (2026-10-08) it ran out before the scripted reply was read ("time
* budget reached (414ms of 400ms) — no more tool calls"), nothing was
* written, and the bar rightly failed the empty tree; `passedAtEnd` failed
* 14 runs in 20. The budget is read live (`deadlineAt`), so arming it at the
* write makes the clock stop the turn after the work, under any load: during
* the 0.7 s command if the write came early, at once if it came late.
*/
async function stoppedAfterWrite(engine: Engine, opts?: RunOptions) {
const events: EngineEvent[] = [];
for await (const e of engine.run("write out.txt", allowAll, opts)) {
events.push(e);
if (e.kind === "tool" && e.name === "write_file") engine.setTurnDeadline(400);
}
return events;
}

describe("a turn stopped by the clock", () => {
// Mercury 2.5: three half-finished trees passed weak drafted checks at the
// deadline and were "verified"; the grader failed all three.
Expand All @@ -40,9 +60,9 @@ describe("a turn stopped by the clock", () => {
const p = scriptedProvider(slowWrite("hello\n"));
const engine = new Engine({
baseUrl: "http://provider.test/v1", model: "m", cwd: ws.dir, fetchFn: p.fetchFn, bar: null, stream: false,
autonomy: "high", turnDeadlineMs: 400, receipts: new Receipts(ws.dir),
autonomy: "high", receipts: new Receipts(ws.dir),
});
const events = await drain(engine.run("write out.txt", allowAll, { taskChecks: [greet()] }));
const events = await stoppedAfterWrite(engine, { taskChecks: [greet()] });
const e = end(events);
assert.equal(e.outcome, "unverified");
assert.equal(e.passedAtEnd, true);
Expand All @@ -65,10 +85,10 @@ describe("a turn stopped by the clock", () => {
const p = scriptedProvider(slowWrite("goodbye\n"));
const engine = new Engine({
baseUrl: "http://provider.test/v1", model: "m", cwd: ws.dir, fetchFn: p.fetchFn, bar: null, stream: false,
autonomy: "high", turnDeadlineMs: 400,
autonomy: "high",
});
// A person's check, so the refusal is not softened to "unverified".
const events = await drain(engine.run("write out.txt", allowAll, { taskChecks: [greet(false)] }));
const events = await stoppedAfterWrite(engine, { taskChecks: [greet(false)] });
const e = end(events);
assert.equal(e.outcome, "not proven");
assert.equal(e.deadline, true);
Expand All @@ -83,9 +103,9 @@ describe("a turn stopped by the clock", () => {
const p = scriptedProvider([...slowWrite("hello\n"), { text: "Wrote out.txt; ran out of time." }]);
const engine = new Engine({
baseUrl: "http://provider.test/v1", model: "m", cwd: ws.dir, fetchFn: p.fetchFn, bar: null, stream: false,
autonomy: "high", turnDeadlineMs: 400,
autonomy: "high",
});
const e = end(await drain(engine.run("write out.txt", allowAll)));
const e = end(await stoppedAfterWrite(engine));
assert.equal(e.outcome, "unverified");
assert.equal(e.deadline, true);
} finally {
Expand Down
Loading
Loading