From 7d71a1f385d828b3c4e066de07aee0862ef44525 Mon Sep 17 00:00:00 2001 From: Tyler Skelton Date: Wed, 7 Oct 2026 09:05:05 -0700 Subject: [PATCH 1/2] Wait for late drafted checks until the time budget's margin A claim of done waited a fixed 120 s for checks still being drafted. A separate judge drafting for 170-300 s (a local model past 6 min) finished after that, and correct work was judged without task checks and labelled plain unverified. Before waiting longer, the drafts had to be independent of the work, and they were not: every drafter stage re-read the work folder. A redraft or cover step running after the cut saw the worker's output (a task naming out.txt got "out.txt ... first line: 3" in its prompt), and the second draft attempt re-read scripts, commands and lessons live. - The drafter's inputs (project view, tree, commands, scripts, lessons) are frozen once before the first step and used by every stage: first draft, lint redraft, critic, cover step, redraft, and the second try. - drafterInputsHash covers that snapshot. Each stage reports the hash of what it actually put in its prompt; the join compares them with the turn-start hash and refuses drafts built from anything else. - A subprocess (ACP) drafter starts in an empty temp folder, not the work. - With frozen inputs and a time budget, a claim waits for late drafts until what is left less max(60 s, 15% of the budget), never less than before. No budget, or inputs that differ: the old wait. Only after a claim; the deadline without a claim is unchanged. - Late checks join the same seal and bar; a failing one refuses the claim. Journal and info lines say how long it waited and whether the drafts arrived. --- src/cli.tsx | 33 ++++-- src/criteria.ts | 131 ++++++++++++++++++++---- src/engine.ts | 81 +++++++++++++-- test/drafter-view.test.ts | 103 ++++++++++++++++++- test/late-criteria.test.ts | 201 ++++++++++++++++++++++++++++++++++++- 5 files changed, 513 insertions(+), 36 deletions(-) diff --git a/src/cli.tsx b/src/cli.tsx index 94f699b..fc937e3 100644 --- a/src/cli.tsx +++ b/src/cli.tsx @@ -29,7 +29,7 @@ import { draftMission, missionStatus, runMission, writePlan, type MissionSummary import { parseDuration } from "./session-commands.js"; import { passedChecksWords } from "./tiers.js"; import { judgeEffort, judgeTarget } from "./judge.js"; -import { commandsHere,draftCriteriaCritiqued, preflightCriteria, taskChecksFrom, type Draft } from "./criteria.js"; +import { commandsHere, draftCriteriaCritiqued, drafterInputsHash, drafterSnapshot, preflightCriteria, taskChecksFrom, type Draft, type DrafterInputs } from "./criteria.js"; import { listProject, removeNew } from "./leftovers.js"; import { projectScripts } from "./interview.js"; import { @@ -1189,6 +1189,15 @@ async function cmdMission(args: Args): Promise { } } +/** What the criteria drafter may read, taken before the turn's first step. */ +function drafterSnapshotFor(args: Args): DrafterInputs { + return drafterSnapshot(args.task ?? "", args.cwd, { + commands: commandsHere(args.cwd), + scripts: projectScripts(args.cwd), + lessons: new Judgments(args.cwd).lessons(), + }); +} + /** * `--criteria auto`: the model writes the exam, and is then held to it. * @@ -1205,17 +1214,23 @@ async function autoDraft( engine: Engine, args: Args, soFar?: { draft?: Draft; sealed?: boolean; late?: boolean }, + inputs?: { snapshot: DrafterInputs; used: string[] }, ): Promise> { + // Taken once, now, before the first step: the second try below and every + // stage of each draft read this and never the folder the work is changing. + const snapshot = inputs?.snapshot ?? drafterSnapshotFor(args); const none: ReturnType = { taskChecks: [], taskNotes: [] }; // Drafted, then read cold by a critic against the task text: a check that // invents or guesses is dropped (with a task quote), and a draft where // nothing runs the deliverable is asked for once more. See criteria.ts. const draftOnce = () => draftCriteriaCritiqued({ - commands: commandsHere(args.cwd), - lessons: new Judgments(args.cwd).lessons(), + snapshot, + ...(snapshot.commands ? { commands: snapshot.commands } : {}), + lessons: [...snapshot.lessons], task: args.task ?? "", - scripts: projectScripts(args.cwd), + scripts: [...snapshot.scripts], + onInputs: (sha) => inputs?.used.push(sha), barChecks: (engine.cfg.bar?.checks ?? []).map((c) => c.name), ...judgeTarget({ baseUrl: args.url, apiKey: args.key, model: args.model }), cwd: args.cwd, @@ -1518,7 +1533,13 @@ async function cmdRun(args: Args, ask = false): Promise { // Drafted while the model starts reading: the engine seals them before the // first change or claim, so they still predate the work (see RunOptions). const soFar: { draft?: Draft; sealed?: boolean; late?: boolean } = {}; - const pendingCriteria = args.autoCriteria && !ask ? autoDraft(engine, args, soFar) : undefined; + // The drafter's inputs, frozen here; the engine journals their hash at turn + // start and compares it with what each drafter stage actually used. + const drafterInputs = args.autoCriteria && !ask ? { snapshot: drafterSnapshotFor(args), used: [] as string[] } : undefined; + const pendingCriteria = drafterInputs ? autoDraft(engine, args, soFar, drafterInputs) : undefined; + const draftInputs = drafterInputs + ? { sha: drafterInputsHash(drafterInputs.snapshot), used: () => [...drafterInputs.used] } + : undefined; const criteriaSoFar = pendingCriteria ? () => { // Nothing reviewed yet: leave the draft running to join at the claim @@ -1535,7 +1556,7 @@ async function cmdRun(args: Args, ask = false): Promise { // How long a timed turn waits for its drafted checks; for tests and debugging. const waitEnv = Number(env("CRITERIA_WAIT_MS")); const criteriaWait = Number.isFinite(waitEnv) && waitEnv > 0 ? { criteriaWaitMs: waitEnv } : {}; - for await (const ev of engine.run(args.task, confirm, { ask, taskChecks, taskNotes, pendingCriteria, criteriaSoFar, referenceCheck, ...criteriaWait })) { + for await (const ev of engine.run(args.task, confirm, { ask, taskChecks, taskNotes, pendingCriteria, criteriaSoFar, referenceCheck, ...(draftInputs ? { draftInputs } : {}), ...criteriaWait })) { emit(ev); if (ev.kind === "proof_exhausted" && ev.result.undetermined?.length) undetermined = true; // The sentence that explains an undetermined bar arrives as an error, so diff --git a/src/criteria.ts b/src/criteria.ts index e532ad6..d54a503 100644 --- a/src/criteria.ts +++ b/src/criteria.ts @@ -22,14 +22,15 @@ * A sentence dressed as a check is worse than no check. */ import { execFileSync } from "node:child_process"; -import { openSync, readSync, closeSync, realpathSync, statSync } from "node:fs"; +import { createHash } from "node:crypto"; +import { openSync, readSync, closeSync, mkdtempSync, realpathSync, rmSync, statSync } from "node:fs"; import { tmpdir } from "node:os"; -import { isAbsolute, relative } from "node:path"; +import { isAbsolute, join, relative } from "node:path"; import { topLevel } from "./brief.js"; import { namedInputs, profileLine } from "./inspect.js"; import { askModel, type AskOptions } from "./ask.js"; import { runCommand, draftedShell, bashPath } from "./run.js"; -import { lintAll, readTree, type LintCtx } from "./checklint.js"; +import { lintAll, readTree, type LintCtx, type Tree } from "./checklint.js"; import { diagnoseFailure } from "./bar.js"; import { normalizeRequirements } from "./signout.js"; import { evidenceTags } from "./tiers.js"; @@ -390,6 +391,61 @@ export function projectView(task: string, cwd?: string): string { return lines.join("\n"); } +/** + * Everything the drafter reads about the project, taken once, before the first + * step. Every drafter stage — the first draft, the lint redraft, the critic, + * the cover step, the redraft after review, and a whole second try — reads + * this and never the live folder: those stages run while the worker is + * already changing files, and a redraft that re-read the tree once saw the + * worker's out.txt and could have sealed its answer as the check. + */ +export type DrafterInputs = { + /** The task text as given, before any review findings are appended. */ + readonly task: string; + /** projectView(task, cwd) at turn start. */ + readonly view: string; + /** readTree(cwd) at turn start, for the seal-time lint. */ + readonly tree?: Tree; + readonly commands?: { present: string[]; missing: string[] }; + readonly scripts: readonly string[]; + readonly lessons: readonly string[]; +}; + +/** Take the drafter's inputs now. */ +export function drafterSnapshot( + task: string, + cwd: string | undefined, + rest: { commands?: { present: string[]; missing: string[] }; scripts?: string[]; lessons?: string[] }, +): DrafterInputs { + return Object.freeze({ + task, + view: projectView(task, cwd), + ...(cwd ? { tree: readTree(cwd) } : {}), + ...(rest.commands ? { commands: { present: [...rest.commands.present], missing: [...rest.commands.missing] } } : {}), + scripts: Object.freeze([...(rest.scripts ?? [])]), + lessons: Object.freeze([...(rest.lessons ?? [])]), + }); +} + +/** + * A hash of the drafter's inputs. Taken at turn start over the snapshot, and + * again by each drafter stage over the values it actually put in its prompt, + * so "drafted without sight of the work" is a comparison the record can fail. + */ +export function drafterInputsHash(d: DrafterInputs): string { + const h = createHash("sha256"); + const part = (k: string, v: string) => h.update(`\0${k}\0${v}`, "utf8"); + part("task", d.task); + part("view", d.view); + part("files", d.tree ? [...d.tree.files].sort().join("\n") : ""); + part("text", d.tree?.text ?? ""); + part("present", (d.commands?.present ?? []).join(",")); + part("missing", (d.commands?.missing ?? []).join(",")); + part("scripts", d.scripts.join("\n")); + part("lessons", d.lessons.join("\n")); + return h.digest("hex").slice(0, 16); +} + const SYSTEM = [ "You draft acceptance criteria for one coding task. You are not doing the task", "and you will not judge whether it was done — a person approves what you write", @@ -532,7 +588,7 @@ async function edgeProbes(opts: Parameters[0], view: strin model: opts.model, system: EDGE_SYSTEM, prompt: [`Task: ${opts.task}`, ...(view ? ["", view] : [])].join("\n"), - cwd: opts.cwd, + cwd: opts.askCwd ?? opts.cwd, what: "probing edge cases", fetchFn: opts.fetchFn, acpSpawn: opts.acpSpawn, @@ -590,21 +646,42 @@ export async function draftCriteria(opts: { * the mistakes not to seal again. */ lessons?: string[]; + /** + * The project as it was before the work (drafterSnapshot). When given, the + * view, scripts, commands and lessons come from it and the folder is not + * read again; without it they are read now. + */ + snapshot?: DrafterInputs; + /** Called with drafterInputsHash of what this call put in its prompt. */ + onInputs?: (sha: string) => void; + /** + * Where a subprocess (ACP) drafter is started. Kept apart from the work + * folder so the agent answering cannot open the worker's files. + */ + askCwd?: string; }): Promise<{ ok: true; draft: Draft } | { ok: false; error: string }> { - const view = projectView(opts.task, opts.cwd); + const snap = opts.snapshot; + const view = snap ? snap.view : projectView(opts.task, opts.cwd); + const scripts = snap ? [...snap.scripts] : opts.scripts; + const commands = snap ? snap.commands : opts.commands; + const lessons = snap ? [...snap.lessons] : opts.lessons; + opts.onInputs?.( + drafterInputsHash({ task: snap?.task ?? opts.task, view, ...(snap?.tree ? { tree: snap.tree } : {}), ...(commands ? { commands } : {}), scripts, lessons: lessons ?? [] }), + ); + const askCwd = opts.askCwd ?? opts.cwd; const context = [ `Task: ${opts.task}`, ...(view ? ["", view] : []), "", - `Scripts available: ${opts.scripts.length ? opts.scripts.join(", ") : "(none found)"}`, + `Scripts available: ${scripts.length ? scripts.join(", ") : "(none found)"}`, `The project already checks: ${opts.barChecks.length ? opts.barChecks.join(", ") : "(nothing)"}`, - ...(opts.commands ? [`Commands on this machine: ${opts.commands.present.join(", ")}${opts.commands.missing.length ? ` (not installed: ${opts.commands.missing.join(", ")})` : ""}`] : []), + ...(commands ? [`Commands on this machine: ${commands.present.join(", ")}${commands.missing.length ? ` (not installed: ${commands.missing.join(", ")})` : ""}`] : []), ...(process.platform === "darwin" ? ["This is macOS: BSD tools, not GNU. No `find -printf`, no `stat -c`, no GNU `touch -d`, `sed -i ''` needs the empty argument. Prefer python3 for anything beyond plain shell."] : []), "`.maat/` is Maat's own folder in the project: a check that lists or counts files must ignore it.", - ...(opts.lessons?.length - ? ["", "A person ruled these earlier drafted checks wrong in this project. Do not make the same mistake:", ...opts.lessons.map((l) => `- ${l}`)] + ...(lessons?.length + ? ["", "A person ruled these earlier drafted checks wrong in this project. Do not make the same mistake:", ...lessons.map((l) => `- ${l}`)] : []), "", "Do not repeat what the project already checks. Add only what is specific to", @@ -619,7 +696,7 @@ export async function draftCriteria(opts: { model: opts.model, system: SYSTEM, prompt: prompted, - cwd: opts.cwd, + cwd: askCwd, what: "drafting criteria", fetchFn: opts.fetchFn, acpSpawn: opts.acpSpawn, @@ -639,7 +716,7 @@ export async function draftCriteria(opts: { model: opts.model, system: SYSTEM, prompt: `${prompted}\n\nYour last reply could not be parsed as JSON. Reply with the JSON object only.`, - cwd: opts.cwd, + cwd: askCwd, what: "drafting criteria", fetchFn: opts.fetchFn, acpSpawn: opts.acpSpawn, @@ -663,7 +740,7 @@ export async function draftCriteria(opts: { model: opts.model, system: SYSTEM, prompt: context, - cwd: opts.cwd, + cwd: askCwd, what: "drafting criteria", fetchFn: opts.fetchFn, acpSpawn: opts.acpSpawn, @@ -821,8 +898,21 @@ export async function draftCriteriaCritiqued( // back, however many ways the function below returns. const sink: Sink = { requirements: [], lint: [] }; const withReqs = (d: Draft): Draft => (sink.requirements.length ? { ...d, requirements: sink.requirements } : d); - const r = await critiqued({ ...opts, onProgress: opts.onProgress && ((d) => opts.onProgress!(withReqs(d))) }, sink); - return r.ok ? { ...r, draft: withReqs(r.draft), ...(sink.lint.length ? { lint: sink.lint } : {}) } : r; + // Every stage below reads the project as it is now, not as the worker + // leaves it: the later stages run while the work is under way. + const snapshot = + opts.snapshot ?? drafterSnapshot(opts.task, opts.cwd, { commands: opts.commands, scripts: opts.scripts, lessons: opts.lessons }); + // A subprocess drafter starts in an empty folder of its own, not the work. + const ownDir = opts.askCwd === undefined ? mkdtempSync(join(tmpdir(), "maat-draft-")) : undefined; + try { + const r = await critiqued( + { ...opts, snapshot, askCwd: opts.askCwd ?? ownDir, onProgress: opts.onProgress && ((d) => opts.onProgress!(withReqs(d))) }, + sink, + ); + return r.ok ? { ...r, draft: withReqs(r.draft), ...(sink.lint.length ? { lint: sink.lint } : {}) } : r; + } finally { + if (ownDir) rmSync(ownDir, { recursive: true, force: true }); + } } /** A drafted check the seal-time lint retired (src/checklint.ts), for the journal. */ @@ -838,14 +928,17 @@ async function critiqued( }, sink: Sink, ): Promise<{ ok: true; draft: Draft; critique: string[] } | { ok: false; error: string }> { + const snap = opts.snapshot ?? drafterSnapshot(opts.task, opts.cwd, { commands: opts.commands, scripts: opts.scripts, lessons: opts.lessons }); + opts = { ...opts, snapshot: snap }; + const commands = snap.commands; const fix = (d: Draft): Draft => - opts.commands ? { ...d, checks: d.checks.map((c) => ({ ...c, run: fixInterpreters(c.run, opts.commands!) })) } : d; - const view = projectView(opts.task, opts.cwd); - // The project as it is now, before the model has changed anything. + commands ? { ...d, checks: d.checks.map((c) => ({ ...c, run: fixInterpreters(c.run, commands) })) } : d; + const view = snap.view; + // The project as it was before the model changed anything (the snapshot). const lintCtx: LintCtx = { task: opts.task, shell: bashPath() ? "bash" : "sh", - ...(opts.cwd ? { cwd: opts.cwd, tree: readTree(opts.cwd) } : {}), + ...(opts.cwd ? { cwd: opts.cwd, tree: snap.tree ?? readTree(opts.cwd) } : {}), }; const said: string[] = []; /** Split a draft's checks into the ones that pass the lint and the ones that do not. Drops are recorded, never silent. */ @@ -904,7 +997,7 @@ async function critiqued( model: opts.model, system: CRITIC_SYSTEM, prompt: `TASK TEXT:\n${opts.task}\n\n${view ? `${view}\n\n` : ""}CHECKS:\n${d.checks.map((c) => `- ${c.name}: ${c.run}`).join("\n")}`, - cwd: opts.cwd, + cwd: opts.askCwd ?? opts.cwd, what: "reviewing the drafted checks", fetchFn: opts.fetchFn, acpSpawn: opts.acpSpawn, diff --git a/src/engine.ts b/src/engine.ts index 29657b8..1b69b42 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -1036,6 +1036,16 @@ export type RunOptions = { referenceWaitMs?: number; /** Overrides how long a claim waits for checks still drafting after the time budget's cut. Tests only. */ lateCriteriaWaitMs?: number; + /** + * The drafter's inputs, frozen before the first step (criteria.ts + * drafterSnapshot): `sha` is their hash, `used` the hash each drafter stage + * computed over what it actually put in its prompt. When every stage used + * the snapshot, the drafts are independent of the work, and a claim of done + * may wait for them until the time budget's safety margin + * (claimWaitForDraftsMs). A mismatch means a stage read the folder the work + * changed: its checks are not joined. + */ + draftInputs?: { sha: string; used: () => string[] }; /** * Criteria stated in words rather than as commands. * @@ -1173,6 +1183,23 @@ export function lateCriteriaWaitMs(timeLeftMs: number | undefined): number { return Math.max(0, Math.min(120_000, Math.round(timeLeftMs * 0.6))); } +/** + * How long a claim of done waits for drafted checks still on their way, when + * the drafter's inputs are known to be sealed before the work (RunOptions + * .draftInputs): until what is left of the time budget, less a margin for + * running the checks and the review — the larger of 60 s and 15% of the + * budget. Never shorter than lateCriteriaWaitMs. A separate judge (Grok at + * 170–300 s, a local model past 6 min) routinely finished after the fixed + * two minutes, and correct work was labelled unverified. No budget, the old + * two minutes. + */ +export function claimWaitForDraftsMs(timeLeftMs: number | undefined, budgetMs: number): number { + const floor = lateCriteriaWaitMs(timeLeftMs); + if (timeLeftMs === undefined || !budgetMs) return floor; + const margin = Math.max(60_000, Math.round(budgetMs * 0.15)); + return Math.max(floor, timeLeftMs - margin); +} + /** * A hash of everything the drafter reads: the task text and the project's file * listing before the first step. Journalled at turn start and again when late @@ -4990,7 +5017,7 @@ export class Engine { let lateCriteria: typeof pendingCriteria; let draftInputs = ""; if (pendingCriteria && !opts.ask) { - draftInputs = draftInputsHash(userText, listProject(this.cwd)); + draftInputs = opts.draftInputs?.sha ?? draftInputsHash(userText, listProject(this.cwd)); log?.append("note", { kind: "draft-inputs", text: "drafter inputs fixed at turn start", inputsSha: draftInputs }); } /** Sealed mid-step: tell the model once this step's tool results are in. */ @@ -5164,6 +5191,17 @@ export class Engine { let bar = opts.ask ? withoutWriteChecks(withTaskChecks(this.cfg.bar, taskChecks)) : withTaskChecks(this.cfg.bar, taskChecks); + /** + * Whether the drafted checks are known to be independent of the work: + * the caller froze the drafter's inputs before the first step, and every + * drafter stage so far used exactly those. + */ + const draftsIndependent = (): boolean => + opts.draftInputs !== undefined && opts.draftInputs.used().every((h) => h === draftInputs); + /** How long a claim waits for checks still drafting (see claimWaitForDraftsMs). */ + const lateWait = (): number => + opts.lateCriteriaWaitMs ?? + (draftsIndependent() ? claimWaitForDraftsMs(self.timeLeftMs(), self.turnDeadlineMs) : lateCriteriaWaitMs(self.timeLeftMs())); /** * Seal criteria that were still being drafted. Called before the first * call that changes anything and before the first claim, so the checks @@ -5199,7 +5237,7 @@ export class Engine { kind: "info", text: `checks were still being drafted after ${Math.round(cap / 1000)}s of the time budget and none was ready; ` + - `not sealing an empty set — the first claim waits up to ${Math.round((opts.lateCriteriaWaitMs ?? lateCriteriaWaitMs(self.timeLeftMs())) / 1000)}s for them and judges with them if they arrive`, + `not sealing an empty set — the first claim waits up to ${Math.round(lateWait() / 1000)}s for them and judges with them if they arrive`, }; } else { yield { @@ -5276,7 +5314,9 @@ export class Engine { async function* joinLateCriteria(): AsyncGenerator { const p = lateCriteria; if (!p) return; - const wait = opts.lateCriteriaWaitMs ?? lateCriteriaWaitMs(self.timeLeftMs()); + const independent = draftsIndependent(); + const wait = lateWait(); + const began = Date.now(); let timer: NodeJS.Timeout | undefined; const late = Symbol("late"); const got = await Promise.race([ @@ -5284,11 +5324,36 @@ export class Engine { new Promise((r) => { timer = setTimeout(() => r(late), wait); }), ]); clearTimeout(timer); + const waited = Math.round((Date.now() - began) / 1000); + const budgetNote = independent && self.turnDeadlineMs ? " (waited to the time budget's margin: the drafter's inputs were sealed before the work)" : ""; if (got === late) { - yield { kind: "info", text: `the drafted checks did not arrive within ${Math.round(wait / 1000)}s; this claim is judged without them, so it cannot be verified by task checks` }; + log?.append("note", { kind: "late-checks", text: `drafted checks did not arrive after ${waited}s`, waitedMs: Date.now() - began, arrived: false, inputsSha: draftInputs }); + yield { + kind: "info", + text: `waited ${waited}s for the drafted checks${budgetNote}; they did not arrive within ${Math.round(wait / 1000)}s, so this claim is judged without them and cannot be verified by task checks`, + }; return; } lateCriteria = undefined; + // Recomputed from what the drafter stages actually used, now that they + // are done: a stage that read the changed folder shows here. + const used = opts.draftInputs?.used() ?? []; + const strays = used.filter((h) => h !== draftInputs); + if (opts.draftInputs && strays.length) { + log?.append("note", { + kind: "late-checks", + text: "late drafted checks NOT joined: a drafter stage used inputs that differ from the turn-start snapshot", + inputsSha: draftInputs, + used, + arrived: true, + waitedMs: Date.now() - began, + }); + yield { + kind: "info", + text: `the drafted checks arrived after ${waited}s but were drafted from a view of the project that differs from the one taken before the work; they are not joined, so this claim cannot be verified by task checks`, + }; + return; + } const have = new Set(taskChecks.map((c) => c.name)); const added = (got?.taskChecks ?? []) .map((c) => Object.freeze({ ...c, name: c.name.startsWith("task:") ? c.name : `task:${c.name}` })) @@ -5305,14 +5370,16 @@ export class Engine { bar = withTaskChecks(self.cfg.bar, taskChecks); log?.append("note", { kind: "late-checks", - text: `late drafted checks joined the sealed checks: ${added.length} check(s)`, - inputsSha: draftInputs, + text: `late drafted checks joined the sealed checks: ${added.length} check(s) after ${waited}s`, + inputsSha: used.length ? used[used.length - 1] : draftInputs, + waitedMs: Date.now() - began, + arrived: true, seal: taskSeal, checks: added.map((c) => c.name), }); yield { kind: "info", - text: `${added.length} drafted check(s) joined this turn's checks — written from the task text and the project as it was before the work`, + text: `waited ${waited}s for the drafted checks${budgetNote}; ${added.length} arrived and joined this turn's checks — written from the task text and the project as it was before the work`, }; self.transcript.push({ role: "user", diff --git a/test/drafter-view.test.ts b/test/drafter-view.test.ts index 2b11e86..24219a6 100644 --- a/test/drafter-view.test.ts +++ b/test/drafter-view.test.ts @@ -3,10 +3,12 @@ * check that reaches outside the project is dropped before it is sealed. */ import assert from "node:assert/strict"; -import { mkdirSync, writeFileSync } from "node:fs"; +import { existsSync, mkdirSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { describe, it } from "node:test"; -import { draftCriteria, draftCriteriaCritiqued, preflightCriteria, projectView, strayPath } from "../src/criteria.js"; +import { draftCriteria, draftCriteriaCritiqued, drafterInputsHash, drafterSnapshot, preflightCriteria, projectView, strayPath } from "../src/criteria.js"; +import { ACP_AGENTS } from "../src/acp.js"; +import { scriptedAcpAgent } from "./acp-agent.js"; import { workspace } from "./helpers.js"; function capturing(texts: string[]): { fetchFn: typeof fetch; prompts: string[] } { @@ -140,3 +142,100 @@ describe("absolute paths in drafted checks", () => { } }); }); + +/** + * The drafter's later stages (cover step, redraft after review, lint redraft, + * a whole second try) run while the worker is already changing files. They + * read the snapshot taken before the work, never the live folder: a redraft + * that re-read the tree once saw the worker's out.txt, and could have sealed + * its answer as the check. + */ +describe("the drafter's inputs are frozen before the work", () => { + const TASK = "Write the word count of input.txt to out.txt"; + const draft = '{"checks":[{"name":"counts","run":"test \\"$(cat out.txt)\\" = 3"}],"notes":[]}'; + + /** A provider that writes the worker's out.txt the moment the first draft is in. */ + function workerWritesAfterFirstDraft(dir: string, critique: string) { + const prompts: string[] = []; + const fetchFn = (async (_u: string, init?: RequestInit) => { + const msgs = (JSON.parse(String(init?.body)) as { messages: { role: string; content: string }[] }).messages; + const prompt = msgs.filter((m) => m.role !== "system").map((m) => m.content).join("\n"); + prompts.push(prompt); + // The worker, past the seal, writes its answer while the drafter works. + if (prompts.length === 1) writeFileSync(join(dir, "out.txt"), "3\n"); + const content = prompt.startsWith("TASK TEXT:") ? critique : draft; + return { ok: true, status: 200, json: async () => ({ choices: [{ message: { content }, finish_reason: "stop" }] }), text: async () => "" } as unknown as Response; + }) as unknown as typeof fetch; + return { fetchFn, prompts }; + } + + for (const [path, critique] of [ + ["the cover step", '{"checks":[{"name":"counts","verdict":"runs","quote":""}],"uncovered":["word count of input.txt"],"requirements":[]}'], + ["the redraft after review", '{"checks":[{"name":"counts","verdict":"surface","quote":""}],"uncovered":[],"requirements":[]}'], + ] as const) { + it(`${path} never shows the worker's out.txt (the word-count leak)`, async () => { + const ws = workspace(); + try { + writeFileSync(join(ws.dir, "input.txt"), "a b c\n"); + const snapshot = drafterSnapshot(TASK, ws.dir, { scripts: [], lessons: [] }); + const used: string[] = []; + const { fetchFn, prompts } = workerWritesAfterFirstDraft(ws.dir, critique); + const r = await draftCriteriaCritiqued({ ...base, task: TASK, cwd: ws.dir, fetchFn, snapshot, onInputs: (h) => used.push(h) }); + assert.ok(r.ok); + const drafts = prompts.filter((p) => p.startsWith("Task:")); + assert.ok(drafts.length >= 2, `a later drafter stage ran (${prompts.length} prompts)`); + assert.ok(existsSync(join(ws.dir, "out.txt")), "the worker's file is there while the later stages run"); + // The live view would now show it; the drafter must not. + assert.match(projectView(TASK, ws.dir), /out\.txt: .*\n\s+first line: 3/); + for (const p of prompts) { + assert.ok(!/out\.txt: \d+ bytes/.test(p) && !/first line: 3/.test(p), `a drafter prompt saw the worker's out.txt:\n${p}`); + assert.ok(!/Project files \(top level\): .*out\.txt/.test(p), "nor lists it"); + } + // Every stage hashed what it used, and it is the snapshot. + assert.ok(used.length >= 2); + for (const h of used) assert.equal(h, drafterInputsHash(snapshot)); + } finally { + ws.cleanup(); + } + }); + } + + it("the hash tells a live read from the snapshot, so the comparison can fail", async () => { + const ws = workspace(); + try { + writeFileSync(join(ws.dir, "input.txt"), "a b c\n"); + const snapshot = drafterSnapshot(TASK, ws.dir, { scripts: [], lessons: [] }); + writeFileSync(join(ws.dir, "out.txt"), "3\n"); + const reply = (async () => + ({ ok: true, status: 200, json: async () => ({ choices: [{ message: { content: '{"checks":[],"notes":[]}' }, finish_reason: "stop" }] }), text: async () => "" }) as unknown as Response) as unknown as typeof fetch; + const frozen: string[] = []; + const live: string[] = []; + await draftCriteria({ ...base, task: TASK, cwd: ws.dir, fetchFn: reply, snapshot, onInputs: (h) => frozen.push(h) }); + // A stage that reads the folder as it is now (the old behaviour). + await draftCriteria({ ...base, task: TASK, cwd: ws.dir, fetchFn: reply, onInputs: (h) => live.push(h) }); + assert.deepEqual(frozen, [drafterInputsHash(snapshot)]); + assert.equal(live.length, 1); + assert.notEqual(live[0], drafterInputsHash(snapshot), "reading the changed folder shows in the hash"); + } finally { + ws.cleanup(); + } + }); + + it("a subprocess (ACP) drafter is started in an empty folder of its own, not the work", async () => { + const ws = workspace(); + try { + writeFileSync(join(ws.dir, "input.txt"), "a b c\n"); + const agent = scriptedAcpAgent([{ text: '{"checks":[],"notes":[]}' }]); + const grok = ACP_AGENTS.find((a) => a.name === "grok-build")!; + const r = await draftCriteriaCritiqued({ ...base, baseUrl: grok.url, model: "grok-4.6", task: TASK, cwd: ws.dir, acpSpawn: agent.spawnFn }); + assert.ok(r.ok, r.ok ? "" : r.error); + const cwd = String(agent.sessionParams().cwd ?? ""); + assert.ok(cwd, "the session was opened with a folder"); + assert.notEqual(cwd, ws.dir); + assert.ok(!cwd.startsWith(ws.dir), `drafter started inside the work: ${cwd}`); + assert.ok(!existsSync(cwd), "and the folder is removed afterwards"); + } finally { + ws.cleanup(); + } + }); +}); diff --git a/test/late-criteria.test.ts b/test/late-criteria.test.ts index 56aea43..1713fb0 100644 --- a/test/late-criteria.test.ts +++ b/test/late-criteria.test.ts @@ -7,8 +7,9 @@ * runs, a zero-check seal followed, and every correct result went unverified. */ import assert from "node:assert/strict"; -import { describe, it } from "node:test"; -import { draftInputsHash, Engine } from "../src/engine.js"; +import { describe, it, mock } from "node:test"; +import { claimWaitForDraftsMs, draftInputsHash, Engine, lateCriteriaWaitMs } from "../src/engine.js"; +import type { EngineEvent } from "../src/types.js"; import { Journal } from "../src/journal.js"; import type { Check } from "../src/types.js"; import { allowAll, drain, scriptedProvider, workspace } from "./helpers.js"; @@ -120,3 +121,199 @@ describe("zero checks at the time budget's cut", () => { } }); }); + +/** + * A claim of done waits for drafts still on their way until the time budget's + * margin — not a fixed two minutes — when the drafter's inputs were frozen + * before the work (RunOptions.draftInputs). Measured 2026-10-07: a separate + * judge drafting for 170–300 s (a local model past 6 min) finished after the + * claim's 120 s, and correct work was labelled plain unverified. + */ +describe("a claim waits for late drafts until the time budget's margin", () => { + const realTimeout = setTimeout; + const slow = (name: string, run: string): Check => ({ ...mk(name, run), timeoutMs: 10_000_000 }); + /** Run a turn on a virtual clock: one simulated second per real 2 ms. */ + async function onVirtualClock(start: (at: (v: T, ms: number) => Promise) => AsyncGenerator): Promise<{ ev: EngineEvent[]; simMs: number }> { + mock.timers.enable({ apis: ["setTimeout", "Date"], now: Date.now() }); + const t0 = Date.now(); + try { + const ev: EngineEvent[] = []; + let done = false; + let err: unknown; + const run = (async () => { + for await (const e of start((v, ms) => new Promise((r) => setTimeout(() => r(v), ms)))) ev.push(e); + })().then( + () => { done = true; }, + (e: unknown) => { err = e; done = true; }, + ); + for (let i = 0; i < 2_000 && !done; i++) { + await new Promise((r) => realTimeout(r, 2)); + mock.timers.tick(1_000); + } + await run; + if (err) throw err; + return { ev, simMs: Date.now() - t0 }; + } finally { + mock.timers.reset(); + } + } + const infos = (ev: EngineEvent[]) => ev.filter((e) => e.kind === "info").map((e) => (e as { text: string }).text); + const outcome = (ev: EngineEvent[]) => { + const end = ev.find((e) => e.kind === "job_end"); + return end && end.kind === "job_end" ? end.outcome : undefined; + }; + const sealed = { sha: "frozen-inputs", used: () => ["frozen-inputs"] }; + + it("the numbers: a 600 s budget waits to its 90 s margin, never less than before; no budget keeps 120 s", () => { + assert.equal(claimWaitForDraftsMs(590_000, 600_000), 500_000); + assert.ok(claimWaitForDraftsMs(590_000, 600_000) > 200_000, "drafts at 200 s are inside the wait"); + assert.equal(lateCriteriaWaitMs(590_000), 120_000, "the old wait ended before them"); + // The margin is the larger of 60 s and 15% of the budget. + assert.equal(claimWaitForDraftsMs(250_000, 300_000), 190_000); + // Late in the budget it never waits less than the old rule. + assert.equal(claimWaitForDraftsMs(80_000, 600_000), lateCriteriaWaitMs(80_000)); + assert.equal(claimWaitForDraftsMs(undefined, 0), 120_000); + }); + + it("drafts arriving at 200 s of a 600 s budget are used, and judge the claim", async () => { + const ws = workspace(); + try { + const { engine, journal } = setup(ws.dir, "late-wait-1"); + const { ev, simMs } = await onVirtualClock((at) => + engine.run(TASK, allowAll, { + pendingCriteria: at({ taskChecks: [slow("greeting", "grep -qx hello out.txt")], taskNotes: [] }, 200_000), + criteriaSoFar: nothing, + criteriaWaitMs: 100, + draftInputs: sealed, + }), + ); + assert.ok(infos(ev).some((t) => /^waited (19\d|20\d)s for the drafted checks \(waited to the time budget's margin.*1 arrived and joined/.test(t)), infos(ev).join("\n")); + const proof = ev.find((e) => e.kind === "proof_result"); + assert.ok(proof && proof.kind === "proof_result" && proof.result.results.some((r: { name: string }) => r.name === "task:greeting")); + assert.equal(outcome(ev), "verified"); + assert.ok(simMs < 600_000); + const join = notes(journal, "late-checks"); + assert.equal(join.length, 1); + assert.equal(join[0]!.data.arrived, true); + assert.ok(Number(join[0]!.data.waitedMs) >= 195_000); + assert.equal(notes(journal, "draft-inputs")[0]!.data.inputsSha, "frozen-inputs"); + assert.equal(join[0]!.data.inputsSha, "frozen-inputs"); + } finally { + ws.cleanup(); + } + }); + + it("a late check that fails the work goes down the refusal path, never dropped", async () => { + const ws = workspace(); + try { + const { engine } = setup(ws.dir, "late-wait-2"); + const { ev } = await onVirtualClock((at) => + engine.run(TASK, allowAll, { + pendingCriteria: at({ taskChecks: [slow("farewell", "grep -qx goodbye out.txt")], taskNotes: [] }, 200_000), + criteriaSoFar: nothing, + criteriaWaitMs: 100, + draftInputs: sealed, + }), + ); + assert.ok(infos(ev).some((t) => /1 arrived and joined/.test(t))); + assert.ok(ev.some((e) => (e.kind === "proof_refused" || e.kind === "proof_exhausted") && e.result.results.some((r: { name: string; ok: boolean }) => r.name === "task:farewell" && !r.ok))); + assert.notEqual(outcome(ev), "verified"); + } finally { + ws.cleanup(); + } + }); + + it("drafts that never arrive: the claim is judged without them at the deadline margin", async () => { + const ws = workspace(); + try { + const { engine, journal } = setup(ws.dir, "late-wait-3"); + const { ev, simMs } = await onVirtualClock(() => + engine.run(TASK, allowAll, { + pendingCriteria: new Promise(() => {}), + criteriaSoFar: nothing, + criteriaWaitMs: 100, + draftInputs: sealed, + }), + ); + const said = infos(ev).find((t) => /did not arrive/.test(t)); + assert.ok(said, infos(ev).join("\n")); + // 600 s budget, 90 s margin: about 510 s waited, then judged in what is left. + const waited = Number(/^waited (\d+)s/.exec(said!)?.[1]); + assert.ok(waited >= 500 && waited <= 512, said); + assert.match(said!, /judged without them/); + assert.notEqual(outcome(ev), "verified"); + assert.ok(ev.some((e) => e.kind === "job_end")); + assert.ok(simMs < 600_000, `ended inside the budget (${simMs} ms)`); + assert.equal(notes(journal, "late-checks")[0]!.data.arrived, false); + } finally { + ws.cleanup(); + } + }); + + it("with no time budget the old 120 s applies", async () => { + const ws = workspace(); + try { + const { engine } = setup(ws.dir, "late-wait-4"); + engine.setTurnDeadline(0); + const { ev } = await onVirtualClock((at) => + engine.run(TASK, allowAll, { + pendingCriteria: at({ taskChecks: [slow("greeting", "grep -qx hello out.txt")], taskNotes: [] }, 200_000), + criteriaSoFar: nothing, + criteriaWaitMs: 100, + draftInputs: sealed, + }), + ); + assert.ok(infos(ev).some((t) => /^waited 12\ds for the drafted checks; they did not arrive within 120s/.test(t)), infos(ev).join("\n")); + assert.notEqual(outcome(ev), "verified"); + } finally { + ws.cleanup(); + } + }); + + it("drafts whose inputs are not the frozen snapshot get the old 120 s, not the long wait", async () => { + const ws = workspace(); + try { + const { engine } = setup(ws.dir, "late-wait-5"); + const { ev } = await onVirtualClock((at) => + engine.run(TASK, allowAll, { + pendingCriteria: at({ taskChecks: [slow("greeting", "grep -qx hello out.txt")], taskNotes: [] }, 200_000), + criteriaSoFar: nothing, + criteriaWaitMs: 100, + draftInputs: { sha: "frozen-inputs", used: () => ["frozen-inputs", "read-the-live-folder"] }, + }), + ); + assert.ok(infos(ev).some((t) => /^waited 12\ds for the drafted checks; they did not arrive within 120s/.test(t)), infos(ev).join("\n")); + assert.notEqual(outcome(ev), "verified"); + } finally { + ws.cleanup(); + } + }); + + it("drafts that arrive having used other inputs are not joined, and the claim is not verified by them", async () => { + const ws = workspace(); + try { + const { engine, journal } = setup(ws.dir, "late-wait-6"); + const used = ["frozen-inputs"]; + const { ev } = await onVirtualClock((at) => + engine.run(TASK, allowAll, { + // A later stage read the changed folder; that shows once it is done. + pendingCriteria: at(null, 150_000).then(() => { + used.push("read-the-live-folder"); + return { taskChecks: [slow("greeting", "grep -qx hello out.txt")], taskNotes: [] }; + }), + criteriaSoFar: nothing, + criteriaWaitMs: 100, + draftInputs: { sha: "frozen-inputs", used: () => [...used] }, + }), + ); + assert.ok(infos(ev).some((t) => /arrived after \d+s but were drafted from a view of the project that differs/.test(t)), infos(ev).join("\n")); + assert.ok(!ev.some((e) => e.kind === "proof_result" && e.result.results.some((r: { name: string }) => r.name === "task:greeting"))); + assert.notEqual(outcome(ev), "verified"); + const n = notes(journal, "late-checks"); + assert.equal(n.length, 1); + assert.match(String(n[0]!.data.text), /NOT joined/); + } finally { + ws.cleanup(); + } + }); +}); From 4118f293c3f9ec9e05e02b4808e34f19a16bd02d Mon Sep 17 00:00:00 2001 From: Tyler Skelton Date: Wed, 7 Oct 2026 14:12:35 -0700 Subject: [PATCH 2/2] CI: run on PRs into claude/** too; keep the loop alive in the ask-timeout test On Node 22 AbortSignal.timeout's timer does not hold the event loop open and the mock stream has no socket, so the loop emptied before the 30 ms timeout and the runner cancelled the rest of local-ask.test.ts. That is why check was red on every PR in the stack, #23 included. --- .github/workflows/check.yml | 2 ++ test/local-ask.test.ts | 12 +++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index c52ac12..cbd429e 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -3,7 +3,9 @@ name: check on: push: branches: [main] + # Stacked work is reviewed as PRs into claude/* branches, not only main. pull_request: + branches: [main, "claude/**"] jobs: typecheck-and-test: diff --git a/test/local-ask.test.ts b/test/local-ask.test.ts index 84b1529..c49b75f 100644 --- a/test/local-ask.test.ts +++ b/test/local-ask.test.ts @@ -88,7 +88,17 @@ describe("an ask to a self-hosted server", () => { }); return new Response(body, { headers: { "content-type": "text/event-stream" } }); }) as unknown as typeof fetch; - const r = await askModel({ baseUrl: LOCAL, model: "m", system: "s", prompt: "p", fetchFn, timeoutMs: 30 }); + // AbortSignal.timeout's timer does not hold the event loop open, and a + // mock stream has no socket that would. On Node 22 the loop emptied before + // the 30 ms were up and the test runner cancelled the rest of the file. + // A real request's socket keeps the process alive; this stands in for it. + const keepAlive = setInterval(() => {}, 1000); + let r: Awaited>; + try { + r = await askModel({ baseUrl: LOCAL, model: "m", system: "s", prompt: "p", fetchFn, timeoutMs: 30 }); + } finally { + clearInterval(keepAlive); + } assert.equal(aborted, true); assert.match(!r.ok ? r.error : "", /did not answer within/); });