diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a2f8fd..3b3b646 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -147,6 +147,22 @@ background logs, and paths the task names). bash is not blocked; this is a budget control, not isolation. +### Fixed + +- **Two lean-session defects (lean-sessions study, PR #50).** When the kept + user turns are mostly Maat's notes, a shed cut on user turns must now free + at least 25% of the history, or it cuts on recent messages instead: Maat's own notes (acceptance criteria, bar refusals) are + user messages, so a shed could cost the whole prompt cache for 0.7% + (60,788 -> 60,387 tokens), or come back empty on every step so auto-shed + never fired again that turn. Superseded tool results are now elided per + call, not per step: rewriting `dur.py` no longer drops `test_dur.py` read in + the same step. A bash rerun that came back different supersedes the earlier + output (a rerun with different options, such as a longer `timeout_s`, is a + different call), and a plain `cat`/`head`/`tail`/`nl`/`sed -n` of one file + counts as a read a later write makes stale. A shed that cuts on recent + messages keeps the acceptance criteria and the live bar refusal verbatim. + See `docs/lean.md`. + ## 0.2.2 — subscription backends removed ### Removed diff --git a/docs/lean.md b/docs/lean.md index f7a8667..7246761 100644 --- a/docs/lean.md +++ b/docs/lean.md @@ -49,6 +49,36 @@ To see them: `npx tsc -p tsconfig.test.json && node --test dist-test/test/lean-b - The numbers are deterministic (scripted turns, no clock in the bodies), so 1.5x leaves room for small prompt edits, not for noise. +## Shed and elision rules the study fixed + +From the lean-sessions study (PR #50); tests in `test/lean-fixes.test.ts`. + +- **Shed min-free** (`SHED_MIN_FREE` = 0.25 in `src/transcript.ts`). When + the user turns a shed would keep are mostly Maat's own notes (acceptance + criteria, bar refusals, nudges, all tagged in `molt`), the cut on user turns + must free at least a quarter of the history, or `planShed` cuts on recent + messages instead. A shed costs the whole prompt cache, and those notes are + user messages, so the user-turn cut could land near the start of a long + turn (a real run: 60,788 -> 60,387 tokens) or return no plan at all, step + after step, so auto-shed never fired again. An interactive session whose + kept turns are the person's own is cut on user turns as before. +- **Criteria and refusal survive a recent-message cut.** On a cut on recent + messages, the latest criteria note (`molt.criteria`) and the live bar + refusal (the latest non-stale `molt.barFailure`) stay verbatim right after + the digest instead of being excerpted to 300 characters in it. +- **Per-call elision.** `elideSupersededReads` elides the superseded call's + own result, not every result of its step. A bash rerun with new output + supersedes the earlier output; the same command with different options + (`timeout_s`, compared with keys sorted) is a different call, and a + `background: true` run is never superseded (its result is the job handle). + A plain `cat`/`head`/`tail`/`nl`/`sed -n` of one file (no pipe, redirect or + second command) is a read that a later write of that file makes stale; a + bash rerun answered with a "same call" pointer is not a new read, so the + copy it points at stays the one a write invalidates. + +The flag-gated experiments from the study (shed threshold, ageing) are not +part of this. + ## The bench cost alarm (`bench/local/run.py`) After each run, run.py reads the run's cost (`spend.costUsd`) and prompt diff --git a/src/engine.ts b/src/engine.ts index 5816edb..513109a 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -6062,7 +6062,7 @@ export class Engine { "never reported as verified — do not describe one as passing. You cannot", "edit these; attempting to is itself a failure.", ]; - this.transcript.push({ role: "user", content: lines.join("\n") }); + this.transcript.push({ role: "user", content: lines.join("\n"), molt: { criteria: true } }); } }; if (!pendingCriteria) yield* sealCriteria(opts.taskChecks ?? [], opts.taskNotes ?? []); diff --git a/src/transcript.ts b/src/transcript.ts index 9fef4a9..b52f25c 100644 --- a/src/transcript.ts +++ b/src/transcript.ts @@ -24,6 +24,17 @@ import { estTokens, type Bom, type Msg } from "./types.js"; export const STALE_FAILURE_PREFIX = "[molt: superseded]"; export const ELIDED_PREFIX = "[molt: superseded tool result —"; +/** How the engine sends a repeated call whose result has not changed (see engine.ts). */ +const SAME_CALL_PREFIX = "[molt: this is the same "; +/** How the engine sends a read of lines the model has already been shown. */ +const SHOWN_PREFIX = "[molt: you have already been shown"; + +/** + * A bash command that only prints one file: `cat f`, `head -n 40 f`, + * `sed -n '1,80p' f`, `nl f`, `tail f`. Anything with a pipe, a redirect or a + * second command is not a plain read and is left alone. + */ +const SIMPLE_READ = /^(?:cat|nl(?: -\S+)*|head(?: -\S+(?: \d+)?)*|tail(?: -\S+(?: \d+)?)*|sed -n ['"]?[\d,$p]+['"]?)\s+([^\s|;&<>$`*?]+)$/; /** * How many steps an elision has to pay for itself in, when a cache is working. @@ -48,6 +59,12 @@ const MAX_ACTION_LINES = 25; * turns to cut on. Fall back to keeping this many recent messages. */ const KEEP_RECENT_MESSAGES = 6; +/** + * The least share of the history a shed cut on user turns must free, or the + * shed cuts on recent messages instead (see planShed). A shed costs the whole + * prompt cache, so one that frees almost nothing is worse than none. + */ +export const SHED_MIN_FREE = 0.25; /** Never bother shedding fewer than this many messages. */ const MIN_DROPPED = 2; @@ -59,6 +76,14 @@ export type ShedPlan = { /** Messages being removed from the working context. */ dropped: Msg[]; droppedCount: number; + /** + * Messages from before the cut that stay, verbatim, right after the digest: + * the acceptance criteria and the live bar refusal (see planShed). Empty + * on a plain cut. + */ + carried: Msg[]; + /** Where the cut is: everything before it is either dropped or carried. */ + cutAt: number; beforeTokens: number; afterTokens: number; }; @@ -70,6 +95,8 @@ export class Transcript { private archived: Msg[][] = []; /** What this turn is for. Sent every request, shed never. */ private task: string | null = null; + /** See SHED_MIN_FREE. 0 restores the plain cut on user turns. */ + shedMinFree = SHED_MIN_FREE; constructor(systemPrompt: string) { this.system = { role: "system", content: systemPrompt }; @@ -218,7 +245,7 @@ export class Transcript { bom(toolSchemaJson: string, session: { prompt: number; completion: number }): Bom { const historyTokens = this.working.reduce( - (n, m) => n + estTokens(m.content ?? "") + estTokens(JSON.stringify(m.tool_calls ?? "")), + (n, m) => n + msgTokens(m), 0, ); // The standing note is part of every request, so it is part of the fixed @@ -238,7 +265,7 @@ export class Transcript { historyTokens(): number { return this.working.reduce( - (n, m) => n + estTokens(m.content ?? "") + estTokens(JSON.stringify(m.tool_calls ?? "")), + (n, m) => n + msgTokens(m), 0, ); } @@ -271,8 +298,34 @@ export class Transcript { .filter((i) => i >= 0); let cutAt: number; + /** The cut is on recent messages, not on a user turn. */ + let recentCut = false; if (userIdxs.length > keepExchanges) { cutAt = userIdxs[userIdxs.length - keepExchanges]; + // A cut on user turns must buy real headroom. Maat's own notes (the + // acceptance criteria, a bar refusal) arrive as user messages, so a + // single long request has three "exchanges" with nearly all of its + // history after the second. A real run shed 6 messages, 60,788 -> + // 60,387 tokens, threw away its whole prompt cache for 0.7%, and shed + // again one step later. With a short first exchange the digest cost + // more than the cut freed, the plan came back null on every step, and + // auto-shed never fired again that turn. Either way, cut on recent + // messages instead, as a turn with no user turn to cut on does. + // + // Only for that shape, though: the user turns kept must be mostly + // Maat's notes. In an interactive session whose last two real + // exchanges hold most of the history, those exchanges are what the + // person is working on, and the cut stays on them as it always did. + const kept = this.working.slice(cutAt).filter((m) => m.role === "user"); + const notes = kept.filter(isMaatNote).length; + const freed = this.working.slice(0, cutAt).reduce((n, m) => n + msgTokens(m), 0); + if (notes > kept.length - notes && freed < this.shedMinFree * this.historyTokens()) { + const fallback = this.findSafeCut(this.working.length - Math.max(2, keepRecent)); + if (fallback !== null && fallback > cutAt) { + cutAt = fallback; + recentCut = true; + } + } } else { // A single request can produce dozens of tool calls with no user turn // to cut on — which is exactly when context runs out. Fall back to @@ -286,12 +339,30 @@ export class Transcript { const fallback = this.findSafeCut(this.working.length - Math.max(2, keepRecent)); if (fallback === null) return null; cutAt = fallback; + recentCut = true; } - const dropped = this.working.slice(0, cutAt); - const kept = this.working.slice(cutAt); + // The acceptance criteria and the live bar refusal are what the model + // needs most right after a refusal, and a digest caps each message at + // EXCERPT_CHARS. A cut on recent messages can land after both, and with + // sheds every few steps the refusal's detail would last about three. So + // on such a cut the latest of each, when it is before the cut, stays + // verbatim. A cut on user turns keeps whole exchanges and is unchanged. + const lastIdx = (f: (m: Msg) => boolean) => { + for (let i = this.working.length - 1; i >= 0; i--) if (f(this.working[i]!)) return i; + return -1; + }; + const carryIdx = new Set( + [ + lastIdx((m) => m.molt?.criteria === true), + lastIdx((m) => m.molt?.barFailure === true && !!m.content && !m.content.startsWith(STALE_FAILURE_PREFIX)), + ].filter((i) => recentCut && i >= 0 && i < cutAt), + ); + const carried = this.working.slice(0, cutAt).filter((_, i) => carryIdx.has(i)); + const dropped = this.working.slice(0, cutAt).filter((_, i) => !carryIdx.has(i)); + const kept = [...carried, ...this.working.slice(cutAt)]; if (dropped.length < MIN_DROPPED || dropped.every(isDigest)) return null; - if (kept.length > 0 && kept[0].role === "tool") return null; + if (this.working[cutAt]?.role === "tool") return null; const beforeTokens = this.historyTokens(); const digest = buildDigest(dropped); @@ -303,7 +374,7 @@ export class Transcript { molt: { digest: true }, }; const afterTokens = [digestMsg, ...kept].reduce( - (n, m) => n + estTokens(m.content ?? "") + estTokens(JSON.stringify(m.tool_calls ?? "")), + (n, m) => n + msgTokens(m), 0, ); @@ -311,7 +382,7 @@ export class Transcript { // more than the messages it replaces. if (afterTokens >= beforeTokens) return null; - return { exuvia, digest, dropped, droppedCount: dropped.length, beforeTokens, afterTokens }; + return { exuvia, digest, dropped, droppedCount: dropped.length, carried, cutAt, beforeTokens, afterTokens }; } /** @@ -333,12 +404,11 @@ export class Transcript { * has been durably archived — that ordering is the guarantee. */ commitShed(plan: ShedPlan): void { - const cut = plan.droppedCount; - const dropped = this.working.slice(0, cut); - this.archived.push(dropped); + this.archived.push(plan.dropped); this.working = [ { role: "system", content: plan.digest, molt: { digest: true } }, - ...this.working.slice(cut), + ...plan.carried, + ...this.working.slice(plan.cutAt), ]; } @@ -456,7 +526,26 @@ export class Transcript { elideSupersededReads( opts: { protectCache?: boolean } = {}, ): { elided: number; tokensSaved: number; deferred: number } { + /** + * What is superseded is a CALL, not the step it was made in: result + * index -> why. Keyed by the assistant message, a write to one file + * elided every result of the step that read it. A real run read dur.py + * and test_dur.py in one step, rewrote dur.py, and lost test_dur.py with + * it, under a marker saying the current copy was further down. It was not. + */ const supersededBy = new Map(); + /** A call's place in the conversation: the index of its result, or -1. */ + type At = number; + const resultAt = (i: number, k: number, id: string | undefined): At => { + for (let j = i + 1; j < this.working.length && this.working[j].role === "tool"; j++) { + const r = this.working[j]; + if (id ? r.tool_call_id === id : j === i + 1 + k) return j; + } + return -1; + }; + const mark = (at: At, why: string) => { + if (at >= 0) supersededBy.set(at, why); + }; /** * Reads still worth keeping, keyed by the exact window they returned. * @@ -468,90 +557,146 @@ export class Transcript { * it back to read the same file again, forever. Two features that were * each correct alone. */ - const lastRead = new Map(); + const lastRead = new Map(); /** Every live read of a path, so a write can invalidate all of them. */ const readsOf = new Map(); + const noteRead = (path: string, window: string, at: At) => { + lastRead.set(window, at); + const windows = readsOf.get(path) ?? []; + if (!windows.includes(window)) windows.push(window); + readsOf.set(path, windows); + }; + /** The last live run of each bash call (command and options), by result index. */ + const lastRun = new Map(); for (let i = 0; i < this.working.length; i++) { const m = this.working[i]; - for (const call of m.tool_calls ?? []) { + (m.tool_calls ?? []).forEach((call, k) => { let args: Record = {}; try { args = JSON.parse(call.function.arguments || "{}") as Record; } catch { - continue; + return; + } + const at = resultAt(i, k, call.id); + + if (call.function.name === "bash" && typeof args.command === "string") { + const cmd = args.command.replace(/\s+/g, " ").trim(); + // The same call is the same command with the same options. A rerun + // with a longer timeout_s, or in the background, is a different + // call, and the earlier result (a timeout and how to avoid it) is + // the reason for it. + // Keys sorted: {timeout_s, command} and {command, timeout_s} are one call. + const key = JSON.stringify( + Object.entries({ ...args, command: cmd }).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)), + ); + const now = at >= 0 ? (this.working[at].content ?? "") : ""; + const pointer = now.startsWith(SAME_CALL_PREFIX); + // A rerun that came back the same is sent as a pointer to the + // earlier copy, which must then stay. One that came back different + // makes the earlier output history. + // + // A background run is left out: its result names a job that may + // still be running, and it is the model's handle for stop_job. + // A `commands: [...]` call has no `command` and never gets here, so + // it takes no part in rerun supersession either. + if (!args.background) { + const prior = lastRun.get(key); + if (prior !== undefined && at >= 0 && !pointer && now !== (this.working[prior]?.content ?? "")) + mark(prior, `rerun at step ${i}`); + if (!pointer && at >= 0) lastRun.set(key, at); + } + // A plain read of one file through bash is a read of that file: a + // later write makes it stale exactly as it does a read_file. A + // pointer to an earlier copy is not a read: the copy it points at + // is the one a write must invalidate. + const read = SIMPLE_READ.exec(cmd); + if (read && !pointer) noteRead(read[1]!.replace(/^\.\//, ""), `bash:${cmd}`, at); + return; } - const path = String(args.path ?? ""); - if (!path) continue; + + const path = String(args.path ?? "").replace(/^\.\//, ""); + if (!path) return; if (call.function.name === "read_file") { + // A repeat the engine answered with a pointer ("already shown", + // "the same call") points at the earlier copy: that copy must stay, + // and stays the one a later write invalidates. + const got = at >= 0 ? (this.working[at].content ?? "") : ""; + if (got.startsWith(SAME_CALL_PREFIX) || got.startsWith(SHOWN_PREFIX)) return; // Identical arguments return identical bytes; anything else is a // different part of the file and stands on its own. const window = `${path}@${Number(args.offset ?? 0)}+${String(args.limit ?? "all")}`; const prior = lastRead.get(window); - if (prior !== undefined) supersededBy.set(prior, `re-read at step ${i}`); - lastRead.set(window, i); - const windows = readsOf.get(path) ?? []; - if (!windows.includes(window)) windows.push(window); - readsOf.set(path, windows); + if (prior !== undefined) mark(prior, `re-read at step ${i}`); + noteRead(path, window, at); } else if (call.function.name === "write_file" || call.function.name === "edit_file") { // A change to the file invalidates every part of it that was read, // whichever window it came from: what is in context is no longer // what is on disk. for (const window of readsOf.get(path) ?? []) { const prior = lastRead.get(window); - if (prior !== undefined) supersededBy.set(prior, `changed at step ${i}`); + if (prior !== undefined) mark(prior, `changed at step ${i}`); lastRead.delete(window); } readsOf.delete(path); } - } + }); } let elided = 0; let tokensSaved = 0; let deferred = 0; - for (const [callIdx, reason] of supersededBy) { - // The tool result follows its assistant turn. - for (let j = callIdx + 1; j < this.working.length; j++) { - const m = this.working[j]; - if (m.role !== "tool") break; - if (!m.content || m.content.startsWith(ELIDED_PREFIX)) continue; - const before = estTokens(m.content); - // Wording matters here. "Full contents remain in the archived record" - // reads, to a model, as an invitation to go and get them — which it - // can only do by re-reading the file, which is what elided this copy - // in the first place. Point at the newer copy instead. - const marker = - `${ELIDED_PREFIX} ${reason}. The current contents are further down this ` + + for (const [j, reason] of supersededBy) { + const m = this.working[j]; + if (!m || m.role !== "tool") continue; + if (!m.content || m.content.startsWith(ELIDED_PREFIX)) continue; + const before = estTokens(m.content); + // Wording matters here. "Full contents remain in the archived record" + // reads, to a model, as an invitation to go and get them — which it + // can only do by re-reading the file, which is what elided this copy + // in the first place. Point at the newer copy instead. + const marker = reason.startsWith("rerun") + ? `${ELIDED_PREFIX} ${reason}. The newer output of the same command is further ` + + `down this conversation.` + : `${ELIDED_PREFIX} ${reason}. The current contents are further down this ` + `conversation; do not read the file again to recover this.`; - // A short result costs less than the notice explaining its absence. - // Eliding it would drop content AND grow the context — which is how - // the meter came to report "−-17 tokens" saved. - if (estTokens(marker) >= before) continue; - const saving = before - estTokens(marker); - // What this edit strands: everything after it shares a prefix that is - // about to change, so the next request pays full price for all of it. - // Elide only if the saving earns that back inside ELISION_PAYBACK_STEPS. - if (opts.protectCache) { - let stranded = 0; - for (let k = j + 1; k < this.working.length; k++) { - stranded += estTokens(this.working[k].content ?? ""); - } - if (saving * ELISION_PAYBACK_STEPS < stranded) { - deferred++; - continue; - } + // A short result costs less than the notice explaining its absence. + // Eliding it would drop content AND grow the context — which is how + // the meter came to report "−-17 tokens" saved. + if (estTokens(marker) >= before) continue; + const saving = before - estTokens(marker); + // What this edit strands: everything after it shares a prefix that is + // about to change, so the next request pays full price for all of it. + // Elide only if the saving earns that back inside ELISION_PAYBACK_STEPS. + if (opts.protectCache) { + let stranded = 0; + for (let k = j + 1; k < this.working.length; k++) { + stranded += estTokens(this.working[k].content ?? ""); + } + if (saving * ELISION_PAYBACK_STEPS < stranded) { + deferred++; + continue; } - m.content = marker; - tokensSaved += saving; - elided++; } + m.content = marker; + tokensSaved += saving; + elided++; } return { elided, tokensSaved, deferred }; } } +/** A user message Maat wrote itself: the criteria, a bar refusal, a nudge. */ +function isMaatNote(m: Msg): boolean { + return m.molt?.criteria === true || m.molt?.barFailure === true || m.molt?.nudge === true; +} + +/** What a message costs in the history, in molt's token units. */ +function msgTokens(m: Msg): number { + return estTokens(m.content ?? "") + estTokens(JSON.stringify(m.tool_calls ?? "")); +} + /** * A digest is verbatim excerpts, never a paraphrase. Prior digests are * carried forward whole rather than re-excerpted — re-truncating a diff --git a/src/types.ts b/src/types.ts index 396bbba..67745f8 100644 --- a/src/types.ts +++ b/src/types.ts @@ -41,6 +41,11 @@ export type MsgMeta = { regrown?: true; /** True for a bar-failure notice injected by the proof loop. */ barFailure?: true; + /** + * True for the acceptance-criteria note. A shed that cuts on recent + * messages keeps the latest one verbatim (transcript.ts planShed). + */ + criteria?: true; /** * True for a note molt wrote to the model about the model's own behaviour * — an empty turn, or a step that only repeated itself. Distinct from a diff --git a/test/lean-budget.test.ts b/test/lean-budget.test.ts index c607a74..55a1b24 100644 --- a/test/lean-budget.test.ts +++ b/test/lean-budget.test.ts @@ -127,7 +127,10 @@ function within(name: string, m: Measured, c: Ceiling): void { const CEILINGS = { small: { largest: 11_700, total: 34_000, steps: 5 }, // 7,805 / 22,690 / 3 bigRead: { largest: 62_000, total: 195_000, steps: 6 }, // 41,234 / 130,307 / 4 - reread: { largest: 61_000, total: 173_000, steps: 14 }, // 40,732 / 115,399 / 9 + // 2026-10-08: was 40,732 / 115,399 / 9, but only because the original copy was + // elided under its own "already shown" pointer, leaving the model no copy at + // all (Grok #51 item 3). One kept 40k copy per step is the honest cost. + reread: { largest: 61_000, total: 420_000, steps: 14 }, // 44,725 / 348,643 / 9 longShed: { largest: 390_000, total: 10_900_000, steps: 92 }, // 259,597 / 7,262,564 / 61 malformedRaw: { largest: 18_700, total: 104_000, steps: 11 }, // 12,450 / 69,197 / 7 malformedAct: { largest: 14_100, total: 73_000, steps: 11 }, // 9,384 / 48,455 / 7 diff --git a/test/lean-fixes.test.ts b/test/lean-fixes.test.ts new file mode 100644 index 0000000..622ad00 --- /dev/null +++ b/test/lean-fixes.test.ts @@ -0,0 +1,294 @@ +/** + * Two defects found by the lean-sessions study (PR #50), fixed as defaults. + * Each test is built from the real case. + * + * 1. Shed min-free. Maat's own notes (acceptance criteria, a bar refusal) are + * user messages, so the cut on user turns could land near the start of a + * long single-request turn: the duration-bug run shed 6 messages, 60,788 -> + * 60,387 tokens (0.7%), lost its whole prompt cache, and shed again a step + * later. With a short first exchange the plan came back null on every step + * and auto-shed never fired again that turn. + * 2. Per-call elision. Elision was keyed by step: a step that read dur.py and + * test_dur.py together lost test_dur.py when dur.py was rewritten, under a + * marker saying the current copy was further down. It was not. + */ +import assert from "node:assert/strict"; +import { readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, it } from "node:test"; +import { Engine } from "../src/engine.js"; +import { ELIDED_PREFIX, SHED_MIN_FREE, Transcript } from "../src/transcript.js"; +import { allowAll, drain, scriptedProvider, toolCall, workspace } from "./helpers.js"; + +describe("shed min-free (default)", () => { + /** + * The shape of the duration-bug run: the request and a small first + * exchange (six messages), Maat's acceptance-criteria note, a long tool + * run, a bar refusal, another long tool run. Two exchanges kept puts the + * cut on the criteria note. `"short"` is the tiny first exchange (one + * short read) that makes the digest cost more than the cut frees. + */ + function durationBugShape(firstRead = "f".repeat(800)): Transcript { + const t = new Transcript("SYSTEM"); + t.push({ role: "user", content: "Fix the duration parser so `1h30m` parses." }); + t.push({ role: "assistant", content: null, tool_calls: [toolCall("read_file", { path: "dur.py" }, "r0")] }); + t.push({ role: "tool", tool_call_id: "r0", content: firstRead }); + if (firstRead !== "short") { + t.push({ role: "assistant", content: null, tool_calls: [toolCall("read_file", { path: "test_dur.py" }, "r1")] }); + t.push({ role: "tool", tool_call_id: "r1", content: firstRead }); + t.push({ role: "assistant", content: "Reading done." }); + } + t.push({ role: "user", content: "Acceptance criteria for this task: ...", molt: { criteria: true } }); + for (let i = 0; i < 40; i++) { + t.push({ role: "assistant", content: null, tool_calls: [toolCall("bash", { command: `s${i}` }, `a${i}`)] }); + t.push({ role: "tool", tool_call_id: `a${i}`, content: `${i}:${"o".repeat(3000)}` }); + } + t.pushBarFailure("[molt] You indicated the task is complete, but 1 of 3 checks did not pass."); + for (let i = 0; i < 40; i++) { + t.push({ role: "assistant", content: null, tool_calls: [toolCall("bash", { command: `t${i}` }, `b${i}`)] }); + t.push({ role: "tool", tool_call_id: `b${i}`, content: `${i}:${"p".repeat(3000)}` }); + } + return t; + } + + it("is on by default at a quarter of the history", () => { + assert.equal(SHED_MIN_FREE, 0.25); + assert.equal(new Transcript("S").shedMinFree, SHED_MIN_FREE); + }); + + it("the 0.7% shed: without min-free the cut drops 6 messages and frees under 1%", () => { + const t = durationBugShape(); + t.shedMinFree = 0; + const plan = t.planShed()!; + assert.equal(plan.droppedCount, 6); + assert.ok(plan.afterTokens / plan.beforeTokens > 0.99, `${plan.beforeTokens} -> ${plan.afterTokens}`); + }); + + it("the 0.7% shed: by default it is a real shed", () => { + const plan = durationBugShape().planShed()!; + assert.ok(plan.afterTokens / plan.beforeTokens < 0.75, `${plan.beforeTokens} -> ${plan.afterTokens}`); + assert.ok(plan.afterTokens / plan.beforeTokens < 0.1, "it cuts on recent messages, as a single request does"); + }); + + it("the empty plan: without min-free a short first exchange gives no shed at all", () => { + const t = durationBugShape("short"); + t.shedMinFree = 0; + assert.equal(t.planShed(), null); + }); + + it("the empty plan: by default the shed happens, and keeps happening as the turn goes on", () => { + const t = durationBugShape("short"); + const auto = 20_000; + let sheds = 0; + let peak = 0; + for (let i = 0; i < 60; i++) { + // The engine's auto-shed, once per step. + if (t.historyTokens() > auto) { + const plan = t.planShed(); + assert.ok(plan, `no shed at step ${i} with ${t.historyTokens()} tokens of history`); + t.commitShed(plan); + sheds++; + } + peak = Math.max(peak, t.historyTokens()); + t.push({ role: "assistant", content: null, tool_calls: [toolCall("bash", { command: `u${i}` }, `c${i}`)] }); + t.push({ role: "tool", tool_call_id: `c${i}`, content: `${i}:${"q".repeat(3000)}` }); + } + assert.ok(sheds >= 3, `only ${sheds} shed(s)`); + assert.ok(peak < auto * 1.5, `history reached ${peak}`); + }); + + it("leaves a cut on user turns alone when it frees enough", () => { + const t = new Transcript("S"); + for (let k = 0; k < 4; k++) { + t.push({ role: "user", content: `ask ${k}` }); + for (let i = 0; i < 5; i++) { + t.push({ role: "assistant", content: null, tool_calls: [toolCall("bash", { command: `${k}.${i}` }, `c${k}.${i}`)] }); + t.push({ role: "tool", tool_call_id: `c${k}.${i}`, content: "x".repeat(2000) }); + } + } + const plan = t.planShed()!; + // Two of four exchanges go: the cut is the user turn, not the recent fallback. + assert.equal(plan.droppedCount, 22); + }); + + it("keeps the criteria and the live bar refusal verbatim through a fallback shed (review probe)", () => { + const t = new Transcript("SYSTEM"); + t.push({ role: "user", content: "Fix the duration parser so `1h30m` parses." }); + t.push({ role: "assistant", content: null, tool_calls: [toolCall("read_file", { path: "dur.py" }, "r0")] }); + t.push({ role: "tool", tool_call_id: "r0", content: "f".repeat(800) }); + const criteria = `Acceptance criteria for this task: ${"c".repeat(560)} CRITERIA_END`; + t.push({ role: "user", content: criteria, molt: { criteria: true } }); + for (let i = 0; i < 40; i++) { + t.push({ role: "assistant", content: null, tool_calls: [toolCall("bash", { command: `s${i}` }, `a${i}`)] }); + t.push({ role: "tool", tool_call_id: `a${i}`, content: `${i}:${"o".repeat(3000)}` }); + } + const failure = `[molt] attempt 1 was refused: ${"e".repeat(1150)} FAILURE_END`; + t.pushBarFailure(failure); + const steps = (from: number, n: number) => { + for (let i = from; i < from + n; i++) { + t.push({ role: "assistant", content: null, tool_calls: [toolCall("bash", { command: `t${i}` }, `b${i}`)] }); + t.push({ role: "tool", tool_call_id: `b${i}`, content: `${i}:${"p".repeat(3000)}` }); + } + }; + steps(0, 4); + const plan = t.planShed()!; + assert.ok(plan.afterTokens / plan.beforeTokens < 0.5, `${plan.beforeTokens} -> ${plan.afterTokens}`); + assert.equal(plan.carried.length, 2); + t.commitShed(plan); + const onWire = () => t.wire().map((m) => m.content ?? ""); + assert.ok(onWire().includes(failure), "the refusal's detail was digested"); + assert.ok(onWire().includes(criteria), "the criteria were digested"); + // And the next shed, a few steps on, keeps them too. + steps(4, 10); + const again = t.planShed()!; + t.commitShed(again); + assert.ok(onWire().includes(failure) && onWire().includes(criteria), "lost on the second shed"); + // A stale refusal is not carried: the next one replaces it. + t.pushBarFailure("[molt] attempt 2 was refused: SECOND"); + steps(14, 10); + t.commitShed(t.planShed()!); + assert.ok(!onWire().includes(failure)); + assert.ok(onWire().some((c) => c.includes("SECOND")) && onWire().includes(criteria)); + }); + + it("an interactive session whose last two exchanges hold most of the history is unchanged", () => { + const t = new Transcript("S"); + for (let k = 0; k < 3; k++) { + t.push({ role: "user", content: `ask ${k}` }); + const n = k === 0 ? 1 : 15; + for (let i = 0; i < n; i++) { + t.push({ role: "assistant", content: null, tool_calls: [toolCall("bash", { command: `${k}.${i}` }, `c${k}.${i}`)] }); + t.push({ role: "tool", tool_call_id: `c${k}.${i}`, content: "x".repeat(2000) }); + } + } + const plan = t.planShed()!; + // The first exchange alone goes, though it frees well under 25%: the + // two kept exchanges are the person's, not Maat's notes. + assert.ok(plan.beforeTokens - plan.afterTokens < 0.25 * plan.beforeTokens); + assert.equal(plan.cutAt, 3); + assert.equal(plan.droppedCount, 3); + assert.equal(plan.carried.length, 0); + }); +}); + +describe("per-call elision (default)", () => { + function step(t: Transcript, calls: [string, Record, string, string][]) { + t.push({ role: "assistant", content: null, tool_calls: calls.map(([n, a, id]) => toolCall(n, a, id)) }); + for (const [, , id, out] of calls) t.push({ role: "tool", tool_call_id: id, content: out }); + } + const body = (s: string) => `${s}\n${"x".repeat(2000)}`; + + it("dur.py / test_dur.py: a write elides the read of that file only, not the rest of its step", () => { + const t = new Transcript("S"); + t.push({ role: "user", content: "task" }); + step(t, [ + ["read_file", { path: "dur.py" }, "a", body("DUR_OLD")], + ["read_file", { path: "test_dur.py" }, "b", body("TESTS")], + ]); + step(t, [["write_file", { path: "dur.py", content: "new" }, "w", "wrote"]]); + assert.equal(t.elideSupersededReads().elided, 1); + const results = t.all().filter((m) => m.role === "tool"); + assert.ok(results[0].content!.startsWith(ELIDED_PREFIX), "dur.py's stale copy goes"); + assert.match(results[1].content!, /^TESTS/, "test_dur.py was never rewritten and stays"); + }); + + it("a write to ./dur.py is a write to dur.py", () => { + const t = new Transcript("S"); + step(t, [["read_file", { path: "dur.py" }, "a", body("DUR_OLD")]]); + step(t, [["edit_file", { path: "./dur.py", old: "a", new: "b" }, "e", "edited"]]); + assert.equal(t.elideSupersededReads().elided, 1); + }); + + it("a bash rerun that came back different elides the earlier output", () => { + const t = new Transcript("S"); + step(t, [["bash", { command: "pytest -q" }, "a", body("3 failed")]]); + step(t, [["bash", { command: "pytest -q" }, "b", body("3 passed")]]); + assert.equal(t.elideSupersededReads().elided, 1); + const wire = JSON.stringify(t.wire()); + assert.ok(!wire.includes("3 failed") && wire.includes("3 passed")); + assert.match(wire, /newer output of the same command/); + }); + + it("a rerun with different options (a longer timeout_s) is a different call: the timeout note stays", () => { + const t = new Transcript("S"); + step(t, [["bash", { command: "make" }, "a", body("timeout after 120s; call again with timeout_s")]]); + step(t, [["bash", { command: "make", timeout_s: 600 }, "b", body("built")]]); + assert.equal(t.elideSupersededReads().elided, 0); + }); + + it("a rerun that came back the same keeps the earlier output it points at", () => { + const t = new Transcript("S"); + step(t, [["bash", { command: "ls" }, "a", body("files")]]); + step(t, [["bash", { command: "ls" }, "b", "[molt: this is the same bash call you made at step 1, and nothing has changed since.]"]]); + assert.equal(t.elideSupersededReads().elided, 0); + }); + + it("cat or sed -n of a file is a read a later write makes stale; a pipeline is not", () => { + const t = new Transcript("S"); + step(t, [["bash", { command: "sed -n '1,80p' src/a.py" }, "a", body("OLD A")]]); + step(t, [["bash", { command: "cat src/b.py | head" }, "b", body("B PIPE")]]); + step(t, [["edit_file", { path: "src/a.py", old: "x", new: "y" }, "e", "edited"]]); + step(t, [["edit_file", { path: "src/b.py", old: "x", new: "y" }, "f", "edited"]]); + assert.equal(t.elideSupersededReads().elided, 1); + const wire = JSON.stringify(t.wire()); + assert.ok(!wire.includes("OLD A") && wire.includes("B PIPE")); + }); + + it("in an engine run: rewriting dur.py keeps test_dur.py on the wire", async () => { + const ws = workspace(); + try { + writeFileSync(join(ws.dir, "dur.py"), `def parse(s):\n return 0\n${"# pad\n".repeat(300)}`); + writeFileSync(join(ws.dir, "test_dur.py"), `def test_hours():\n assert parse('1h30m') == 5400 # TEST_MARK\n${"# pad\n".repeat(300)}`); + const p = scriptedProvider([ + { calls: [{ name: "read_file", args: { path: "dur.py" } }, { name: "read_file", args: { path: "test_dur.py" } }] }, + { calls: [{ name: "write_file", args: { path: "dur.py", content: "def parse(s):\n return 5400\n" } }] }, + { text: "Done." }, + ] as never); + const e = new Engine({ baseUrl: "http://p.test/v1", model: "m", cwd: ws.dir, fetchFn: p.fetchFn, bar: null, stream: false, autonomy: "high", maxSteps: 0 }); + await drain(e.run("fix dur.py", allowAll)); + const last = p.bodies.at(-1)!; + assert.match(last, /TEST_MARK/, "test_dur.py was elided with dur.py"); + assert.doesNotMatch(last, /return 0/, "dur.py's stale copy is still on the wire"); + assert.match(readFileSync(join(ws.dir, "dur.py"), "utf8"), /5400/); + } finally { + ws.cleanup(); + } + }); + + it("a plain cat whose rerun came back as a pointer is still invalidated by a write", () => { + const t = new Transcript("S"); + step(t, [["bash", { command: "cat dur.py" }, "a", body("DUR_OLD")]]); + step(t, [["bash", { command: "cat dur.py" }, "b", "[molt: this is the same bash call you made at step 1, and nothing has changed since.]"]]); + step(t, [["write_file", { path: "dur.py", content: "new" }, "w", "wrote"]]); + assert.equal(t.elideSupersededReads().elided, 1); + assert.ok(!JSON.stringify(t.wire()).includes("DUR_OLD")); + }); + + it("a read_file answered with a pointer keeps the copy it points at, and a write still finds it", () => { + const t = new Transcript("S"); + step(t, [["read_file", { path: "dur.py" }, "a", body("DUR_OLD")]]); + step(t, [["read_file", { path: "dur.py" }, "b", "[molt: you have already been shown lines 1-40 of dur.py. Scroll up.]"]]); + step(t, [["read_file", { path: "dur.py" }, "c", "[molt: this is the same read_file call you made at step 1, and nothing has changed since.]"]]); + assert.equal(t.elideSupersededReads().elided, 0, "the original was elided under its own pointer"); + assert.ok(JSON.stringify(t.wire()).includes("DUR_OLD")); + step(t, [["write_file", { path: "dur.py", content: "new" }, "w", "wrote"]]); + assert.equal(t.elideSupersededReads().elided, 1); + assert.ok(!JSON.stringify(t.wire()).includes("DUR_OLD")); + }); + + it("the rerun key does not depend on argument order", () => { + const t = new Transcript("S"); + t.push({ role: "assistant", content: null, tool_calls: [{ id: "a", type: "function", function: { name: "bash", arguments: '{"timeout_s":600,"command":"make"}' } }] }); + t.push({ role: "tool", tool_call_id: "a", content: body("1 error") }); + t.push({ role: "assistant", content: null, tool_calls: [{ id: "b", type: "function", function: { name: "bash", arguments: '{"command":"make","timeout_s":600}' } }] }); + t.push({ role: "tool", tool_call_id: "b", content: body("built") }); + assert.equal(t.elideSupersededReads().elided, 1); + }); + + it("a background rerun is left alone: its result is the handle for the job", () => { + const t = new Transcript("S"); + step(t, [["bash", { command: "npm run dev", background: true }, "a", body("started job 1")]]); + step(t, [["bash", { command: "npm run dev", background: true }, "b", body("started job 2")]]); + assert.equal(t.elideSupersededReads().elided, 0); + }); +});