From 578d3b4375e264ea7ea769ba67480366abccebbc Mon Sep 17 00:00:00 2001 From: Tyler Skelton Date: Thu, 8 Oct 2026 00:55:01 -0700 Subject: [PATCH 1/5] Shed min-free and per-call elision as defaults A shed cut on user turns must free at least 25% of the history, or it cuts on recent messages instead. Maat's notes arrive as user messages, so a real run shed 6 messages for 0.7% (60,788 -> 60,387 tokens) and lost its prompt cache; with a short first exchange the plan came back null every step and auto-shed never fired again that turn. Superseded results are elided per call, not per step: rewriting dur.py no longer drops test_dur.py read in the same step. A bash rerun with new output supersedes the earlier one; a rerun with different options is a different call. A plain cat/head/tail/nl/sed -n of one file is a read. Regression tests from the real cases in test/lean-fixes.test.ts. --- src/transcript.ts | 180 ++++++++++++++++++++++++++---------- test/lean-fixes.test.ts | 198 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 330 insertions(+), 48 deletions(-) create mode 100644 test/lean-fixes.test.ts diff --git a/src/transcript.ts b/src/transcript.ts index 9fef4a9..bb198c2 100644 --- a/src/transcript.ts +++ b/src/transcript.ts @@ -24,6 +24,15 @@ import { estTokens, type Bom, type Msg } from "./types.js"; export const STALE_FAILURE_PREFIX = "[molt: superseded]"; export const ELIDED_PREFIX = "[molt: superseded tool result —"; +/** How the engine sends a repeated call whose result has not changed (see engine.ts). */ +const SAME_CALL_PREFIX = "[molt: this is the same "; + +/** + * A bash command that only prints one file: `cat f`, `head -n 40 f`, + * `sed -n '1,80p' f`, `nl f`, `tail f`. Anything with a pipe, a redirect or a + * second command is not a plain read and is left alone. + */ +const SIMPLE_READ = /^(?:cat|nl(?: -\S+)*|head(?: -\S+(?: \d+)?)*|tail(?: -\S+(?: \d+)?)*|sed -n ['"]?[\d,$p]+['"]?)\s+([^\s|;&<>$`*?]+)$/; /** * How many steps an elision has to pay for itself in, when a cache is working. @@ -48,6 +57,12 @@ const MAX_ACTION_LINES = 25; * turns to cut on. Fall back to keeping this many recent messages. */ const KEEP_RECENT_MESSAGES = 6; +/** + * The least share of the history a shed cut on user turns must free, or the + * shed cuts on recent messages instead (see planShed). A shed costs the whole + * prompt cache, so one that frees almost nothing is worse than none. + */ +export const SHED_MIN_FREE = 0.25; /** Never bother shedding fewer than this many messages. */ const MIN_DROPPED = 2; @@ -70,6 +85,8 @@ export class Transcript { private archived: Msg[][] = []; /** What this turn is for. Sent every request, shed never. */ private task: string | null = null; + /** See SHED_MIN_FREE. 0 restores the plain cut on user turns. */ + shedMinFree = SHED_MIN_FREE; constructor(systemPrompt: string) { this.system = { role: "system", content: systemPrompt }; @@ -218,7 +235,7 @@ export class Transcript { bom(toolSchemaJson: string, session: { prompt: number; completion: number }): Bom { const historyTokens = this.working.reduce( - (n, m) => n + estTokens(m.content ?? "") + estTokens(JSON.stringify(m.tool_calls ?? "")), + (n, m) => n + msgTokens(m), 0, ); // The standing note is part of every request, so it is part of the fixed @@ -238,7 +255,7 @@ export class Transcript { historyTokens(): number { return this.working.reduce( - (n, m) => n + estTokens(m.content ?? "") + estTokens(JSON.stringify(m.tool_calls ?? "")), + (n, m) => n + msgTokens(m), 0, ); } @@ -273,6 +290,20 @@ export class Transcript { let cutAt: number; if (userIdxs.length > keepExchanges) { cutAt = userIdxs[userIdxs.length - keepExchanges]; + // A cut on user turns must buy real headroom. Maat's own notes (the + // acceptance criteria, a bar refusal) arrive as user messages, so a + // single long request has three "exchanges" with nearly all of its + // history after the second. A real run shed 6 messages, 60,788 -> + // 60,387 tokens, threw away its whole prompt cache for 0.7%, and shed + // again one step later. With a short first exchange the digest cost + // more than the cut freed, the plan came back null on every step, and + // auto-shed never fired again that turn. Either way, cut on recent + // messages instead, as a turn with no user turn to cut on does. + const freed = this.working.slice(0, cutAt).reduce((n, m) => n + msgTokens(m), 0); + if (freed < this.shedMinFree * this.historyTokens()) { + const fallback = this.findSafeCut(this.working.length - Math.max(2, keepRecent)); + if (fallback !== null && fallback > cutAt) cutAt = fallback; + } } else { // A single request can produce dozens of tool calls with no user turn // to cut on — which is exactly when context runs out. Fall back to @@ -303,7 +334,7 @@ export class Transcript { molt: { digest: true }, }; const afterTokens = [digestMsg, ...kept].reduce( - (n, m) => n + estTokens(m.content ?? "") + estTokens(JSON.stringify(m.tool_calls ?? "")), + (n, m) => n + msgTokens(m), 0, ); @@ -456,7 +487,26 @@ export class Transcript { elideSupersededReads( opts: { protectCache?: boolean } = {}, ): { elided: number; tokensSaved: number; deferred: number } { + /** + * What is superseded is a CALL, not the step it was made in: result + * index -> why. Keyed by the assistant message, a write to one file + * elided every result of the step that read it. A real run read dur.py + * and test_dur.py in one step, rewrote dur.py, and lost test_dur.py with + * it, under a marker saying the current copy was further down. It was not. + */ const supersededBy = new Map(); + /** A call's place in the conversation: the index of its result, or -1. */ + type At = number; + const resultAt = (i: number, k: number, id: string | undefined): At => { + for (let j = i + 1; j < this.working.length && this.working[j].role === "tool"; j++) { + const r = this.working[j]; + if (id ? r.tool_call_id === id : j === i + 1 + k) return j; + } + return -1; + }; + const mark = (at: At, why: string) => { + if (at >= 0) supersededBy.set(at, why); + }; /** * Reads still worth keeping, keyed by the exact window they returned. * @@ -468,90 +518,124 @@ export class Transcript { * it back to read the same file again, forever. Two features that were * each correct alone. */ - const lastRead = new Map(); + const lastRead = new Map(); /** Every live read of a path, so a write can invalidate all of them. */ const readsOf = new Map(); + const noteRead = (path: string, window: string, at: At) => { + lastRead.set(window, at); + const windows = readsOf.get(path) ?? []; + if (!windows.includes(window)) windows.push(window); + readsOf.set(path, windows); + }; + /** The last live run of each bash call (command and options), by result index. */ + const lastRun = new Map(); for (let i = 0; i < this.working.length; i++) { const m = this.working[i]; - for (const call of m.tool_calls ?? []) { + (m.tool_calls ?? []).forEach((call, k) => { let args: Record = {}; try { args = JSON.parse(call.function.arguments || "{}") as Record; } catch { - continue; + return; + } + const at = resultAt(i, k, call.id); + + if (call.function.name === "bash" && typeof args.command === "string") { + const cmd = args.command.replace(/\s+/g, " ").trim(); + // The same call is the same command with the same options. A rerun + // with a longer timeout_s, or in the background, is a different + // call, and the earlier result (a timeout and how to avoid it) is + // the reason for it. + const key = JSON.stringify({ ...args, command: cmd }); + const now = at >= 0 ? (this.working[at].content ?? "") : ""; + const pointer = now.startsWith(SAME_CALL_PREFIX); + // A rerun that came back the same is sent as a pointer to the + // earlier copy, which must then stay. One that came back different + // makes the earlier output history. + const prior = lastRun.get(key); + if (prior !== undefined && at >= 0 && !pointer && now !== (this.working[prior]?.content ?? "")) + mark(prior, `rerun at step ${i}`); + if (!pointer && at >= 0) lastRun.set(key, at); + // A plain read of one file through bash is a read of that file: a + // later write makes it stale exactly as it does a read_file. + const read = SIMPLE_READ.exec(cmd); + if (read) noteRead(read[1]!.replace(/^\.\//, ""), `bash:${cmd}`, at); + return; } - const path = String(args.path ?? ""); - if (!path) continue; + + const path = String(args.path ?? "").replace(/^\.\//, ""); + if (!path) return; if (call.function.name === "read_file") { // Identical arguments return identical bytes; anything else is a // different part of the file and stands on its own. const window = `${path}@${Number(args.offset ?? 0)}+${String(args.limit ?? "all")}`; const prior = lastRead.get(window); - if (prior !== undefined) supersededBy.set(prior, `re-read at step ${i}`); - lastRead.set(window, i); - const windows = readsOf.get(path) ?? []; - if (!windows.includes(window)) windows.push(window); - readsOf.set(path, windows); + if (prior !== undefined) mark(prior, `re-read at step ${i}`); + noteRead(path, window, at); } else if (call.function.name === "write_file" || call.function.name === "edit_file") { // A change to the file invalidates every part of it that was read, // whichever window it came from: what is in context is no longer // what is on disk. for (const window of readsOf.get(path) ?? []) { const prior = lastRead.get(window); - if (prior !== undefined) supersededBy.set(prior, `changed at step ${i}`); + if (prior !== undefined) mark(prior, `changed at step ${i}`); lastRead.delete(window); } readsOf.delete(path); } - } + }); } let elided = 0; let tokensSaved = 0; let deferred = 0; - for (const [callIdx, reason] of supersededBy) { - // The tool result follows its assistant turn. - for (let j = callIdx + 1; j < this.working.length; j++) { - const m = this.working[j]; - if (m.role !== "tool") break; - if (!m.content || m.content.startsWith(ELIDED_PREFIX)) continue; - const before = estTokens(m.content); - // Wording matters here. "Full contents remain in the archived record" - // reads, to a model, as an invitation to go and get them — which it - // can only do by re-reading the file, which is what elided this copy - // in the first place. Point at the newer copy instead. - const marker = - `${ELIDED_PREFIX} ${reason}. The current contents are further down this ` + + for (const [j, reason] of supersededBy) { + const m = this.working[j]; + if (!m || m.role !== "tool") continue; + if (!m.content || m.content.startsWith(ELIDED_PREFIX)) continue; + const before = estTokens(m.content); + // Wording matters here. "Full contents remain in the archived record" + // reads, to a model, as an invitation to go and get them — which it + // can only do by re-reading the file, which is what elided this copy + // in the first place. Point at the newer copy instead. + const marker = reason.startsWith("rerun") + ? `${ELIDED_PREFIX} ${reason}. The newer output of the same command is further ` + + `down this conversation.` + : `${ELIDED_PREFIX} ${reason}. The current contents are further down this ` + `conversation; do not read the file again to recover this.`; - // A short result costs less than the notice explaining its absence. - // Eliding it would drop content AND grow the context — which is how - // the meter came to report "−-17 tokens" saved. - if (estTokens(marker) >= before) continue; - const saving = before - estTokens(marker); - // What this edit strands: everything after it shares a prefix that is - // about to change, so the next request pays full price for all of it. - // Elide only if the saving earns that back inside ELISION_PAYBACK_STEPS. - if (opts.protectCache) { - let stranded = 0; - for (let k = j + 1; k < this.working.length; k++) { - stranded += estTokens(this.working[k].content ?? ""); - } - if (saving * ELISION_PAYBACK_STEPS < stranded) { - deferred++; - continue; - } + // A short result costs less than the notice explaining its absence. + // Eliding it would drop content AND grow the context — which is how + // the meter came to report "−-17 tokens" saved. + if (estTokens(marker) >= before) continue; + const saving = before - estTokens(marker); + // What this edit strands: everything after it shares a prefix that is + // about to change, so the next request pays full price for all of it. + // Elide only if the saving earns that back inside ELISION_PAYBACK_STEPS. + if (opts.protectCache) { + let stranded = 0; + for (let k = j + 1; k < this.working.length; k++) { + stranded += estTokens(this.working[k].content ?? ""); + } + if (saving * ELISION_PAYBACK_STEPS < stranded) { + deferred++; + continue; } - m.content = marker; - tokensSaved += saving; - elided++; } + m.content = marker; + tokensSaved += saving; + elided++; } return { elided, tokensSaved, deferred }; } } +/** What a message costs in the history, in molt's token units. */ +function msgTokens(m: Msg): number { + return estTokens(m.content ?? "") + estTokens(JSON.stringify(m.tool_calls ?? "")); +} + /** * A digest is verbatim excerpts, never a paraphrase. Prior digests are * carried forward whole rather than re-excerpted — re-truncating a diff --git a/test/lean-fixes.test.ts b/test/lean-fixes.test.ts new file mode 100644 index 0000000..89b1057 --- /dev/null +++ b/test/lean-fixes.test.ts @@ -0,0 +1,198 @@ +/** + * Two defects found by the lean-sessions study (reports/lean-sessions-study.md), + * fixed as defaults. Each test is built from the real case. + * + * 1. Shed min-free. Maat's own notes (acceptance criteria, a bar refusal) are + * user messages, so the cut on user turns could land near the start of a + * long single-request turn: the duration-bug run shed 6 messages, 60,788 -> + * 60,387 tokens (0.7%), lost its whole prompt cache, and shed again a step + * later. With a short first exchange the plan came back null on every step + * and auto-shed never fired again that turn. + * 2. Per-call elision. Elision was keyed by step: a step that read dur.py and + * test_dur.py together lost test_dur.py when dur.py was rewritten, under a + * marker saying the current copy was further down. It was not. + */ +import assert from "node:assert/strict"; +import { readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, it } from "node:test"; +import { Engine } from "../src/engine.js"; +import { ELIDED_PREFIX, SHED_MIN_FREE, Transcript } from "../src/transcript.js"; +import { allowAll, drain, scriptedProvider, toolCall, workspace } from "./helpers.js"; + +describe("shed min-free (default)", () => { + /** + * The shape of the duration-bug run: the request and a small first + * exchange (six messages), Maat's acceptance-criteria note, a long tool + * run, a bar refusal, another long tool run. Two exchanges kept puts the + * cut on the criteria note. `"short"` is the tiny first exchange (one + * short read) that makes the digest cost more than the cut frees. + */ + function durationBugShape(firstRead = "f".repeat(800)): Transcript { + const t = new Transcript("SYSTEM"); + t.push({ role: "user", content: "Fix the duration parser so `1h30m` parses." }); + t.push({ role: "assistant", content: null, tool_calls: [toolCall("read_file", { path: "dur.py" }, "r0")] }); + t.push({ role: "tool", tool_call_id: "r0", content: firstRead }); + if (firstRead !== "short") { + t.push({ role: "assistant", content: null, tool_calls: [toolCall("read_file", { path: "test_dur.py" }, "r1")] }); + t.push({ role: "tool", tool_call_id: "r1", content: firstRead }); + t.push({ role: "assistant", content: "Reading done." }); + } + t.push({ role: "user", content: "Acceptance criteria for this task: ..." }); + for (let i = 0; i < 40; i++) { + t.push({ role: "assistant", content: null, tool_calls: [toolCall("bash", { command: `s${i}` }, `a${i}`)] }); + t.push({ role: "tool", tool_call_id: `a${i}`, content: `${i}:${"o".repeat(3000)}` }); + } + t.push({ role: "user", content: "[molt] You indicated the task is complete, but 1 of 3 checks did not pass." }); + for (let i = 0; i < 40; i++) { + t.push({ role: "assistant", content: null, tool_calls: [toolCall("bash", { command: `t${i}` }, `b${i}`)] }); + t.push({ role: "tool", tool_call_id: `b${i}`, content: `${i}:${"p".repeat(3000)}` }); + } + return t; + } + + it("is on by default at a quarter of the history", () => { + assert.equal(SHED_MIN_FREE, 0.25); + assert.equal(new Transcript("S").shedMinFree, SHED_MIN_FREE); + }); + + it("the 0.7% shed: without min-free the cut drops 6 messages and frees under 1%", () => { + const t = durationBugShape(); + t.shedMinFree = 0; + const plan = t.planShed()!; + assert.equal(plan.droppedCount, 6); + assert.ok(plan.afterTokens / plan.beforeTokens > 0.99, `${plan.beforeTokens} -> ${plan.afterTokens}`); + }); + + it("the 0.7% shed: by default it is a real shed", () => { + const plan = durationBugShape().planShed()!; + assert.ok(plan.afterTokens / plan.beforeTokens < 0.75, `${plan.beforeTokens} -> ${plan.afterTokens}`); + assert.ok(plan.afterTokens / plan.beforeTokens < 0.1, "it cuts on recent messages, as a single request does"); + }); + + it("the empty plan: without min-free a short first exchange gives no shed at all", () => { + const t = durationBugShape("short"); + t.shedMinFree = 0; + assert.equal(t.planShed(), null); + }); + + it("the empty plan: by default the shed happens, and keeps happening as the turn goes on", () => { + const t = durationBugShape("short"); + const auto = 20_000; + let sheds = 0; + let peak = 0; + for (let i = 0; i < 60; i++) { + // The engine's auto-shed, once per step. + if (t.historyTokens() > auto) { + const plan = t.planShed(); + assert.ok(plan, `no shed at step ${i} with ${t.historyTokens()} tokens of history`); + t.commitShed(plan); + sheds++; + } + peak = Math.max(peak, t.historyTokens()); + t.push({ role: "assistant", content: null, tool_calls: [toolCall("bash", { command: `u${i}` }, `c${i}`)] }); + t.push({ role: "tool", tool_call_id: `c${i}`, content: `${i}:${"q".repeat(3000)}` }); + } + assert.ok(sheds >= 3, `only ${sheds} shed(s)`); + assert.ok(peak < auto * 1.5, `history reached ${peak}`); + }); + + it("leaves a cut on user turns alone when it frees enough", () => { + const t = new Transcript("S"); + for (let k = 0; k < 4; k++) { + t.push({ role: "user", content: `ask ${k}` }); + for (let i = 0; i < 5; i++) { + t.push({ role: "assistant", content: null, tool_calls: [toolCall("bash", { command: `${k}.${i}` }, `c${k}.${i}`)] }); + t.push({ role: "tool", tool_call_id: `c${k}.${i}`, content: "x".repeat(2000) }); + } + } + const plan = t.planShed()!; + // Two of four exchanges go: the cut is the user turn, not the recent fallback. + assert.equal(plan.droppedCount, 22); + }); +}); + +describe("per-call elision (default)", () => { + function step(t: Transcript, calls: [string, Record, string, string][]) { + t.push({ role: "assistant", content: null, tool_calls: calls.map(([n, a, id]) => toolCall(n, a, id)) }); + for (const [, , id, out] of calls) t.push({ role: "tool", tool_call_id: id, content: out }); + } + const body = (s: string) => `${s}\n${"x".repeat(2000)}`; + + it("dur.py / test_dur.py: a write elides the read of that file only, not the rest of its step", () => { + const t = new Transcript("S"); + t.push({ role: "user", content: "task" }); + step(t, [ + ["read_file", { path: "dur.py" }, "a", body("DUR_OLD")], + ["read_file", { path: "test_dur.py" }, "b", body("TESTS")], + ]); + step(t, [["write_file", { path: "dur.py", content: "new" }, "w", "wrote"]]); + assert.equal(t.elideSupersededReads().elided, 1); + const results = t.all().filter((m) => m.role === "tool"); + assert.ok(results[0].content!.startsWith(ELIDED_PREFIX), "dur.py's stale copy goes"); + assert.match(results[1].content!, /^TESTS/, "test_dur.py was never rewritten and stays"); + }); + + it("a write to ./dur.py is a write to dur.py", () => { + const t = new Transcript("S"); + step(t, [["read_file", { path: "dur.py" }, "a", body("DUR_OLD")]]); + step(t, [["edit_file", { path: "./dur.py", old: "a", new: "b" }, "e", "edited"]]); + assert.equal(t.elideSupersededReads().elided, 1); + }); + + it("a bash rerun that came back different elides the earlier output", () => { + const t = new Transcript("S"); + step(t, [["bash", { command: "pytest -q" }, "a", body("3 failed")]]); + step(t, [["bash", { command: "pytest -q" }, "b", body("3 passed")]]); + assert.equal(t.elideSupersededReads().elided, 1); + const wire = JSON.stringify(t.wire()); + assert.ok(!wire.includes("3 failed") && wire.includes("3 passed")); + assert.match(wire, /newer output of the same command/); + }); + + it("a rerun with different options (a longer timeout_s) is a different call: the timeout note stays", () => { + const t = new Transcript("S"); + step(t, [["bash", { command: "make" }, "a", body("timeout after 120s; call again with timeout_s")]]); + step(t, [["bash", { command: "make", timeout_s: 600 }, "b", body("built")]]); + assert.equal(t.elideSupersededReads().elided, 0); + }); + + it("a rerun that came back the same keeps the earlier output it points at", () => { + const t = new Transcript("S"); + step(t, [["bash", { command: "ls" }, "a", body("files")]]); + step(t, [["bash", { command: "ls" }, "b", "[molt: this is the same bash call you made at step 1, and nothing has changed since.]"]]); + assert.equal(t.elideSupersededReads().elided, 0); + }); + + it("cat or sed -n of a file is a read a later write makes stale; a pipeline is not", () => { + const t = new Transcript("S"); + step(t, [["bash", { command: "sed -n '1,80p' src/a.py" }, "a", body("OLD A")]]); + step(t, [["bash", { command: "cat src/b.py | head" }, "b", body("B PIPE")]]); + step(t, [["edit_file", { path: "src/a.py", old: "x", new: "y" }, "e", "edited"]]); + step(t, [["edit_file", { path: "src/b.py", old: "x", new: "y" }, "f", "edited"]]); + assert.equal(t.elideSupersededReads().elided, 1); + const wire = JSON.stringify(t.wire()); + assert.ok(!wire.includes("OLD A") && wire.includes("B PIPE")); + }); + + it("in an engine run: rewriting dur.py keeps test_dur.py on the wire", async () => { + const ws = workspace(); + try { + writeFileSync(join(ws.dir, "dur.py"), `def parse(s):\n return 0\n${"# pad\n".repeat(300)}`); + writeFileSync(join(ws.dir, "test_dur.py"), `def test_hours():\n assert parse('1h30m') == 5400 # TEST_MARK\n${"# pad\n".repeat(300)}`); + const p = scriptedProvider([ + { calls: [{ name: "read_file", args: { path: "dur.py" } }, { name: "read_file", args: { path: "test_dur.py" } }] }, + { calls: [{ name: "write_file", args: { path: "dur.py", content: "def parse(s):\n return 5400\n" } }] }, + { text: "Done." }, + ] as never); + const e = new Engine({ baseUrl: "http://p.test/v1", model: "m", cwd: ws.dir, fetchFn: p.fetchFn, bar: null, stream: false, autonomy: "high", maxSteps: 0 }); + await drain(e.run("fix dur.py", allowAll)); + const last = p.bodies.at(-1)!; + assert.match(last, /TEST_MARK/, "test_dur.py was elided with dur.py"); + assert.doesNotMatch(last, /return 0/, "dur.py's stale copy is still on the wire"); + assert.match(readFileSync(join(ws.dir, "dur.py"), "utf8"), /5400/); + } finally { + ws.cleanup(); + } + }); +}); From 7ed16d22555c46751814ed708d2c4cb03a39ad23 Mon Sep 17 00:00:00 2001 From: Tyler Skelton Date: Thu, 8 Oct 2026 00:55:01 -0700 Subject: [PATCH 2/5] CHANGELOG and docs/lean.md: shed min-free and per-call elision --- CHANGELOG.md | 14 ++++++++++++++ docs/lean.md | 21 +++++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a2f8fd..b1cb82d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -147,6 +147,20 @@ background logs, and paths the task names). bash is not blocked; this is a budget control, not isolation. +### Fixed + +- **Two lean-session defects (lean-sessions study).** A shed cut on user + turns must now free at least 25% of the history, or it cuts on recent + messages instead: Maat's own notes (acceptance criteria, bar refusals) are + user messages, so a shed could cost the whole prompt cache for 0.7% + (60,788 -> 60,387 tokens), or come back empty on every step so auto-shed + never fired again that turn. Superseded tool results are now elided per + call, not per step: rewriting `dur.py` no longer drops `test_dur.py` read in + the same step. A bash rerun that came back different supersedes the earlier + output (a rerun with different options, such as a longer `timeout_s`, is a + different call), and a plain `cat`/`head`/`tail`/`nl`/`sed -n` of one file + counts as a read a later write makes stale. See `docs/lean.md`. + ## 0.2.2 — subscription backends removed ### Removed diff --git a/docs/lean.md b/docs/lean.md index f7a8667..ab88f72 100644 --- a/docs/lean.md +++ b/docs/lean.md @@ -49,6 +49,27 @@ To see them: `npx tsc -p tsconfig.test.json && node --test dist-test/test/lean-b - The numbers are deterministic (scripted turns, no clock in the bodies), so 1.5x leaves room for small prompt edits, not for noise. +## Shed and elision rules the study fixed + +From `reports/lean-sessions-study.md`; tests in `test/lean-fixes.test.ts`. + +- **Shed min-free** (`SHED_MIN_FREE` = 0.25 in `src/transcript.ts`). A shed + cut on user turns must free at least a quarter of the history, or `planShed` + cuts on recent messages instead. A shed costs the whole prompt cache, and + Maat's own notes (acceptance criteria, bar refusals) arrive as user + messages, so the user-turn cut could land near the start of a long turn + (a real run: 60,788 -> 60,387 tokens) or return no plan at all, step after + step, so auto-shed never fired again. +- **Per-call elision.** `elideSupersededReads` elides the superseded call's + own result, not every result of its step. A bash rerun with new output + supersedes the earlier output; the same command with different options + (`timeout_s`, background) is a different call and keeps its result. A plain + `cat`/`head`/`tail`/`nl`/`sed -n` of one file (no pipe, redirect or second + command) is a read that a later write of that file makes stale. + +The flag-gated experiments from the study (shed threshold, ageing) are not +part of this. + ## The bench cost alarm (`bench/local/run.py`) After each run, run.py reads the run's cost (`spend.costUsd`) and prompt From 54d8002de7970e9ed8d729ebd540e4acaba78c04 Mon Sep 17 00:00:00 2001 From: Tyler Skelton Date: Thu, 8 Oct 2026 01:41:18 -0700 Subject: [PATCH 3/5] Review fixes: carry criteria and live refusal through a recent cut; tighter supersede - A shed that cuts on recent messages keeps the latest criteria note (now tagged molt.criteria) and the live bar refusal verbatim after the digest, instead of excerpting them to 300 chars. - Min-free falls back only when the kept user turns are mostly Maat's notes; an interactive session is cut on user turns as before. - A bash read answered with a same-call pointer is not noted as a read, so a later write still elides the full copy. - The rerun key sorts its keys; background runs take no part in rerun supersession; commands:[...] calls stay out (noted in a comment). - Docs and the test header cite PR #50 instead of a report not on this base. --- CHANGELOG.md | 10 +++-- docs/lean.md | 33 +++++++++------ src/engine.ts | 2 +- src/transcript.ts | 88 +++++++++++++++++++++++++++++++-------- src/types.ts | 5 +++ test/lean-fixes.test.ts | 92 +++++++++++++++++++++++++++++++++++++++-- 6 files changed, 192 insertions(+), 38 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b1cb82d..3b3b646 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -149,9 +149,9 @@ ### Fixed -- **Two lean-session defects (lean-sessions study).** A shed cut on user - turns must now free at least 25% of the history, or it cuts on recent - messages instead: Maat's own notes (acceptance criteria, bar refusals) are +- **Two lean-session defects (lean-sessions study, PR #50).** When the kept + user turns are mostly Maat's notes, a shed cut on user turns must now free + at least 25% of the history, or it cuts on recent messages instead: Maat's own notes (acceptance criteria, bar refusals) are user messages, so a shed could cost the whole prompt cache for 0.7% (60,788 -> 60,387 tokens), or come back empty on every step so auto-shed never fired again that turn. Superseded tool results are now elided per @@ -159,7 +159,9 @@ the same step. A bash rerun that came back different supersedes the earlier output (a rerun with different options, such as a longer `timeout_s`, is a different call), and a plain `cat`/`head`/`tail`/`nl`/`sed -n` of one file - counts as a read a later write makes stale. See `docs/lean.md`. + counts as a read a later write makes stale. A shed that cuts on recent + messages keeps the acceptance criteria and the live bar refusal verbatim. + See `docs/lean.md`. ## 0.2.2 — subscription backends removed diff --git a/docs/lean.md b/docs/lean.md index ab88f72..7246761 100644 --- a/docs/lean.md +++ b/docs/lean.md @@ -51,21 +51,30 @@ To see them: `npx tsc -p tsconfig.test.json && node --test dist-test/test/lean-b ## Shed and elision rules the study fixed -From `reports/lean-sessions-study.md`; tests in `test/lean-fixes.test.ts`. - -- **Shed min-free** (`SHED_MIN_FREE` = 0.25 in `src/transcript.ts`). A shed - cut on user turns must free at least a quarter of the history, or `planShed` - cuts on recent messages instead. A shed costs the whole prompt cache, and - Maat's own notes (acceptance criteria, bar refusals) arrive as user - messages, so the user-turn cut could land near the start of a long turn - (a real run: 60,788 -> 60,387 tokens) or return no plan at all, step after - step, so auto-shed never fired again. +From the lean-sessions study (PR #50); tests in `test/lean-fixes.test.ts`. + +- **Shed min-free** (`SHED_MIN_FREE` = 0.25 in `src/transcript.ts`). When + the user turns a shed would keep are mostly Maat's own notes (acceptance + criteria, bar refusals, nudges, all tagged in `molt`), the cut on user turns + must free at least a quarter of the history, or `planShed` cuts on recent + messages instead. A shed costs the whole prompt cache, and those notes are + user messages, so the user-turn cut could land near the start of a long + turn (a real run: 60,788 -> 60,387 tokens) or return no plan at all, step + after step, so auto-shed never fired again. An interactive session whose + kept turns are the person's own is cut on user turns as before. +- **Criteria and refusal survive a recent-message cut.** On a cut on recent + messages, the latest criteria note (`molt.criteria`) and the live bar + refusal (the latest non-stale `molt.barFailure`) stay verbatim right after + the digest instead of being excerpted to 300 characters in it. - **Per-call elision.** `elideSupersededReads` elides the superseded call's own result, not every result of its step. A bash rerun with new output supersedes the earlier output; the same command with different options - (`timeout_s`, background) is a different call and keeps its result. A plain - `cat`/`head`/`tail`/`nl`/`sed -n` of one file (no pipe, redirect or second - command) is a read that a later write of that file makes stale. + (`timeout_s`, compared with keys sorted) is a different call, and a + `background: true` run is never superseded (its result is the job handle). + A plain `cat`/`head`/`tail`/`nl`/`sed -n` of one file (no pipe, redirect or + second command) is a read that a later write of that file makes stale; a + bash rerun answered with a "same call" pointer is not a new read, so the + copy it points at stays the one a write invalidates. The flag-gated experiments from the study (shed threshold, ageing) are not part of this. diff --git a/src/engine.ts b/src/engine.ts index 5816edb..513109a 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -6062,7 +6062,7 @@ export class Engine { "never reported as verified — do not describe one as passing. You cannot", "edit these; attempting to is itself a failure.", ]; - this.transcript.push({ role: "user", content: lines.join("\n") }); + this.transcript.push({ role: "user", content: lines.join("\n"), molt: { criteria: true } }); } }; if (!pendingCriteria) yield* sealCriteria(opts.taskChecks ?? [], opts.taskNotes ?? []); diff --git a/src/transcript.ts b/src/transcript.ts index bb198c2..e50d5de 100644 --- a/src/transcript.ts +++ b/src/transcript.ts @@ -74,6 +74,14 @@ export type ShedPlan = { /** Messages being removed from the working context. */ dropped: Msg[]; droppedCount: number; + /** + * Messages from before the cut that stay, verbatim, right after the digest: + * the acceptance criteria and the live bar refusal (see planShed). Empty + * on a plain cut. + */ + carried: Msg[]; + /** Where the cut is: everything before it is either dropped or carried. */ + cutAt: number; beforeTokens: number; afterTokens: number; }; @@ -288,6 +296,8 @@ export class Transcript { .filter((i) => i >= 0); let cutAt: number; + /** The cut is on recent messages, not on a user turn. */ + let recentCut = false; if (userIdxs.length > keepExchanges) { cutAt = userIdxs[userIdxs.length - keepExchanges]; // A cut on user turns must buy real headroom. Maat's own notes (the @@ -299,10 +309,20 @@ export class Transcript { // more than the cut freed, the plan came back null on every step, and // auto-shed never fired again that turn. Either way, cut on recent // messages instead, as a turn with no user turn to cut on does. + // + // Only for that shape, though: the user turns kept must be mostly + // Maat's notes. In an interactive session whose last two real + // exchanges hold most of the history, those exchanges are what the + // person is working on, and the cut stays on them as it always did. + const kept = this.working.slice(cutAt).filter((m) => m.role === "user"); + const notes = kept.filter(isMaatNote).length; const freed = this.working.slice(0, cutAt).reduce((n, m) => n + msgTokens(m), 0); - if (freed < this.shedMinFree * this.historyTokens()) { + if (notes > kept.length - notes && freed < this.shedMinFree * this.historyTokens()) { const fallback = this.findSafeCut(this.working.length - Math.max(2, keepRecent)); - if (fallback !== null && fallback > cutAt) cutAt = fallback; + if (fallback !== null && fallback > cutAt) { + cutAt = fallback; + recentCut = true; + } } } else { // A single request can produce dozens of tool calls with no user turn @@ -317,12 +337,30 @@ export class Transcript { const fallback = this.findSafeCut(this.working.length - Math.max(2, keepRecent)); if (fallback === null) return null; cutAt = fallback; + recentCut = true; } - const dropped = this.working.slice(0, cutAt); - const kept = this.working.slice(cutAt); + // The acceptance criteria and the live bar refusal are what the model + // needs most right after a refusal, and a digest caps each message at + // EXCERPT_CHARS. A cut on recent messages can land after both, and with + // sheds every few steps the refusal's detail would last about three. So + // on such a cut the latest of each, when it is before the cut, stays + // verbatim. A cut on user turns keeps whole exchanges and is unchanged. + const lastIdx = (f: (m: Msg) => boolean) => { + for (let i = this.working.length - 1; i >= 0; i--) if (f(this.working[i]!)) return i; + return -1; + }; + const carryIdx = new Set( + [ + lastIdx((m) => m.molt?.criteria === true), + lastIdx((m) => m.molt?.barFailure === true && !!m.content && !m.content.startsWith(STALE_FAILURE_PREFIX)), + ].filter((i) => recentCut && i >= 0 && i < cutAt), + ); + const carried = this.working.slice(0, cutAt).filter((_, i) => carryIdx.has(i)); + const dropped = this.working.slice(0, cutAt).filter((_, i) => !carryIdx.has(i)); + const kept = [...carried, ...this.working.slice(cutAt)]; if (dropped.length < MIN_DROPPED || dropped.every(isDigest)) return null; - if (kept.length > 0 && kept[0].role === "tool") return null; + if (this.working[cutAt]?.role === "tool") return null; const beforeTokens = this.historyTokens(); const digest = buildDigest(dropped); @@ -342,7 +380,7 @@ export class Transcript { // more than the messages it replaces. if (afterTokens >= beforeTokens) return null; - return { exuvia, digest, dropped, droppedCount: dropped.length, beforeTokens, afterTokens }; + return { exuvia, digest, dropped, droppedCount: dropped.length, carried, cutAt, beforeTokens, afterTokens }; } /** @@ -364,12 +402,11 @@ export class Transcript { * has been durably archived — that ordering is the guarantee. */ commitShed(plan: ShedPlan): void { - const cut = plan.droppedCount; - const dropped = this.working.slice(0, cut); - this.archived.push(dropped); + this.archived.push(plan.dropped); this.working = [ { role: "system", content: plan.digest, molt: { digest: true } }, - ...this.working.slice(cut), + ...plan.carried, + ...this.working.slice(plan.cutAt), ]; } @@ -547,20 +584,32 @@ export class Transcript { // with a longer timeout_s, or in the background, is a different // call, and the earlier result (a timeout and how to avoid it) is // the reason for it. - const key = JSON.stringify({ ...args, command: cmd }); + // Keys sorted: {timeout_s, command} and {command, timeout_s} are one call. + const key = JSON.stringify( + Object.entries({ ...args, command: cmd }).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)), + ); const now = at >= 0 ? (this.working[at].content ?? "") : ""; const pointer = now.startsWith(SAME_CALL_PREFIX); // A rerun that came back the same is sent as a pointer to the // earlier copy, which must then stay. One that came back different // makes the earlier output history. - const prior = lastRun.get(key); - if (prior !== undefined && at >= 0 && !pointer && now !== (this.working[prior]?.content ?? "")) - mark(prior, `rerun at step ${i}`); - if (!pointer && at >= 0) lastRun.set(key, at); + // + // A background run is left out: its result names a job that may + // still be running, and it is the model's handle for stop_job. + // A `commands: [...]` call has no `command` and never gets here, so + // it takes no part in rerun supersession either. + if (!args.background) { + const prior = lastRun.get(key); + if (prior !== undefined && at >= 0 && !pointer && now !== (this.working[prior]?.content ?? "")) + mark(prior, `rerun at step ${i}`); + if (!pointer && at >= 0) lastRun.set(key, at); + } // A plain read of one file through bash is a read of that file: a - // later write makes it stale exactly as it does a read_file. + // later write makes it stale exactly as it does a read_file. A + // pointer to an earlier copy is not a read: the copy it points at + // is the one a write must invalidate. const read = SIMPLE_READ.exec(cmd); - if (read) noteRead(read[1]!.replace(/^\.\//, ""), `bash:${cmd}`, at); + if (read && !pointer) noteRead(read[1]!.replace(/^\.\//, ""), `bash:${cmd}`, at); return; } @@ -631,6 +680,11 @@ export class Transcript { } } +/** A user message Maat wrote itself: the criteria, a bar refusal, a nudge. */ +function isMaatNote(m: Msg): boolean { + return m.molt?.criteria === true || m.molt?.barFailure === true || m.molt?.nudge === true; +} + /** What a message costs in the history, in molt's token units. */ function msgTokens(m: Msg): number { return estTokens(m.content ?? "") + estTokens(JSON.stringify(m.tool_calls ?? "")); diff --git a/src/types.ts b/src/types.ts index 396bbba..67745f8 100644 --- a/src/types.ts +++ b/src/types.ts @@ -41,6 +41,11 @@ export type MsgMeta = { regrown?: true; /** True for a bar-failure notice injected by the proof loop. */ barFailure?: true; + /** + * True for the acceptance-criteria note. A shed that cuts on recent + * messages keeps the latest one verbatim (transcript.ts planShed). + */ + criteria?: true; /** * True for a note molt wrote to the model about the model's own behaviour * — an empty turn, or a step that only repeated itself. Distinct from a diff --git a/test/lean-fixes.test.ts b/test/lean-fixes.test.ts index 89b1057..64bb19b 100644 --- a/test/lean-fixes.test.ts +++ b/test/lean-fixes.test.ts @@ -1,6 +1,6 @@ /** - * Two defects found by the lean-sessions study (reports/lean-sessions-study.md), - * fixed as defaults. Each test is built from the real case. + * Two defects found by the lean-sessions study (PR #50), fixed as defaults. + * Each test is built from the real case. * * 1. Shed min-free. Maat's own notes (acceptance criteria, a bar refusal) are * user messages, so the cut on user turns could land near the start of a @@ -38,12 +38,12 @@ describe("shed min-free (default)", () => { t.push({ role: "tool", tool_call_id: "r1", content: firstRead }); t.push({ role: "assistant", content: "Reading done." }); } - t.push({ role: "user", content: "Acceptance criteria for this task: ..." }); + t.push({ role: "user", content: "Acceptance criteria for this task: ...", molt: { criteria: true } }); for (let i = 0; i < 40; i++) { t.push({ role: "assistant", content: null, tool_calls: [toolCall("bash", { command: `s${i}` }, `a${i}`)] }); t.push({ role: "tool", tool_call_id: `a${i}`, content: `${i}:${"o".repeat(3000)}` }); } - t.push({ role: "user", content: "[molt] You indicated the task is complete, but 1 of 3 checks did not pass." }); + t.pushBarFailure("[molt] You indicated the task is complete, but 1 of 3 checks did not pass."); for (let i = 0; i < 40; i++) { t.push({ role: "assistant", content: null, tool_calls: [toolCall("bash", { command: `t${i}` }, `b${i}`)] }); t.push({ role: "tool", tool_call_id: `b${i}`, content: `${i}:${"p".repeat(3000)}` }); @@ -110,6 +110,65 @@ describe("shed min-free (default)", () => { // Two of four exchanges go: the cut is the user turn, not the recent fallback. assert.equal(plan.droppedCount, 22); }); + + it("keeps the criteria and the live bar refusal verbatim through a fallback shed (review probe)", () => { + const t = new Transcript("SYSTEM"); + t.push({ role: "user", content: "Fix the duration parser so `1h30m` parses." }); + t.push({ role: "assistant", content: null, tool_calls: [toolCall("read_file", { path: "dur.py" }, "r0")] }); + t.push({ role: "tool", tool_call_id: "r0", content: "f".repeat(800) }); + const criteria = `Acceptance criteria for this task: ${"c".repeat(560)} CRITERIA_END`; + t.push({ role: "user", content: criteria, molt: { criteria: true } }); + for (let i = 0; i < 40; i++) { + t.push({ role: "assistant", content: null, tool_calls: [toolCall("bash", { command: `s${i}` }, `a${i}`)] }); + t.push({ role: "tool", tool_call_id: `a${i}`, content: `${i}:${"o".repeat(3000)}` }); + } + const failure = `[molt] attempt 1 was refused: ${"e".repeat(1150)} FAILURE_END`; + t.pushBarFailure(failure); + const steps = (from: number, n: number) => { + for (let i = from; i < from + n; i++) { + t.push({ role: "assistant", content: null, tool_calls: [toolCall("bash", { command: `t${i}` }, `b${i}`)] }); + t.push({ role: "tool", tool_call_id: `b${i}`, content: `${i}:${"p".repeat(3000)}` }); + } + }; + steps(0, 4); + const plan = t.planShed()!; + assert.ok(plan.afterTokens / plan.beforeTokens < 0.5, `${plan.beforeTokens} -> ${plan.afterTokens}`); + assert.equal(plan.carried.length, 2); + t.commitShed(plan); + const onWire = () => t.wire().map((m) => m.content ?? ""); + assert.ok(onWire().includes(failure), "the refusal's detail was digested"); + assert.ok(onWire().includes(criteria), "the criteria were digested"); + // And the next shed, a few steps on, keeps them too. + steps(4, 10); + const again = t.planShed()!; + t.commitShed(again); + assert.ok(onWire().includes(failure) && onWire().includes(criteria), "lost on the second shed"); + // A stale refusal is not carried: the next one replaces it. + t.pushBarFailure("[molt] attempt 2 was refused: SECOND"); + steps(14, 10); + t.commitShed(t.planShed()!); + assert.ok(!onWire().includes(failure)); + assert.ok(onWire().some((c) => c.includes("SECOND")) && onWire().includes(criteria)); + }); + + it("an interactive session whose last two exchanges hold most of the history is unchanged", () => { + const t = new Transcript("S"); + for (let k = 0; k < 3; k++) { + t.push({ role: "user", content: `ask ${k}` }); + const n = k === 0 ? 1 : 15; + for (let i = 0; i < n; i++) { + t.push({ role: "assistant", content: null, tool_calls: [toolCall("bash", { command: `${k}.${i}` }, `c${k}.${i}`)] }); + t.push({ role: "tool", tool_call_id: `c${k}.${i}`, content: "x".repeat(2000) }); + } + } + const plan = t.planShed()!; + // The first exchange alone goes, though it frees well under 25%: the + // two kept exchanges are the person's, not Maat's notes. + assert.ok(plan.beforeTokens - plan.afterTokens < 0.25 * plan.beforeTokens); + assert.equal(plan.cutAt, 3); + assert.equal(plan.droppedCount, 3); + assert.equal(plan.carried.length, 0); + }); }); describe("per-call elision (default)", () => { @@ -195,4 +254,29 @@ describe("per-call elision (default)", () => { ws.cleanup(); } }); + + it("a plain cat whose rerun came back as a pointer is still invalidated by a write", () => { + const t = new Transcript("S"); + step(t, [["bash", { command: "cat dur.py" }, "a", body("DUR_OLD")]]); + step(t, [["bash", { command: "cat dur.py" }, "b", "[molt: this is the same bash call you made at step 1, and nothing has changed since.]"]]); + step(t, [["write_file", { path: "dur.py", content: "new" }, "w", "wrote"]]); + assert.equal(t.elideSupersededReads().elided, 1); + assert.ok(!JSON.stringify(t.wire()).includes("DUR_OLD")); + }); + + it("the rerun key does not depend on argument order", () => { + const t = new Transcript("S"); + t.push({ role: "assistant", content: null, tool_calls: [{ id: "a", type: "function", function: { name: "bash", arguments: '{"timeout_s":600,"command":"make"}' } }] }); + t.push({ role: "tool", tool_call_id: "a", content: body("1 error") }); + t.push({ role: "assistant", content: null, tool_calls: [{ id: "b", type: "function", function: { name: "bash", arguments: '{"command":"make","timeout_s":600}' } }] }); + t.push({ role: "tool", tool_call_id: "b", content: body("built") }); + assert.equal(t.elideSupersededReads().elided, 1); + }); + + it("a background rerun is left alone: its result is the handle for the job", () => { + const t = new Transcript("S"); + step(t, [["bash", { command: "npm run dev", background: true }, "a", body("started job 1")]]); + step(t, [["bash", { command: "npm run dev", background: true }, "b", body("started job 2")]]); + assert.equal(t.elideSupersededReads().elided, 0); + }); }); From 70b7b8c434ff504eb75ce763feda3c42c7d34f6a Mon Sep 17 00:00:00 2001 From: Tyler Skelton Date: Thu, 8 Oct 2026 01:41:32 -0700 Subject: [PATCH 4/5] read_file: a pointer repeat keeps the copy it points at (HELD: raises lean-budget c) Review item 3 on #51. A repeat read_file answered with an 'already shown' or same-call pointer no longer marks the original as re-read, so the model is not told to scroll up to an elision marker; the original stays the copy a later write invalidates. Not merged into claude/lean-fixes: lean-budget scenario c (the same 40 KB file read 8 times) goes from 40,732 / 115,399 / 9 to 44,725 / 348,643 / 9 (largest / total / steps) and fails its ceiling. The base number is low only because the original was elided under its own pointer. --- src/transcript.ts | 7 +++++++ test/lean-fixes.test.ts | 12 ++++++++++++ 2 files changed, 19 insertions(+) diff --git a/src/transcript.ts b/src/transcript.ts index e50d5de..b52f25c 100644 --- a/src/transcript.ts +++ b/src/transcript.ts @@ -26,6 +26,8 @@ export const STALE_FAILURE_PREFIX = "[molt: superseded]"; export const ELIDED_PREFIX = "[molt: superseded tool result —"; /** How the engine sends a repeated call whose result has not changed (see engine.ts). */ const SAME_CALL_PREFIX = "[molt: this is the same "; +/** How the engine sends a read of lines the model has already been shown. */ +const SHOWN_PREFIX = "[molt: you have already been shown"; /** * A bash command that only prints one file: `cat f`, `head -n 40 f`, @@ -617,6 +619,11 @@ export class Transcript { if (!path) return; if (call.function.name === "read_file") { + // A repeat the engine answered with a pointer ("already shown", + // "the same call") points at the earlier copy: that copy must stay, + // and stays the one a later write invalidates. + const got = at >= 0 ? (this.working[at].content ?? "") : ""; + if (got.startsWith(SAME_CALL_PREFIX) || got.startsWith(SHOWN_PREFIX)) return; // Identical arguments return identical bytes; anything else is a // different part of the file and stands on its own. const window = `${path}@${Number(args.offset ?? 0)}+${String(args.limit ?? "all")}`; diff --git a/test/lean-fixes.test.ts b/test/lean-fixes.test.ts index 64bb19b..622ad00 100644 --- a/test/lean-fixes.test.ts +++ b/test/lean-fixes.test.ts @@ -264,6 +264,18 @@ describe("per-call elision (default)", () => { assert.ok(!JSON.stringify(t.wire()).includes("DUR_OLD")); }); + it("a read_file answered with a pointer keeps the copy it points at, and a write still finds it", () => { + const t = new Transcript("S"); + step(t, [["read_file", { path: "dur.py" }, "a", body("DUR_OLD")]]); + step(t, [["read_file", { path: "dur.py" }, "b", "[molt: you have already been shown lines 1-40 of dur.py. Scroll up.]"]]); + step(t, [["read_file", { path: "dur.py" }, "c", "[molt: this is the same read_file call you made at step 1, and nothing has changed since.]"]]); + assert.equal(t.elideSupersededReads().elided, 0, "the original was elided under its own pointer"); + assert.ok(JSON.stringify(t.wire()).includes("DUR_OLD")); + step(t, [["write_file", { path: "dur.py", content: "new" }, "w", "wrote"]]); + assert.equal(t.elideSupersededReads().elided, 1); + assert.ok(!JSON.stringify(t.wire()).includes("DUR_OLD")); + }); + it("the rerun key does not depend on argument order", () => { const t = new Transcript("S"); t.push({ role: "assistant", content: null, tool_calls: [{ id: "a", type: "function", function: { name: "bash", arguments: '{"timeout_s":600,"command":"make"}' } }] }); From 162be3334d02d8b3db938e86ad4c648809eae8b0 Mon Sep 17 00:00:00 2001 From: Tyler Skelton Date: Thu, 8 Oct 2026 01:53:45 -0700 Subject: [PATCH 5/5] lean-budget: re-read ceiling reflects keeping one real copy The old 115,399 total was only reachable because the original read was elided under its own 'already shown' pointer, so the model had no copy. With the pointer fix the scenario carries one 40k copy per step. --- test/lean-budget.test.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/lean-budget.test.ts b/test/lean-budget.test.ts index c607a74..55a1b24 100644 --- a/test/lean-budget.test.ts +++ b/test/lean-budget.test.ts @@ -127,7 +127,10 @@ function within(name: string, m: Measured, c: Ceiling): void { const CEILINGS = { small: { largest: 11_700, total: 34_000, steps: 5 }, // 7,805 / 22,690 / 3 bigRead: { largest: 62_000, total: 195_000, steps: 6 }, // 41,234 / 130,307 / 4 - reread: { largest: 61_000, total: 173_000, steps: 14 }, // 40,732 / 115,399 / 9 + // 2026-10-08: was 40,732 / 115,399 / 9, but only because the original copy was + // elided under its own "already shown" pointer, leaving the model no copy at + // all (Grok #51 item 3). One kept 40k copy per step is the honest cost. + reread: { largest: 61_000, total: 420_000, steps: 14 }, // 44,725 / 348,643 / 9 longShed: { largest: 390_000, total: 10_900_000, steps: 92 }, // 259,597 / 7,262,564 / 61 malformedRaw: { largest: 18_700, total: 104_000, steps: 11 }, // 12,450 / 69,197 / 7 malformedAct: { largest: 14_100, total: 73_000, steps: 11 }, // 9,384 / 48,455 / 7