Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
72 changes: 72 additions & 0 deletions scripts/lean-replay-extract.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
#!/usr/bin/env python3
"""Extract a lean-sessions replay fixture from a bench run: exuviae (verbatim turns) plus the journal tail.

Usage: python3 scripts/lean-replay-extract.py <run-dir> <out.json>
See test/fixtures/README.md and reports/lean-sessions-study.md."""
import re, json, sys, glob, os
run = sys.argv[1]; out = sys.argv[2]
ex = sorted(glob.glob(os.path.join(run, '.maat/exuviae/0*.md')))
msgs = []
for fn in ex:
t = open(fn).read()
parts = re.split(r'\n## (user|assistant|tool|system)( \((?:digest|regrown)\))?\n', t)
i = 1
while i < len(parts):
role, tag, body = parts[i], parts[i+1], parts[i+2]; i += 3
if tag: continue # digest of an earlier batch: not a real message
m = re.search(r'\n```json\n(\[.*\])\n```\n?$', body, re.S)
calls = json.loads(m.group(1)) if m else []
content = (body[:m.start()] if m else body).strip('\n')
msgs.append({'role': role, 'content': content, 'calls': calls})
# Strip molt's own clock/notes from tool results: the replay engine adds its own.
def clean(s):
return re.sub(r'\n\[molt: \d+m of \d+m used\]$', '', s)
steps = []; pending_text = ''; user_msgs = []
task = next(m['content'] for m in msgs if m['role']=='user')
i = 0
while i < len(msgs):
m = msgs[i]
if m['role'] == 'user':
user_msgs.append(len(m['content'])); i += 1; continue
if m['role'] == 'assistant':
if not m['calls']:
pending_text += (('\n\n' if pending_text else '') + m['content']); i += 1; continue
results = []
j = i + 1
while j < len(msgs) and msgs[j]['role'] == 'tool':
results.append(clean(msgs[j]['content'])); j += 1
calls = []
for k, c in enumerate(m['calls']):
calls.append({'name': c['function']['name'], 'args': json.loads(c['function']['arguments'] or '{}'),
'result': results[k] if k < len(results) else ''})
text = ((pending_text + '\n\n') if pending_text else '') + m['content']
steps.append({'text': text.strip(), 'calls': calls}); pending_text = ''
i = j; continue
i += 1
covered_calls = sum(len(s['calls']) for s in steps)
# The rest of the session, from the journal: commands and result sizes only.
journal = glob.glob(os.path.join(run, '.maat/log/*.jsonl'))[0]
ev = [json.loads(l) for l in open(journal)]
seq = [] # (step, name, detail, bytes, contentChars)
cur = None
for e in ev:
k, d = e['kind'], e['data']
if k == 'response': cur = {'contentChars': d.get('contentChars', 0), 'calls': []}; seq.append(cur)
elif k == 'tool_call' and cur is not None: cur['calls'].append({'name': d['name'], 'detail': d.get('detail', '')})
elif k == 'tool_result' and cur is not None and cur['calls']: cur['calls'][-1]['bytes'] = d.get('bytes', 0)
journal_calls = [c for s in seq for c in s['calls']]
tail = []
n = 0
for s in seq:
if not s['calls']: continue
if n >= covered_calls:
tail.append({'textChars': s['contentChars'], 'calls': [{'name': c['name'], 'detail': c['detail'], 'bytes': c.get('bytes', 0)} for c in s['calls']]})
n += len(s['calls'])
json.dump({'source': os.path.relpath(run, os.path.expanduser('~/.cache/maat-bench/container-work')),
'note': 'steps: real assistant turns and tool results recovered from the run\'s exuviae; tail: the rest of the run from its journal (command and result size only)',
'task': task, 'droppedUserMessages': user_msgs[1:], 'steps': steps, 'tail': tail}, open(out, 'w'), indent=1)
print('steps', len(steps), 'calls', covered_calls, 'journal calls', len(journal_calls), 'tail steps', len(tail), 'user msgs dropped', user_msgs)
print('exuvia step names', sorted(set(c['name'] for s in steps for c in s['calls'])))
# sanity: do the exuvia call details line up with the journal's?
for a, b in list(zip([c for s in steps for c in s['calls']], journal_calls))[:200]:
if a['name'] != b['name']: print('MISMATCH', a['name'], b['name']); break
72 changes: 72 additions & 0 deletions scripts/lean-sessions-measure.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
#!/usr/bin/env node
/**
* Measure the lean-sessions prototypes: run test/lean-sessions.test.ts once per
* env configuration and print a markdown table of the results.
*
* npx tsc -p tsconfig.test.json && node scripts/lean-sessions-measure.mjs [--json out.json]
*
* No model, no network: every scenario is scripted (see the test file).
*/
import { spawnSync } from "node:child_process";
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";

const CONFIGS = [
["baseline", {}],
// 1. the shed threshold, and the shed that frees nothing
["shed-at-40k", { MAAT_LEAN_SHED_AT: "40000" }],
["shed-at-30k", { MAAT_LEAN_SHED_AT: "30000" }],
["shed-at-20k", { MAAT_LEAN_SHED_AT: "20000" }],
["shed-minfree", { MAAT_LEAN_SHED_MINFREE: "0.25" }],
// 2. ageing older results (and 3. long arguments)
["age", { MAAT_LEAN_AGE: "1" }],
["age+args", { MAAT_LEAN_AGE: "1", MAAT_LEAN_AGE_ARGS: "1" }],
["age+args-keep4-batch12k", { MAAT_LEAN_AGE: "keep=4,batch=12000", MAAT_LEAN_AGE_ARGS: "1" }],
["age+args-batch48k", { MAAT_LEAN_AGE: "batch=48000", MAAT_LEAN_AGE_ARGS: "1" }],
["soft-shed-30k", { MAAT_LEAN_AGE: "at=30000,batch=0", MAAT_LEAN_AGE_ARGS: "1" }],
["soft-shed-20k", { MAAT_LEAN_AGE: "at=20000,batch=0", MAAT_LEAN_AGE_ARGS: "1" }],
// superseded output
["supersede", { MAAT_LEAN_SUPERSEDE: "1" }],
// together
["shed-at-30k+minfree+supersede", { MAAT_LEAN_SHED_AT: "30000", MAAT_LEAN_SHED_MINFREE: "0.25", MAAT_LEAN_SUPERSEDE: "1" }],
["all (age+args, supersede, minfree)", { MAAT_LEAN_AGE: "1", MAAT_LEAN_AGE_ARGS: "1", MAAT_LEAN_SUPERSEDE: "1", MAAT_LEAN_SHED_MINFREE: "0.25" }],
];

const only = process.argv.includes("--only") ? process.argv[process.argv.indexOf("--only") + 1].split(",") : null;
const jsonOut = process.argv.includes("--json") ? process.argv[process.argv.indexOf("--json") + 1] : null;
const dir = mkdtempSync(join(tmpdir(), "lean-sessions-"));
const all = {};
try {
for (const [name, flags] of CONFIGS) {
if (only && !only.includes(name)) continue;
const out = join(dir, `${name}.json`);
const env = { ...process.env, ...flags, LEAN_SESSIONS_OUT: out, MOLT_CONFIG_DIR: dir };
for (const k of Object.keys(env)) if (k.startsWith("MAAT_LEAN_") && !(k in flags)) delete env[k];
const r = spawnSync("node", ["--test", "dist-test/test/lean-sessions.test.js"], { env, encoding: "utf8" });
const ok = r.status === 0;
const res = JSON.parse(readFileSync(out, "utf8"));
all[name] = { flags, ok, res };
if (!ok) process.stderr.write(`${name}: test FAILED\n${r.stdout.slice(-3000)}\n`);
}
} finally {
rmSync(dir, { recursive: true, force: true });
}

const fmt = (n) => n.toLocaleString("en-US");
const delta = (n, b) => (b ? `${n <= b ? "−" : "+"}${Math.abs(Math.round((100 * (b - n)) / b))}%` : "");
const base = all.baseline?.res;
for (const scenario of Object.keys(Object.values(all)[0].res)) {
console.log(`\n### ${scenario}\n`);
console.log("| config | ok | steps | largest | total chars | vs base | cache-weighted | vs base | cacheable | cache breaks | sheds | ageings |");
console.log("|---|---|---|---|---|---|---|---|---|---|---|---|");
for (const [name, { ok, res }] of Object.entries(all)) {
const s = res[scenario];
if (!s) continue;
const b = base?.[scenario];
console.log(
`| ${name} | ${ok ? "yes" : "NO"} | ${s.steps} | ${fmt(s.largest)} | ${fmt(s.total)} | ${b ? delta(s.total, b.total) : ""} | ${fmt(s.weighted)} | ${b ? delta(s.weighted, b.weighted) : ""} | ${(100 * s.cacheable).toFixed(1)}% | ${s.breaks.length} | ${s.sheds} | ${s.ageings} |`,
);
}
}
if (jsonOut) writeFileSync(jsonOut, JSON.stringify(all, null, 1));
43 changes: 39 additions & 4 deletions src/engine.ts
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,7 @@ import {
} from "./types.js";
import { stateDir, stateDirName } from "./statedir.js";
import { env } from "./env.js";
import { leanAging, leanShedAt, leanShedMinFree, leanSupersede } from "./lean.js";
import { Judgments, caseReason, reasonText } from "./judgment.js";

/** A reading of the session meter, for measuring one job against. */
Expand Down Expand Up @@ -2718,7 +2719,7 @@ export class Engine {

/** The history size a shed is triggered at, in molt's own token units. 0 is off. */
get autoShedAtTokens(): number {
return this.cfg.autoShedAtTokens ?? DEFAULT_AUTO_SHED_TOKENS;
return this.cfg.autoShedAtTokens ?? leanShedAt() ?? DEFAULT_AUTO_SHED_TOKENS;
}

setAutoShed(tokens: number): void {
Expand Down Expand Up @@ -3012,9 +3013,9 @@ export class Engine {
* else molt writes, and return the path the model should read. Null if it
* could not be written — the preview still stands on its own.
*/
private spill(text: string, callId: string): string | null {
private spill(text: string, callId: string, prefix = ""): string | null {
try {
const rel = `${stateDirName(this.cwd)}/out/${callId.replace(/[^\w-]/g, "_")}.txt`;
const rel = `${stateDirName(this.cwd)}/out/${prefix}${callId.replace(/[^\w-]/g, "_")}.txt`;
mkdirSync(stateDir(this.cwd, "out"), { recursive: true });
writeFileSync(join(this.cwd, rel), this.maskWithheld(redact(text, this.secrets())), "utf8");
return rel;
Expand Down Expand Up @@ -6131,6 +6132,7 @@ export class Engine {
// as it always did.
const pruned = this.transcript.elideSupersededReads({
protectCache: this.sessionCached > 0,
lean: leanSupersede(),
});
if (pruned.elided > 0) {
log?.append("elide", {
Expand All @@ -6147,7 +6149,40 @@ export class Engine {
}
}

const auto = this.subprocess ? 0 : this.cfg.autoShedAtTokens ?? DEFAULT_AUTO_SHED_TOKENS;
// Ageing (lean-sessions prototype, MAAT_LEAN_AGE): older tool results go
// on the wire as their ends and a pointer to the full text, in batches.
const aging = this.subprocess ? null : leanAging();
if (aging) {
const plan = this.transcript.planAging(aging);
if (plan) {
const kept = new Map<string, string | null>();
for (const r of plan.results) kept.set(`r:${r.callId}`, this.spill(r.full, r.callId, "aged-"));
for (const c of plan.calls) kept.set(`c:${c.callId}`, this.spill(c.full, c.callId, "aged-call-"));
this.transcript.commitAging(plan, aging, (id, kind) => kept.get(`${kind === "call" ? "c" : "r"}:${id}`) ?? null);
// What molt believed it had "already shown" is no longer on the
// wire in full: a re-read must be served, not refused (as after a shed).
shown.clear();
answered.clear();
log?.append("note", {
kind: "age",
text: "aged older tool results",
results: plan.results.length,
calls: plan.calls.length,
chars: plan.saving,
});
if (plan.thenShed) {
const shed = this.shed();
if (shed) yield { kind: "shed", ...shed };
}
yield {
kind: "info",
text: `aged ${plan.results.length} older tool result(s)${plan.calls.length ? ` and ${plan.calls.length} call(s)` : ""} · ~${Math.round(plan.saving / 4)} tokens no longer resent; the full text is in ${stateDirName(this.cwd)}/out/`,
};
}
}

this.transcript.shedMinFree = leanShedMinFree();
const auto = this.subprocess ? 0 : this.cfg.autoShedAtTokens ?? leanShedAt() ?? DEFAULT_AUTO_SHED_TOKENS;
if (auto > 0 && this.transcript.historyTokens() > auto) {
const shed = this.shed();
if (shed) {
Expand Down
65 changes: 65 additions & 0 deletions src/lean.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
/**
* Lean-sessions prototypes: env flags only, every one off by default.
*
* These are measured in test/lean-sessions.test.ts and written up in
* reports/lean-sessions-study.md. None of them changes a default; each is a
* candidate for one, kept switchable so the suite and the bench can compare
* a session with and without it.
*
* MAAT_LEAN_SHED_AT=<tokens> auto-shed threshold when none is configured
* (default 60,000)
* MAAT_LEAN_SHED_MINFREE=<0..1> a shed cut on user turns must free this
* share of the history, or it cuts on recent
* messages instead
* MAAT_LEAN_AGE=1 | k=v,... age older tool results (see AgingOpts):
* keep, min, batch, head, tail, argmin, at
* MAAT_LEAN_AGE_ARGS=1 with MAAT_LEAN_AGE, age long tool-call
* arguments too
* MAAT_LEAN_SUPERSEDE=1 elide per call (not per step), and treat a
* bash rerun with new output, or a plain
* `cat`/`sed -n` of a file written since, as
* superseded
*/
import { env } from "./env.js";
import type { AgingOpts } from "./transcript.js";

const on = (v: string | undefined) => v !== undefined && v !== "" && v !== "0" && v.toLowerCase() !== "false";

export function leanShedAt(): number | undefined {
const n = Number(env("LEAN_SHED_AT"));
return Number.isFinite(n) && n > 0 ? Math.floor(n) : undefined;
}

export function leanShedMinFree(): number {
const n = Number(env("LEAN_SHED_MINFREE"));
return Number.isFinite(n) && n > 0 && n < 1 ? n : 0;
}

export function leanSupersede(): boolean {
return on(env("LEAN_SUPERSEDE"));
}

export const AGING_DEFAULTS: AgingOpts = {
keep: 6,
minChars: 1_500,
batchChars: 24_000,
head: 600,
tail: 300,
args: false,
argMinChars: 1_000,
atTokens: 0,
};

export function leanAging(): AgingOpts | null {
const raw = env("LEAN_AGE");
if (!on(raw)) return null;
const o: AgingOpts = { ...AGING_DEFAULTS, args: on(env("LEAN_AGE_ARGS")) };
const keys: Record<string, keyof AgingOpts> = { keep: "keep", min: "minChars", batch: "batchChars", head: "head", tail: "tail", argmin: "argMinChars", at: "atTokens" };
for (const part of raw!.split(",")) {
const [k, v] = part.split("=");
const key = keys[(k ?? "").trim()];
const n = Number(v);
if (key && Number.isFinite(n) && n >= 0) (o as Record<string, number | boolean>)[key] = Math.floor(n);
}
return o;
}
Loading
Loading