From c51a057d79f276dc60f8f44b31066416159acb31 Mon Sep 17 00:00:00 2001 From: Tyler Skelton Date: Wed, 7 Oct 2026 22:33:44 -0700 Subject: [PATCH 1/2] Refused tool calls go back as a short excerpt; six malformed calls in a row end the turn A worker that sent 17 malformed act calls had every one resent in full on each step: in the call itself, in the complaint quoting it, and in the shed digest. Prompts reached 630k tokens and one task cost $0.82 for a 181-byte file. The record and captures keep the full text; the wire gets an excerpt. --- src/engine.ts | 39 +++++++++++++++++++-- src/transcript.ts | 42 +++++++++++++++++++--- test/malformed-cap.test.ts | 72 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 146 insertions(+), 7 deletions(-) create mode 100644 test/malformed-cap.test.ts diff --git a/src/engine.ts b/src/engine.ts index fa2f83e..ff2eb5b 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -112,7 +112,7 @@ import { takeTurn } from "./localgate.js"; import { parseLenient } from "./lenient-json.js"; import { LONG_RATE_LIMIT_MS, longQuotaText, rateLimitResetAt, untilText, providerErrorText, normalizeMessage, readStream, transientProviderError, type ProviderError, type StreamAccumulator, type Usage } from "./stream.js"; import { Fragments, SafeStream } from "./live.js"; -import { Transcript, toolDetail } from "./transcript.js"; +import { Transcript, excerpt, toolDetail } from "./transcript.js"; import { acpAgentFor, acpHealth, acpModels, AcpSession, backendStallMs, isAcp } from "./acp.js"; import { type BackendSession, type ToolRunner } from "./backend.js"; import { endpointProblem, removedSubscriptionProblem } from "./endpoint.js"; @@ -301,6 +301,10 @@ const CEILING_WARNINGS = [0.5, 0.8]; * original is preserved in `.maat/exuviae/`, `record-intact` fails if it is * not, and `/regrow` pulls it back by pattern. */ +/** Malformed tool calls in a row before the model is told firmly, and before the turn ends. */ +export const MALFORMED_WARN = 3; +export const MALFORMED_STOP = 6; + export const DEFAULT_AUTO_SHED_TOKENS = 60_000; /** @@ -2351,6 +2355,13 @@ export class Engine { * ran; two of them had passing work on disk. */ private turnEndedBy: "deadline" | "provider" | undefined; + /** + * Malformed tool calls in a row this turn. One is a slip; a run that kept + * sending broken `act` calls made 17 in a row and paid for every one + * (2026-10-07). After MALFORMED_WARN the model is told firmly; after + * MALFORMED_STOP the turn ends and the work on disk is judged. + */ + private malformedStreak = 0; /** A subprocess backend went silent past the stall allowance this turn (a provider issue). */ private turnProviderStall = false; /** `--revert` put this turn's work back, so a person cannot accept it as it stands. */ @@ -4362,6 +4373,7 @@ export class Engine { this.turnAllRetired = false; this.turnRevealed = []; this.turnEndedBy = undefined; + this.malformedStreak = 0; this.turnProviderStall = false; this.turnRestored = false; this.refusedThisTurn = false; @@ -5055,6 +5067,10 @@ export class Engine { // sent. Say what actually happened instead. malformed = true; } + // An act reaches a tool call as itself only when its actions could not be + // read (expandAct), so it counts as malformed here too. + const refused = malformed || name === "act"; + this.malformedStreak = refused ? this.malformedStreak + 1 : 0; const detail = toolDetail(name, args); this.turnCalls.add(callId); // What the current autonomy level says about this exact call. The @@ -5128,8 +5144,13 @@ export class Engine { ? `[molt: the arguments for ${name} do not fit its schema, so nothing ran: ` + `${violations.join("; ")}. Send the call again with them fixed.]` : `[molt: the arguments for ${name} were not valid JSON, so nothing ran. ` + - `Send them again as a JSON object. What arrived was: ${raw}]`; - this.transcript.push({ role: "tool", tool_call_id: callId, content: complaint }); + `Send them again as a JSON object. What arrived began: ${excerpt(raw)}]`; + const firm = + this.malformedStreak >= MALFORMED_WARN + ? ` [molt: that is ${this.malformedStreak} malformed calls in a row. Send ONE call whose arguments are a single JSON object matching the ${name} schema, or finish; after ${MALFORMED_STOP} the turn ends.]` + : ""; + this.transcript.push({ role: "tool", tool_call_id: callId, content: complaint + firm }); + this.transcript.markMalformedCall(callId); return { name, result: complaint, auto: true, repeated: false }; } // Timed around execution only. Waiting on a human to approve a gated @@ -5303,6 +5324,7 @@ export class Engine { auto: !decision.ask, }; this.transcript.push({ role: "tool", tool_call_id: callId, content: result }); + if (refused) this.transcript.markMalformedCall(callId); return { name, result, auto: !decision.ask, repeated: repeatedHere }; } @@ -5934,6 +5956,17 @@ export class Engine { // whatever state the work is in when they are up is the state the judge // sees. Unlike a token ceiling it is not a proxy for anything — it is // the thing the person waiting actually spends. + if (this.malformedStreak >= MALFORMED_STOP) { + log?.append("note", { text: `ended: malformed tool calls (${this.malformedStreak} in a row)`, malformedStop: true }); + yield { + kind: "info", + text: `ended: ${this.malformedStreak} malformed tool calls in a row — no more tool calls this turn. The work on disk is judged as it stands.`, + }; + this.turnEndedBy = "provider"; + const judged = this.turnWrites.length > 0 ? yield* judgeOnDisk("provider", "") : false; + if (!judged) yield* this.salvage("You sent several malformed tool calls in a row, so this turn has ended.", fetchFn, log); + return; + } if (deadlineInterrupted || this.pastDeadline()) { const spentMs = Date.now() - this.turnStartedAt; log?.append("deadline", { limitMs: this.turnDeadlineMs, spentMs }); diff --git a/src/transcript.ts b/src/transcript.ts index 2b8d995..5d58cdc 100644 --- a/src/transcript.ts +++ b/src/transcript.ts @@ -70,6 +70,14 @@ export class Transcript { private archived: Msg[][] = []; /** What this turn is for. Sent every request, shed never. */ private task: string | null = null; + /** + * Tool calls Maat refused as malformed. Their arguments go back to the + * provider as a short excerpt, not in full: a run that made 17 malformed + * `act` calls in a row resent every one of them on every step, prompts grew + * to 630k tokens, and one task cost $0.82 for a 181-byte file (2026-10-07). + * The record and captures keep the full text. + */ + private malformedCalls = new Set(); constructor(systemPrompt: string) { this.system = { role: "system", content: systemPrompt }; @@ -79,6 +87,11 @@ export class Transcript { this.working.push(msg); } + /** Send this call's arguments as an excerpt from now on (see `malformedCalls`). */ + markMalformedCall(id: string): void { + this.malformedCalls.add(id); + } + /** * Fold the tool results for `parts` into one result for the call `id`. * @@ -159,7 +172,12 @@ export class Transcript { // in order; a system message anywhere later goes as a user message. const out: Omit[] = []; for (const { molt: _molt, ...m } of this.all()) { - if (m.role !== "system") out.push(repair && Array.isArray(m.tool_calls) ? { ...m, tool_calls: m.tool_calls.map(wireCall) } : m); + if (m.role !== "system") + out.push( + repair && Array.isArray(m.tool_calls) + ? { ...m, tool_calls: m.tool_calls.map((c) => (c.id && this.malformedCalls.has(c.id) ? excerptCall(c) : wireCall(c))) } + : m, + ); else if (out.length === 0) out.push({ ...m }); else if (out.length === 1 && out[0]!.role === "system") out[0] = { ...out[0]!, content: `${out[0]!.content ?? ""}\n\n${m.content ?? ""}` }; else out.push({ role: "user", content: m.content ?? "" }); @@ -556,7 +574,8 @@ export function buildDigest(dropped: Msg[]): string { } catch { detail = "(unparseable arguments)"; } - actions.push(`${c.function.name}: ${detail}`); + // Capped: an act whose actions arrived as one huge string printed all of it here. + actions.push(`${c.function.name}: ${cap(detail, 200)}`); } } @@ -653,11 +672,26 @@ export function wireArgs(text: string): string { } catch { /* fall through */ } - return JSON.stringify({ _unparsed: text }); + return JSON.stringify({ _unparsed: excerpt(text) }); } if (isObject(v)) return text; if (v === null) return "{}"; - return JSON.stringify({ _unparsed: text }); + return JSON.stringify({ _unparsed: excerpt(text) }); +} + +/** How much of a refused call's arguments goes back to the provider. */ +export const MALFORMED_EXCERPT_CHARS = 300; + +/** The start of `text`, and how much was left out. Enough for the model to see its own mistake. */ +export function excerpt(text: string, max = MALFORMED_EXCERPT_CHARS): string { + return text.length <= max ? text : `${text.slice(0, max)}…[${text.length - max} more characters not resent]`; +} + +/** A refused call as it goes back to the provider: always an object, never the whole text. */ +function excerptCall(c: T): T { + const a = c.function?.arguments; + const text = typeof a === "string" ? a : JSON.stringify(a ?? {}); + return { ...c, function: { ...c.function!, arguments: JSON.stringify({ _refused: excerpt(text) }) } }; } function wireCall(c: T): T { diff --git a/test/malformed-cap.test.ts b/test/malformed-cap.test.ts new file mode 100644 index 0000000..2b8746f --- /dev/null +++ b/test/malformed-cap.test.ts @@ -0,0 +1,72 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; +import { Engine, MALFORMED_STOP } from "../src/engine.js"; +import { Transcript, excerpt, MALFORMED_EXCERPT_CHARS, wireArgs } from "../src/transcript.js"; +import { allowAll, drain, scriptedProvider, workspace } from "./helpers.js"; + +/** + * 2026-10-07: a worker sent 17 malformed `act` calls in a row, each carrying a + * large argument payload. Every one went back to the provider in full on + * every later step; prompts reached 630k tokens and one task cost $0.82 for a + * 181-byte file. Refused calls now go back as an excerpt, and the turn ends + * after MALFORMED_STOP of them in a row. + */ +const BIG = "x".repeat(50_000); + +describe("malformed tool calls stay cheap", () => { + it("excerpt keeps the start and says how much was left out", () => { + assert.equal(excerpt("short"), "short"); + const e = excerpt(BIG); + assert.ok(e.length < MALFORMED_EXCERPT_CHARS + 60); + assert.match(e, /49700 more characters not resent/); + }); + + it("unreadable arguments go back as a capped _unparsed", () => { + const w = JSON.parse(wireArgs(`{"broken": ${BIG}`)) as { _unparsed: string }; + assert.ok(w._unparsed.length < MALFORMED_EXCERPT_CHARS + 60); + }); + + it("a refused call's arguments are excerpted on the wire, kept whole in the record", () => { + const t = new Transcript("sys"); + t.push({ role: "user", content: "go" }); + t.push({ role: "assistant", content: null, tool_calls: [{ id: "c1", type: "function", function: { name: "act", arguments: JSON.stringify({ actions: BIG }) } }] }); + t.push({ role: "tool", tool_call_id: "c1", content: "refused" }); + t.markMalformedCall("c1"); + const sent = JSON.stringify(t.wire()); + assert.ok(sent.length < 2_000, `wire was ${sent.length} chars`); + assert.ok(JSON.stringify(t.wire({ repairArgs: false })).length > 50_000, "the record keeps the full text"); + }); + + it("ten 50 KB malformed calls never make a request carry them in full, and the turn ends at the limit", async () => { + const ws = workspace(); + try { + const turns = [ + ...Array.from({ length: 10 }, () => ({ calls: [{ name: "act", args: { actions: BIG } }] })), + { text: "Done." }, + ]; + const provider = scriptedProvider(turns); + const engine = new Engine({ baseUrl: "http://p.test/v1", model: "m", cwd: ws.dir, fetchFn: provider.fetchFn, bar: null, stream: false, autonomy: "high" }); + const ev = await drain(engine.run("write out.txt", allowAll)); + const sizes = provider.bodies.map((b) => b.length); + assert.ok(Math.max(...sizes) < 40_000, `largest request ${Math.max(...sizes)} chars`); + assert.ok(provider.calls <= MALFORMED_STOP + 2, `provider asked ${provider.calls} times`); + assert.ok(ev.some((e) => e.kind === "info" && /malformed tool calls in a row/.test(e.text))); + } finally { + ws.cleanup(); + } + }); + + it("a good call in between resets the count", async () => { + const ws = workspace(); + try { + const bad = { calls: [{ name: "act", args: { actions: "not a list" } }] }; + const good = { calls: [{ name: "list_dir", args: { path: "." } }] }; + const provider = scriptedProvider([bad, bad, bad, bad, bad, good, bad, bad, bad, bad, bad, { text: "Done." }]); + const engine = new Engine({ baseUrl: "http://p.test/v1", model: "m", cwd: ws.dir, fetchFn: provider.fetchFn, bar: null, stream: false, autonomy: "high" }); + const ev = await drain(engine.run("look around", allowAll)); + assert.ok(!ev.some((e) => e.kind === "info" && /malformed tool calls in a row/.test(e.text))); + } finally { + ws.cleanup(); + } + }); +}); From 7a49ff17245d5dc5399677c924088ae5591602e4 Mon Sep 17 00:00:00 2001 From: Tyler Skelton Date: Wed, 7 Oct 2026 22:45:34 -0700 Subject: [PATCH 2/2] Malformed-call stop says the model ended the turn; unknown tools count; marks live on the message Review of #47: endedBy gains "malformed" with its own receipt sentence instead of being filed as a provider failure; calls to a tool that does not exist count toward the streak; the firm warning reaches the model on the act path too; the refused marks are message metadata (survive a restored transcript); constants and the stop block sit under their own comments; tests for each. --- src/engine.ts | 51 +++++++++++++++++++-------------- src/receipts.ts | 8 ++++-- src/transcript.ts | 31 ++++++++++++-------- src/types.ts | 8 +++++- test/malformed-cap.test.ts | 58 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 119 insertions(+), 37 deletions(-) diff --git a/src/engine.ts b/src/engine.ts index ff2eb5b..1440a7d 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -286,6 +286,10 @@ export function systemPromptFor(cwd: string, extra?: string): string { /** Fractions of the ceiling at which molt says something, once each. */ const CEILING_WARNINGS = [0.5, 0.8]; +/** Malformed tool calls in a row before the model is told firmly, and before the turn ends. */ +export const MALFORMED_WARN = 3; +export const MALFORMED_STOP = 6; + /** * When working history gets compacted, unless told otherwise. * @@ -301,10 +305,6 @@ const CEILING_WARNINGS = [0.5, 0.8]; * original is preserved in `.maat/exuviae/`, `record-intact` fails if it is * not, and `/regrow` pulls it back by pattern. */ -/** Malformed tool calls in a row before the model is told firmly, and before the turn ends. */ -export const MALFORMED_WARN = 3; -export const MALFORMED_STOP = 6; - export const DEFAULT_AUTO_SHED_TOKENS = 60_000; /** @@ -2354,7 +2354,7 @@ export class Engine { * all because the clock or a failed request stopped the turn before any check * ran; two of them had passing work on disk. */ - private turnEndedBy: "deadline" | "provider" | undefined; + private turnEndedBy: "deadline" | "provider" | "malformed" | undefined; /** * Malformed tool calls in a row this turn. One is a slip; a run that kept * sending broken `act` calls made 17 in a row and paid for every one @@ -4077,7 +4077,7 @@ export class Engine { result: BarResult, attempts: number, log?: Journal, - endedBy?: "deadline" | "provider", + endedBy?: "deadline" | "provider" | "malformed", ): AsyncGenerator { const onlyWrites = failedOnlyWriteChecks(result); if (this.cfg.receipts) { @@ -4466,7 +4466,7 @@ export class Engine { outcome = "unverified"; yield { kind: "info", - text: `the checks passed on the work as it stood when the ${this.turnEndedBy === "deadline" ? "clock" : "provider"} stopped the model, but it never said it was done — unverified.`, + text: `the checks passed on the work as it stood when the ${this.turnEndedBy === "deadline" ? "clock" : this.turnEndedBy === "malformed" ? "malformed-call limit" : "provider"} stopped the model, but it never said it was done — unverified.`, }; } @@ -5069,7 +5069,8 @@ export class Engine { } // An act reaches a tool call as itself only when its actions could not be // read (expandAct), so it counts as malformed here too. - const refused = malformed || name === "act"; + const known = TOOLS.some((t) => t.function.name === name); + const refused = malformed || name === "act" || !known; this.malformedStreak = refused ? this.malformedStreak + 1 : 0; const detail = toolDetail(name, args); this.turnCalls.add(callId); @@ -5323,7 +5324,11 @@ export class Engine { preview: hide(capture(result)), auto: !decision.ask, }; - this.transcript.push({ role: "tool", tool_call_id: callId, content: result }); + const firmHere = + refused && this.malformedStreak >= MALFORMED_WARN + ? ` [molt: that is ${this.malformedStreak} malformed calls in a row. Send ONE call whose arguments are a single JSON object matching the tool's schema, or finish; after ${MALFORMED_STOP} the turn ends.]` + : ""; + this.transcript.push({ role: "tool", tool_call_id: callId, content: result + firmHere }); if (refused) this.transcript.markMalformedCall(callId); return { name, result, auto: !decision.ask, repeated: repeatedHere }; } @@ -5811,7 +5816,7 @@ export class Engine { * is what it says: verified only if it passed, not proven if it did not, * unverified when nothing was sealed. Returns whether a bar ran. */ - async function* judgeOnDisk(why: "deadline" | "provider", claim: string): AsyncGenerator { + async function* judgeOnDisk(why: "deadline" | "provider" | "malformed", claim: string): AsyncGenerator { self.turnEndedBy = why; if (opts.ask) return false; let barThis = barNow(); @@ -5956,17 +5961,6 @@ export class Engine { // whatever state the work is in when they are up is the state the judge // sees. Unlike a token ceiling it is not a proxy for anything — it is // the thing the person waiting actually spends. - if (this.malformedStreak >= MALFORMED_STOP) { - log?.append("note", { text: `ended: malformed tool calls (${this.malformedStreak} in a row)`, malformedStop: true }); - yield { - kind: "info", - text: `ended: ${this.malformedStreak} malformed tool calls in a row — no more tool calls this turn. The work on disk is judged as it stands.`, - }; - this.turnEndedBy = "provider"; - const judged = this.turnWrites.length > 0 ? yield* judgeOnDisk("provider", "") : false; - if (!judged) yield* this.salvage("You sent several malformed tool calls in a row, so this turn has ended.", fetchFn, log); - return; - } if (deadlineInterrupted || this.pastDeadline()) { const spentMs = Date.now() - this.turnStartedAt; log?.append("deadline", { limitMs: this.turnDeadlineMs, spentMs }); @@ -5985,6 +5979,21 @@ export class Engine { return; } + // A model that keeps sending malformed calls is stopped like a clock that + // ran out: no more tool calls, the work on disk is judged, and the + // receipt says the model, not the provider, ended the turn. + if (this.malformedStreak >= MALFORMED_STOP) { + log?.append("note", { text: `ended: malformed tool calls (${this.malformedStreak} in a row)`, malformedStop: true }); + yield { + kind: "info", + text: `ended: ${this.malformedStreak} malformed tool calls in a row — no more tool calls this turn. The work on disk is judged as it stands.`, + }; + this.turnEndedBy = "malformed"; + const judged = this.turnWrites.length > 0 ? yield* judgeOnDisk("malformed", "") : false; + if (!judged) yield* this.salvage("You sent several malformed tool calls in a row, so this turn has ended.", fetchFn, log); + return; + } + // An explicit budget speaks for itself, and speaks first: one knob // should not produce two different messages. if (this.overBudget()) { diff --git a/src/receipts.ts b/src/receipts.ts index 3a33953..af5c39b 100644 --- a/src/receipts.ts +++ b/src/receipts.ts @@ -60,7 +60,7 @@ export type ReceiptRecord = { */ ask?: boolean; /** Set when the model was stopped and the tree judged as it stood. */ - endedBy?: "deadline" | "provider"; + endedBy?: "deadline" | "provider" | "malformed"; /** True when the cost rests on molt's own token estimate anywhere in the session. */ costEstimated?: boolean; /** @@ -373,7 +373,7 @@ export class Receipts { /** True for a question: the bar ran advisory and could not refuse. */ ask?: boolean; /** The model was stopped (clock, provider) and the tree was judged as it stood. */ - endedBy?: "deadline" | "provider"; + endedBy?: "deadline" | "provider" | "malformed"; /** * Every file the turn changed, with the hashes that prove it — and which * lines it wrote, so the receipt can show the work rather than describe it. @@ -527,7 +527,9 @@ export class Receipts { ? [ args.endedBy === "deadline" ? "The turn's time budget ran out before the model said it was done; the sealed checks ran on the tree as it stood." - : "The provider failed after work had been done; the sealed checks ran on the tree as it stood.", + : args.endedBy === "malformed" + ? "The model sent malformed tool calls several times in a row, so Maat ended the turn; the sealed checks ran on the tree as it stood." + : "The provider failed after work had been done; the sealed checks ran on the tree as it stood.", "", ] : []), diff --git a/src/transcript.ts b/src/transcript.ts index 5d58cdc..9fef4a9 100644 --- a/src/transcript.ts +++ b/src/transcript.ts @@ -70,14 +70,6 @@ export class Transcript { private archived: Msg[][] = []; /** What this turn is for. Sent every request, shed never. */ private task: string | null = null; - /** - * Tool calls Maat refused as malformed. Their arguments go back to the - * provider as a short excerpt, not in full: a run that made 17 malformed - * `act` calls in a row resent every one of them on every step, prompts grew - * to 630k tokens, and one task cost $0.82 for a 181-byte file (2026-10-07). - * The record and captures keep the full text. - */ - private malformedCalls = new Set(); constructor(systemPrompt: string) { this.system = { role: "system", content: systemPrompt }; @@ -87,9 +79,23 @@ export class Transcript { this.working.push(msg); } - /** Send this call's arguments as an excerpt from now on (see `malformedCalls`). */ + /** + * Send this call's arguments as an excerpt from now on. A run that made 17 + * malformed `act` calls in a row resent every one of them on every step, + * prompts grew to 630k tokens, and one task cost $0.82 for a 181-byte file + * (2026-10-07). The mark lives on the message, so it survives a restore; the + * record and captures keep the full text. + */ markMalformedCall(id: string): void { - this.malformedCalls.add(id); + for (let i = this.working.length - 1; i >= 0; i--) { + const m = this.working[i]!; + if (m.role === "assistant" && m.tool_calls?.some((c) => c.id === id)) { + const refused = new Set(m.molt?.refusedCalls ?? []); + refused.add(id); + this.working[i] = { ...m, molt: { ...m.molt, refusedCalls: [...refused] } }; + return; + } + } } /** @@ -171,11 +177,12 @@ export class Transcript { // llama.cpp Qwen failed with a 500. Leading system messages are joined // in order; a system message anywhere later goes as a user message. const out: Omit[] = []; - for (const { molt: _molt, ...m } of this.all()) { + for (const { molt, ...m } of this.all()) { + const refused = new Set(molt?.refusedCalls ?? []); if (m.role !== "system") out.push( repair && Array.isArray(m.tool_calls) - ? { ...m, tool_calls: m.tool_calls.map((c) => (c.id && this.malformedCalls.has(c.id) ? excerptCall(c) : wireCall(c))) } + ? { ...m, tool_calls: m.tool_calls.map((c) => (c.id && refused.has(c.id) ? excerptCall(c) : wireCall(c))) } : m, ); else if (out.length === 0) out.push({ ...m }); diff --git a/src/types.ts b/src/types.ts index 5283af5..f969a9f 100644 --- a/src/types.ts +++ b/src/types.ts @@ -29,6 +29,12 @@ export type Msg = { }; export type MsgMeta = { + /** + * Ids of this assistant message's tool calls that Maat refused as malformed. + * Their arguments go back to the provider as an excerpt (transcript.ts). Kept + * on the message so a restored transcript keeps sending the excerpt. + */ + refusedCalls?: string[]; /** True for a mechanically-generated digest of shed context. */ digest?: true; /** True for context deliberately re-attached from the archive. */ @@ -642,7 +648,7 @@ export type EngineEvent = * happened. The outcome is what the sealed bar said of the tree as it * stood, never a default. */ - endedBy?: "deadline" | "provider"; + endedBy?: "deadline" | "provider" | "malformed"; /** The turn's wall-clock budget ended it (endedBy "deadline"). */ deadline?: boolean; /** diff --git a/test/malformed-cap.test.ts b/test/malformed-cap.test.ts index 2b8746f..33eef7d 100644 --- a/test/malformed-cap.test.ts +++ b/test/malformed-cap.test.ts @@ -2,6 +2,10 @@ import assert from "node:assert/strict"; import { describe, it } from "node:test"; import { Engine, MALFORMED_STOP } from "../src/engine.js"; import { Transcript, excerpt, MALFORMED_EXCERPT_CHARS, wireArgs } from "../src/transcript.js"; +import { readFileSync, readdirSync } from "node:fs"; +import { join } from "node:path"; +import { Receipts } from "../src/receipts.js"; +import { parseBar } from "../src/bar.js"; import { allowAll, drain, scriptedProvider, workspace } from "./helpers.js"; /** @@ -69,4 +73,58 @@ describe("malformed tool calls stay cheap", () => { ws.cleanup(); } }); + + it("tells the model firmly at three in a row", async () => { + const ws = workspace(); + try { + const bad = { calls: [{ name: "act", args: { actions: "not a list" } }] }; + const provider = scriptedProvider([bad, bad, bad, { text: "Done." }]); + const engine = new Engine({ baseUrl: "http://p.test/v1", model: "m", cwd: ws.dir, fetchFn: provider.fetchFn, bar: null, stream: false, autonomy: "high" }); + await drain(engine.run("go", allowAll)); + assert.ok(provider.bodies.some((b) => b.includes("3 malformed calls in a row"))); + } finally { + ws.cleanup(); + } + }); + + it("calls to a tool that does not exist count toward the streak", async () => { + const ws = workspace(); + try { + const ghost = { calls: [{ name: "no_such_tool", args: { payload: BIG } }] }; + const provider = scriptedProvider([...Array.from({ length: 8 }, () => ghost), { text: "Done." }]); + const engine = new Engine({ baseUrl: "http://p.test/v1", model: "m", cwd: ws.dir, fetchFn: provider.fetchFn, bar: null, stream: false, autonomy: "high" }); + const ev = await drain(engine.run("go", allowAll)); + assert.ok(ev.some((e) => e.kind === "info" && /malformed tool calls in a row/.test(e.text))); + assert.ok(Math.max(...provider.bodies.map((b) => b.length)) < 40_000); + } finally { + ws.cleanup(); + } + }); + + it("with work on disk, the turn ends as the model's doing and the receipt says so", async () => { + const ws = workspace(); + try { + const write = { calls: [{ name: "write_file", args: { path: "out.txt", content: "y" } }] }; + const bad = { calls: [{ name: "act", args: { actions: "not a list" } }] }; + const provider = scriptedProvider([write, bad, bad, bad, bad, bad, bad, bad, { text: "Done." }]); + const engine = new Engine({ baseUrl: "http://p.test/v1", model: "m", cwd: ws.dir, fetchFn: provider.fetchFn, bar: parseBar("version: 1\nchecks:\n - name: has-out\n run: test -f out.txt\n")!, receipts: new Receipts(ws.dir), stream: false, autonomy: "high" }); + const ev = await drain(engine.run("write out.txt", allowAll)); + const end = ev.find((e) => e.kind === "job_end") as { endedBy?: string } | undefined; + assert.equal(end?.endedBy, "malformed"); + const dir = join(ws.dir, ".maat", "receipts"); + const text = readdirSync(dir).filter((f) => f.endsWith(".md")).map((f) => readFileSync(join(dir, f), "utf8")).join("\n"); + assert.match(text, /malformed tool calls several times in a row/); + assert.doesNotMatch(text, /The provider failed/); + } finally { + ws.cleanup(); + } + }); + + it("the excerpt mark lives on the message", () => { + const t = new Transcript("sys"); + t.push({ role: "assistant", content: null, tool_calls: [{ id: "c9", type: "function", function: { name: "act", arguments: BIG } }] }); + t.markMalformedCall("c9"); + const rec = t.record().find((m) => m.role === "assistant")!; + assert.deepEqual(rec.molt?.refusedCalls, ["c9"]); + }); });