From a48570ddbd9b4c6764653f0aa0c6995c53c11dcf Mon Sep 17 00:00:00 2001 From: Tyler Skelton Date: Wed, 7 Oct 2026 14:29:09 -0700 Subject: [PATCH 1/2] Send an output cap, and shrink it when the completion reserve overflows the window GMI Cloud serves qwen/qwen3-235b-a22b-2507 with a 131072-token total window. The OpenAI-shaped request carried no max_tokens, so the router asked for the model's whole listed output (131072), and even a 2,454-token prompt was refused. contextOverflow then shed history, which cannot fix a completion overflow, and 11 of 34 Qwen runs ended "too large and nothing left to shed". - completionOverflow reads the window, input and completion counts from the vLLM/OpenAI wordings ("You requested a total of X tokens: A tokens from the input messages and B tokens for the completion", and "(A in the messages, B in the completion)"). When lowering the completion alone fits (window - input - margin >= 1024), max_tokens is lowered to that and the same request is sent again, with no shed; the cap is kept for the session. A prompt that is itself too big still sheds. - The OpenAI-shaped request now always sends max_tokens: the default is DEFAULT_MAX_TOKENS (32768), doubled up to 131072 only when a reply is cut off at it; --max-tokens still wins. - A provider that refuses the field itself (max_completion_tokens models) gets requests without it after one retry. The output-ceiling retry now runs on both request shapes and reads OpenAI's and DeepSeek's wordings too. - contextOverflow reads "maximum context length of W", and takes the input count, not the total, as what the server counted. --- src/anthropic.ts | 6 +- src/engine.ts | 129 +++++++++++++++++++++++- test/completion-reserve.test.ts | 172 ++++++++++++++++++++++++++++++++ 3 files changed, 301 insertions(+), 6 deletions(-) create mode 100644 test/completion-reserve.test.ts diff --git a/src/anthropic.ts b/src/anthropic.ts index e3b9303..7ccba0f 100644 --- a/src/anthropic.ts +++ b/src/anthropic.ts @@ -65,7 +65,11 @@ export function outputCeiling(body: string): number | null { const m = /max_tokens:\s*\d+\s*>\s*(\d+)/i.exec(body) ?? /maximum allowed number of output tokens[^\d]{0,40}(\d+)/i.exec(body) ?? - /max_tokens\D{0,40}(?:less than or equal to|at most|maximum of)\D{0,10}(\d+)/i.exec(body); + /max_tokens\D{0,40}(?:less than or equal to|at most|maximum of)\D{0,10}(\d+)/i.exec(body) ?? + // OpenAI: "max_tokens is too large: 32768. This model supports at most 16384 completion tokens". + /max_tokens is too large[\s\S]{0,80}?supports at most (\d+) completion tokens/i.exec(body) ?? + // DeepSeek: "the valid range of max_tokens is [1, 8192]". + /valid range of max_tokens is \[\d+,\s*(\d+)\]/i.exec(body); if (!m) return null; const n = Number(m[1]); return Number.isFinite(n) && n > 0 ? n : null; diff --git a/src/engine.ts b/src/engine.ts index 5b89a7d..aa34918 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -338,6 +338,57 @@ export type Overflow = { sent: number; }; +/** + * A refusal where the completion reserve, not the prompt, overflowed. + * + * vLLM-style servers count `max_tokens` against the window. GMI Cloud serves + * qwen3-235b with a 131,072-token TOTAL window, and the request went out with + * the model's full listed output (an OpenAI-shaped request with no max_tokens + * is given that by the router), so a 2,454-token prompt was refused: + * + * maximum context length of 131072 tokens. You requested a total of + * 133526 tokens: 2454 tokens from the input messages and 131072 tokens + * for the completion. + * + * Shedding history can never fix that. Returns the window, the input and the + * completion the server counted, or null when the body does not say all three. + */ +export function completionOverflow(body: string): { window: number; input: number; completion: number } | null { + const win = /maximum context length (?:is|of) (\d+)/i.exec(body); + const parts = + /(\d+) tokens? from the input messages and (\d+) tokens? for the completion/i.exec(body) ?? + /\((\d+) in the messages,\s*(\d+) in the completion\)/i.exec(body); + if (!win || !parts) return null; + const r = { window: Number(win[1]), input: Number(parts[1]), completion: Number(parts[2]) }; + return r.window > 0 && r.input >= 0 && r.completion > 0 ? r : null; +} + +/** The most an unasked output cap grows to after replies hit it. */ +export const OUTPUT_CAP_MAX = 131_072; + +/** + * A 400 that refuses the `max_tokens` field itself, as opposed to its value. + * Narrow on purpose: it must name the field and say it is unsupported. + */ +export function refusedMaxTokens(body: string): boolean { + return /max_tokens/i.test(body) && /unsupported|not supported|unrecognized|unknown (field|parameter)|extra inputs are not permitted|use 'max_completion_tokens'|max_completion_tokens/i.test(body); +} + +/** Room left between the prompt and the window, kept free of the completion reserve. */ +export const COMPLETION_MARGIN = 256; +/** A completion cap below this is no reply worth asking for; the prompt is what has to shrink. */ +export const MIN_COMPLETION = 1_024; + +/** + * The completion cap that fits, when lowering it alone fixes the overflow: + * the window less the input and a margin. Null when even the smallest useful + * reply does not fit (the history is the problem, and shedding is the fix). + */ +export function fittedCompletion(o: { window: number; input: number; completion: number }): number | null { + const fit = o.window - o.input - Math.max(COMPLETION_MARGIN, Math.round(o.window * 0.01)); + return fit >= MIN_COMPLETION && fit < o.completion ? fit : null; +} + export function contextOverflow(body: string): Overflow | null { // Overflow wording, not the bare word "context": a pinned OpenRouter provider // answered a rate-limit retry with a 400 that merely mentioned context, and a @@ -352,11 +403,15 @@ export function contextOverflow(body: string): Overflow | null { const win = /"n_ctx"\s*:\s*(\d+)/.exec(body) ?? /context size \((\d+)\s*tokens?\)/i.exec(body) ?? - /maximum context length is (\d+)/i.exec(body); + /maximum context length (?:is|of) (\d+)/i.exec(body); + // The prompt's own count where the server separates it from the completion: + // that is what molt's estimate is compared with. const sent = /"n_prompt_tokens"\s*:\s*(\d+)/.exec(body) ?? /request \((\d+)\s*tokens?\)/i.exec(body) ?? - /you requested (\d+)/i.exec(body); + /(\d+) tokens? from the input messages/i.exec(body) ?? + /\((\d+) in the messages,/i.exec(body) ?? + /you requested (?:a total of )?(\d+)/i.exec(body); return { window: win ? Number(win[1]) : 0, sent: sent ? Number(sent[1]) : 0 }; } @@ -1978,10 +2033,24 @@ export class Engine { */ private modelMaxTokens?: number; + /** + * The output cap this session settled on without being asked: the default + * (DEFAULT_MAX_TOKENS, not the model's whole listed output), doubled each + * time a reply is cut off at it, up to OUTPUT_CAP_MAX. + */ + private outputCap = DEFAULT_MAX_TOKENS; + /** + * The completion reserve a server said fits its window beside the prompt + * (completionOverflow). Sticky for the session: the prompt only grows. + */ + private fittedCompletion?: number; + /** Set once a provider refuses `max_tokens` outright; it is then not sent. */ + private maxTokensUnsupported = false; + /** What to send as `max_tokens`: what was asked for, bounded by what fits. */ private maxTokensFor(): number { - const asked = this.cfg.maxTokens ?? DEFAULT_MAX_TOKENS; - return this.modelMaxTokens ? Math.min(asked, this.modelMaxTokens) : asked; + const asked = this.cfg.maxTokens ?? this.outputCap; + return Math.min(asked, this.modelMaxTokens ?? Infinity, this.fittedCompletion ?? Infinity); } /** * These three are derived from the endpoint, and the endpoint moves. @@ -2291,6 +2360,8 @@ export class Engine { // Output maximums are per-model, and the last one's says nothing // about this one. Discovered again on the next refusal if it matters. this.modelMaxTokens = undefined; + this.fittedCompletion = undefined; + this.maxTokensUnsupported = false; } this.cfg.model = m; } @@ -6080,6 +6151,10 @@ export class Engine { // ignored (see selfHostedThinking). Nothing on a cloud endpoint. ...selfHostedThinking(this.cfg.baseUrl, this.effortNow), ...openRouterProvider(this.cfg.baseUrl, this.cfg.model), + // Always a cap. Without one a router fills in the model's whole + // listed output, and a server that counts it against the window + // refuses every request (completionOverflow). + ...(this.maxTokensUnsupported ? {} : { max_tokens: this.maxTokensFor() }), ...(stream ? { stream: true } : {}), ...(withUsage ? { stream_options: { include_usage: true } } : {}), }; @@ -6215,7 +6290,7 @@ export class Engine { // so, and says what the maximum is. Same shape as the two // fallbacks above: retry once, and only believe the ceiling was // the problem if that works. - if (!res.ok && res.status === 400 && this.native && this.modelMaxTokens === undefined) { + if (!res.ok && res.status === 400 && this.modelMaxTokens === undefined && !this.maxTokensUnsupported) { const cap = outputCeiling(body); if (cap && cap < this.maxTokensFor()) { const was = this.maxTokensFor(); @@ -6236,6 +6311,45 @@ export class Engine { } } + // The completion reserve overflowed, not the prompt: ask for a + // smaller reply and send the same history again. Shedding cannot + // fix this one, and it was ending runs "too large and nothing left + // to shed" on a 2,454-token prompt. + for (let i = 0; i < 2 && !res.ok && res.status === 400; i++) { + const co = completionOverflow(body); + const fit = co ? fittedCompletion(co) : null; + if (!co || fit === null || fit >= this.maxTokensFor()) break; + const was = this.maxTokensFor(); + this.fittedCompletion = fit; + log?.append("note", { + body: body.slice(0, 600), + text: + `the completion reserve overflowed the ${co.window}-token window (${co.input} input + ` + + `${co.completion} completion) — max_tokens ${was} → ${fit}, retried without shedding`, + }); + yield { + kind: "info", + text: `this endpoint counts the reply against its ${co.window}-token window — asking for at most ${fit} output tokens instead of ${was}.`, + }; + res = await send(askForUsage && !this.streamUsageUnsupported); + body = res.ok ? "" : (await res.text().catch(() => "")); + } + + // A provider that will not take `max_tokens` at all (OpenAI's + // reasoning models want max_completion_tokens). Try once without; + // only believe the field was the problem if that works. + if (!res.ok && res.status === 400 && !this.native && !this.maxTokensUnsupported && refusedMaxTokens(body)) { + this.maxTokensUnsupported = true; + const retry = await send(askForUsage && !this.streamUsageUnsupported); + if (retry.ok) { + log?.append("note", { text: "provider refused max_tokens — sending requests without an output cap" }); + } else { + this.maxTokensUnsupported = false; + } + res = retry; + body = res.ok ? "" : (await res.text().catch(() => "")); + } + if (!res.ok) { // "Too big" is not a verdict on the work, and it is the one // refusal that says how to fix itself. Shed and send less rather @@ -7077,6 +7191,11 @@ export class Engine { // sentence. if (finishReason === "length" && truncatedTurns < TRUNCATED_TURN_RETRIES) { truncatedTurns += 1; + // A reply that actually hit the default cap earns a larger one; a cap + // the person set, or one the server said is all that fits, stays. + if (this.cfg.maxTokens === undefined && this.outputCap < OUTPUT_CAP_MAX && this.maxTokensFor() === this.outputCap) { + this.outputCap = Math.min(OUTPUT_CAP_MAX, this.outputCap * 2); + } log?.append("note", { text: `reply cut off at the ${this.maxTokensFor()}-token output ceiling`, step, diff --git a/test/completion-reserve.test.ts b/test/completion-reserve.test.ts new file mode 100644 index 0000000..ebea43c --- /dev/null +++ b/test/completion-reserve.test.ts @@ -0,0 +1,172 @@ +/** + * A server that counts the completion reserve against its window. + * + * GMI Cloud serves qwen/qwen3-235b-a22b-2507 with a 131,072-token TOTAL + * window. The request carried no max_tokens, the router filled in the model's + * whole listed output, and even a 2,454-token prompt was refused; Maat then + * shed history, which cannot fix that, and 11 of 34 runs ended "too large and + * nothing left to shed". Mock providers only. + */ +import assert from "node:assert/strict"; +import { writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, it } from "node:test"; +import { completionOverflow, contextOverflow, Engine, fittedCompletion, refusedMaxTokens } from "../src/engine.js"; +import { outputCeiling } from "../src/anthropic.js"; +import { allowAll, drain, workspace } from "./helpers.js"; + +const GMI = + `{"object":"error","message":"Requested token count exceeds the model's maximum context length of 131072 tokens. ` + + `You requested a total of 133526 tokens: 2454 tokens from the input messages and 131072 tokens for the completion. ` + + `Please reduce the number of tokens in the input messages or the completion to fit within the limit.","type":"BadRequestError","param":null,"code":400}`; + +const gmiBody = (input: number, completion: number) => + GMI.replace("133526", String(input + completion)).replace("2454 tokens", `${input} tokens`).replace("131072 tokens for", `${completion} tokens for`); + +type Body = { max_tokens?: number; messages: unknown[] }; + +const okReply = (message: object, finish = "stop") => + ({ + ok: true, + status: 200, + json: async () => ({ choices: [{ message, finish_reason: finish }], usage: { prompt_tokens: 900, completion_tokens: 20 } }), + text: async () => "", + }) as unknown as Response; +const refused = (text: string) => ({ ok: false, status: 400, text: async () => text, json: async () => ({}) }) as unknown as Response; + +function engineWith(dir: string, fetchFn: typeof fetch, over: Record = {}): Engine { + return new Engine({ baseUrl: "http://provider.test/v1", model: "m", provider: "test", cwd: dir, bar: null, stream: false, fetchFn, retryBackoffMs: [5, 5, 5], ...over }); +} + +describe("reading a completion-reserve overflow", () => { + it("reads the window, the input and the completion out of the GMI body", () => { + assert.deepEqual(completionOverflow(GMI), { window: 131072, input: 2454, completion: 131072 }); + // The older OpenAI/vLLM wording too. + assert.deepEqual( + completionOverflow("This model's maximum context length is 32768 tokens. However, you requested 34000 tokens (1232 in the messages, 32768 in the completion)."), + { window: 32768, input: 1232, completion: 32768 }, + ); + assert.equal(completionOverflow('{"n_ctx":16384,"n_prompt_tokens":17222}'), null); + // contextOverflow's "sent" is the prompt's count, the number Maat's estimate is compared with. + assert.deepEqual(contextOverflow(GMI), { window: 131072, sent: 2454 }); + }); + + it("fits the completion when lowering it alone is enough, and says no when the prompt is the problem", () => { + const fit = fittedCompletion({ window: 131072, input: 2454, completion: 131072 }); + assert.ok(fit !== null && fit >= 1024 && fit <= 131072 - 2454 - 256, String(fit)); + assert.equal(fittedCompletion({ window: 131072, input: 130500, completion: 32768 }), null); + }); + + it("knows a refused max_tokens field and more output-ceiling wordings", () => { + assert.equal(refusedMaxTokens("Unsupported parameter: 'max_tokens' is not supported with this model. Use 'max_completion_tokens' instead."), true); + assert.equal(refusedMaxTokens(GMI), false); + assert.equal(outputCeiling("max_tokens is too large: 32768. This model supports at most 16384 completion tokens, whereas you provided 32768."), 16384); + assert.equal(outputCeiling("Invalid max_tokens value, the valid range of max_tokens is [1, 8192]"), 8192); + }); +}); + +describe("a request whose completion reserve overflows", () => { + it("is sent again with a smaller max_tokens, the same history and no shed, and the cap stays for the session", async () => { + const ws = workspace(); + try { + const bodies: Body[] = []; + let refusals = 0; + const fetchFn = (async (_u: string, init?: RequestInit) => { + const b = JSON.parse(String(init?.body ?? "{}")) as Body; + bodies.push(b); + if ((b.max_tokens ?? 131072) + 2454 > 131072) { + refusals++; + return refused(gmiBody(2454, b.max_tokens ?? 131072)); + } + return okReply({ role: "assistant", content: "answered." }); + }) as unknown as typeof fetch; + // As the router did: the model's full listed output. + const engine = engineWith(ws.dir, fetchFn, { maxTokens: 131072 }); + const events = await drain(engine.run("say hi", allowAll)); + assert.equal(refusals, 1); + assert.equal(bodies.length, 2); + assert.equal(bodies[0]!.max_tokens, 131072); + assert.ok(bodies[1]!.max_tokens! <= 131072 - 2454 - 256 && bodies[1]!.max_tokens! >= 1024, String(bodies[1]!.max_tokens)); + assert.deepEqual(bodies[1]!.messages, bodies[0]!.messages, "the same history went again"); + assert.equal(events.filter((e) => e.kind === "shed").length, 0, "nothing was shed"); + assert.ok(events.some((e) => e.kind === "assistant_text" && e.text.includes("answered."))); + // Remembered: the next turn does not pay the refusal again. + await drain(engine.run("again", allowAll)); + assert.equal(refusals, 1); + assert.equal(bodies[2]!.max_tokens, bodies[1]!.max_tokens); + } finally { + ws.cleanup(); + } + }); + + it("a real history overflow still sheds, and leaves max_tokens alone", async () => { + const ws = workspace(); + try { + writeFileSync(join(ws.dir, "big.txt"), "x".repeat(6000)); + const bodies: Body[] = []; + let refusals = 0; + const fetchFn = (async (_u: string, init?: RequestInit) => { + bodies.push(JSON.parse(String(init?.body ?? "{}")) as Body); + if (bodies.length <= 5) { + return okReply( + { role: "assistant", content: null, tool_calls: [{ id: `c${bodies.length}`, type: "function", function: { name: "read_file", arguments: '{"path":"big.txt"}' } }] }, + "tool_calls", + ); + } + if (refusals < 1) { + refusals++; + return refused(gmiBody(130500, 32768)); + } + return okReply({ role: "assistant", content: "fitted." }); + }) as unknown as typeof fetch; + const engine = engineWith(ws.dir, fetchFn); + const events = await drain(engine.run("read it a few times then answer", allowAll)); + assert.equal(refusals, 1); + assert.ok(events.some((e) => e.kind === "shed"), "the history was shed"); + assert.equal(bodies.at(-1)!.max_tokens, bodies[0]!.max_tokens, "max_tokens unchanged"); + assert.ok(events.some((e) => e.kind === "assistant_text" && e.text.includes("fitted."))); + } finally { + ws.cleanup(); + } + }); +}); + +describe("the default output cap", () => { + it("is not the model's whole listed output, and doubles only when a reply hits it", async () => { + const ws = workspace(); + try { + const bodies: Body[] = []; + const fetchFn = (async (_u: string, init?: RequestInit) => { + bodies.push(JSON.parse(String(init?.body ?? "{}")) as Body); + return bodies.length === 1 + ? okReply({ role: "assistant", content: "a long answer that stops mid" }, "length") + : okReply({ role: "assistant", content: "word. Done." }); + }) as unknown as typeof fetch; + const engine = engineWith(ws.dir, fetchFn); + await drain(engine.run("write", allowAll)); + assert.equal(bodies[0]!.max_tokens, 32768); + assert.equal(bodies[1]!.max_tokens, 65536); + } finally { + ws.cleanup(); + } + }); + + it("a provider that refuses the field gets requests without it", async () => { + const ws = workspace(); + try { + const bodies: Body[] = []; + const fetchFn = (async (_u: string, init?: RequestInit) => { + const b = JSON.parse(String(init?.body ?? "{}")) as Body; + bodies.push(b); + return "max_tokens" in b + ? refused(`{"error":{"message":"Unsupported parameter: 'max_tokens' is not supported with this model. Use 'max_completion_tokens' instead."}}`) + : okReply({ role: "assistant", content: "ok." }); + }) as unknown as typeof fetch; + const events = await drain(engineWith(ws.dir, fetchFn).run("hi", allowAll)); + assert.ok(events.some((e) => e.kind === "assistant_text" && e.text.includes("ok."))); + assert.equal("max_tokens" in bodies.at(-1)!, false); + } finally { + ws.cleanup(); + } + }); +}); From 34c91ca256513fc32f506ce3a9e2be0bd1b5f784 Mon Sep 17 00:00:00 2001 From: Tyler Skelton Date: Wed, 7 Oct 2026 17:23:54 -0700 Subject: [PATCH 2/2] Read every provider's reserve refusal, and try a smaller cap before shedding Sending max_tokens on every request broke endpoints whose refusal wording was not one of the two parsed ones: OpenRouter ("1000 of text input, 900 of tool input, 32768 in the output") and current vLLM ("you requested 32768 output tokens and your prompt contains 1000 input tokens") fell through to the shed path and ended turn 1 with nothing to shed. - completionOverflow reads OpenRouter, current vLLM, vLLM 0.8-0.16 ("'max_tokens' or 'max_completion_tokens' is too large: N ... your request has M input tokens"), OpenAI (including "N in the functions"), Anthropic ("input length and `max_tokens` exceed context limit: A + B > W") and TGI. llama.cpp never refuses for the reserve, so its refusal stays a prompt overflow. - contextOverflow takes the prompt count from those wordings and never reads an output count as what was sent (current vLLM scaled every estimate 8x); it also reads Anthropic's "prompt is too long: N tokens > W maximum". - A window refusal in wording none of these read, from a request with a cap above 8192, gets one request with a smaller cap (fitted from the window and the estimate when the server named a window) before shedding; kept only if it works, and dropped if a reply is later cut off at it. A prompt known to be too big goes straight to the shed path. - refusedMaxTokens no longer matches a body that merely names max_completion_tokens; it needs the field refused. - The cut-off note, the nudge and the info line report the cap the reply hit, not the doubled one; the note says what the next request asks for. - A model change resets the doubled output cap; a shed clears the fitted cap, since the smaller prompt leaves room for a larger reply. - Tests for each wording, end to end on turn 1, the llama.cpp shed path, the fallback and its revert, and the refused-field revert. --- src/engine.ts | 168 +++++++++++++++++++++---- test/completion-reserve.test.ts | 215 +++++++++++++++++++++++++++++++- 2 files changed, 359 insertions(+), 24 deletions(-) diff --git a/src/engine.ts b/src/engine.ts index aa34918..fa2f83e 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -350,17 +350,57 @@ export type Overflow = { * 133526 tokens: 2454 tokens from the input messages and 131072 tokens * for the completion. * - * Shedding history can never fix that. Returns the window, the input and the - * completion the server counted, or null when the body does not say all three. + * Shedding history can never fix that. Every server words it differently: + * vLLM (both its old and current wordings), OpenAI, OpenRouter, Anthropic and + * TGI are read below. llama.cpp never refuses for the reserve (it stops the + * reply when the window fills), so its refusal is always a prompt overflow. + * Returns the window, the input and the completion the server counted, or + * null when the body does not say all three. */ export function completionOverflow(body: string): { window: number; input: number; completion: number } | null { + const ok = (window: number, input: number, completion: number) => + window > 0 && input >= 0 && completion > 0 ? { window, input, completion } : null; + + // Anthropic: "input length and `max_tokens` exceed context limit: 188240 + 21333 > 200000". + const anth = /max_tokens`?\s+exceed context limit:\s*(\d+)\s*\+\s*(\d+)\s*>\s*(\d+)/i.exec(body); + if (anth) return ok(Number(anth[3]), Number(anth[1]), Number(anth[2])); + + // Hugging Face TGI: "`inputs` tokens + `max_new_tokens` must be <= 4096. Given: 1000 `inputs` tokens and 4000 `max_new_tokens`". + const tgi = /must be <= (\d+)\.\s*Given:\s*(\d+) `?inputs`? tokens and (\d+) `?max_new_tokens`?/i.exec(body); + if (tgi) return ok(Number(tgi[1]), Number(tgi[2]), Number(tgi[3])); + const win = /maximum context length (?:is|of) (\d+)/i.exec(body); - const parts = - /(\d+) tokens? from the input messages and (\d+) tokens? for the completion/i.exec(body) ?? - /\((\d+) in the messages,\s*(\d+) in the completion\)/i.exec(body); - if (!win || !parts) return null; - const r = { window: Number(win[1]), input: Number(parts[1]), completion: Number(parts[2]) }; - return r.window > 0 && r.input >= 0 && r.completion > 0 ? r : null; + if (!win) return null; + const window = Number(win[1]); + + // vLLM (since vllm#36197): "you requested 32768 output tokens and your prompt contains 1000 input tokens". + const v2 = /you requested (\d+) output tokens? and your prompt contains (\d+) input tokens?/i.exec(body); + if (v2) return ok(window, Number(v2[2]), Number(v2[1])); + + // vLLM 0.8-0.16: "'max_tokens' or 'max_completion_tokens' is too large: 32768. This model's maximum + // context length is 32768 tokens and your request has 1000 input tokens (32768 > 32768 - 1000)." + const v1 = /is too large:\s*(\d+)[\s\S]{0,200}?your request has (\d+) input tokens?/i.exec(body); + if (v1) return ok(window, Number(v1[2]), Number(v1[1])); + + // GMI / vLLM: "A tokens from the input messages and B tokens for the completion". + const gmi = /(\d+) tokens? from the input messages and (\d+) tokens? for the completion/i.exec(body); + if (gmi) return ok(window, Number(gmi[1]), Number(gmi[2])); + + // OpenRouter: "(1000 of text input, 900 of tool input, 32768 in the output)". Every "of … input" + // part counts against the prompt (text, tool, image). + const orOut = /(\d+) in the output/i.exec(body); + if (orOut) { + const ins = [...body.matchAll(/(\d+) of [a-z ]{0,20}?input/gi)].map((m) => Number(m[1])); + if (ins.length) return ok(window, ins.reduce((a, b) => a + b, 0), Number(orOut[1])); + } + + // OpenAI: "(1232 in the messages, 32768 in the completion)", sometimes with "N in the functions". + const oai = /\(([^)]*?(\d+) in the completion)\)/i.exec(body); + if (oai) { + const ins = [...oai[1]!.matchAll(/(\d+) in the (?!completion)\w+/gi)].map((m) => Number(m[1])); + if (ins.length) return ok(window, ins.reduce((a, b) => a + b, 0), Number(oai[2])); + } + return null; } /** The most an unasked output cap grows to after replies hit it. */ @@ -371,7 +411,7 @@ export const OUTPUT_CAP_MAX = 131_072; * Narrow on purpose: it must name the field and say it is unsupported. */ export function refusedMaxTokens(body: string): boolean { - return /max_tokens/i.test(body) && /unsupported|not supported|unrecognized|unknown (field|parameter)|extra inputs are not permitted|use 'max_completion_tokens'|max_completion_tokens/i.test(body); + return /max_tokens/i.test(body) && /unsupported|not supported|unrecognized|unknown (field|parameter)|extra inputs are not permitted|use 'max_completion_tokens'/i.test(body); } /** Room left between the prompt and the window, kept free of the completion reserve. */ @@ -389,6 +429,33 @@ export function fittedCompletion(o: { window: number; input: number; completion: return fit >= MIN_COMPLETION && fit < o.completion ? fit : null; } +/** The cap tried once when a window refusal names no counts molt can read. */ +export const FALLBACK_COMPLETION = 8_192; + +/** + * A smaller cap to try before shedding, for a window refusal whose wording + * completionOverflow does not know. + * + * Maat always sends max_tokens now, and a server that counts it against the + * window refuses in its own words. Shedding cannot fix a reserve overflow, so + * when the refusal is about the context or the length and the cap sent was + * large, one request with a smaller cap is cheaper than ending a turn "with + * nothing left to shed". `inputEst` is molt's scaled estimate of the prompt, + * used when the server named a window but no prompt count. Null when the + * prompt itself is known not to fit (shed instead), when the cap is already + * small, or when the body is not about the window at all. + */ +export function reserveFallback(body: string, cap: number, inputEst: number): number | null { + if (cap <= FALLBACK_COMPLETION || completionOverflow(body)) return null; + const over = contextOverflow(body); + if (!over && !(/context|length/i.test(body) && /tokens?/i.test(body))) return null; + if (!over || over.window <= 0) return FALLBACK_COMPLETION; + const input = over.sent > 0 ? over.sent : inputEst; + const fit = over.window - input - Math.max(COMPLETION_MARGIN, Math.round(over.window * 0.01)); + if (fit < MIN_COMPLETION) return null; + return Math.min(fit, Math.floor(cap / 2)); +} + export function contextOverflow(body: string): Overflow | null { // Overflow wording, not the bare word "context": a pinned OpenRouter provider // answered a rate-limit retry with a 400 that merely mentioned context, and a @@ -399,19 +466,25 @@ export function contextOverflow(body: string): Overflow | null { ) ) return null; + // Where the server split the prompt from the completion, the prompt's count + // is what molt's estimate is compared with. Never the output count: current + // vLLM puts that first ("you requested 32768 output tokens and your prompt + // contains 1000"), and reading it as the prompt scaled every estimate 8x. + const co = completionOverflow(body); + if (co) return { window: co.window, sent: co.input }; // Every field the common servers use, most specific first. const win = /"n_ctx"\s*:\s*(\d+)/.exec(body) ?? /context size \((\d+)\s*tokens?\)/i.exec(body) ?? - /maximum context length (?:is|of) (\d+)/i.exec(body); - // The prompt's own count where the server separates it from the completion: - // that is what molt's estimate is compared with. + /maximum context length (?:is|of) (\d+)/i.exec(body) ?? + // Anthropic: "prompt is too long: 210000 tokens > 200000 maximum". + /tokens? > (\d+) maximum/i.exec(body); const sent = /"n_prompt_tokens"\s*:\s*(\d+)/.exec(body) ?? /request \((\d+)\s*tokens?\)/i.exec(body) ?? - /(\d+) tokens? from the input messages/i.exec(body) ?? - /\((\d+) in the messages,/i.exec(body) ?? - /you requested (?:a total of )?(\d+)/i.exec(body); + /prompt is too long:\s*(\d+)/i.exec(body) ?? + /your (?:prompt contains|request has|messages resulted in) (\d+)/i.exec(body) ?? + /you requested (?:a total of |about )?(\d+)(?!\d)(?! output)/i.exec(body); return { window: win ? Number(win[1]) : 0, sent: sent ? Number(sent[1]) : 0 }; } @@ -2041,9 +2114,15 @@ export class Engine { private outputCap = DEFAULT_MAX_TOKENS; /** * The completion reserve a server said fits its window beside the prompt - * (completionOverflow). Sticky for the session: the prompt only grows. + * (completionOverflow). Kept until the history is shed: the prompt grows + * between sheds, and after one there is room for a larger reply again. */ private fittedCompletion?: number; + /** + * The fitted cap above was a guess (reserveFallback), not read from the + * server's own counts. A reply that is cut off at a guess drops it. + */ + private fittedGuessed = false; /** Set once a provider refuses `max_tokens` outright; it is then not sent. */ private maxTokensUnsupported = false; @@ -2361,7 +2440,10 @@ export class Engine { // about this one. Discovered again on the next refusal if it matters. this.modelMaxTokens = undefined; this.fittedCompletion = undefined; + this.fittedGuessed = false; this.maxTokensUnsupported = false; + // A cap doubled on the last model says nothing about this one either. + this.outputCap = DEFAULT_MAX_TOKENS; } this.cfg.model = m; } @@ -3130,6 +3212,9 @@ export class Engine { } this.transcript.commitShed(plan); + // A cap fitted beside the old prompt is too small for the new one. + this.fittedCompletion = undefined; + this.fittedGuessed = false; this.ledger = staying; // Journalled here, on every path. It was journalled by the two auto-shed @@ -6321,6 +6406,7 @@ export class Engine { if (!co || fit === null || fit >= this.maxTokensFor()) break; const was = this.maxTokensFor(); this.fittedCompletion = fit; + this.fittedGuessed = false; log?.append("note", { body: body.slice(0, 600), text: @@ -6335,6 +6421,37 @@ export class Engine { body = res.ok ? "" : (await res.text().catch(() => "")); } + // A window refusal in wording none of the above reads, from a + // request that carried a large cap. Try one smaller cap before + // shedding: shedding cannot fix a reserve overflow, and before + // max_tokens was always sent these servers fitted the reply + // themselves. Kept only if it works. + if (!res.ok && res.status === 400 && !this.maxTokensUnsupported) { + const was = this.maxTokensFor(); + const guess = reserveFallback(body, was, Math.round(this.bom().requestTotalEst * this.tokenScale)); + if (guess !== null && guess < was) { + const before = { fitted: this.fittedCompletion, guessed: this.fittedGuessed }; + this.fittedCompletion = guess; + this.fittedGuessed = true; + const retry = await send(askForUsage && !this.streamUsageUnsupported); + if (retry.ok) { + log?.append("note", { + body: body.slice(0, 600), + text: `window refusal with max_tokens ${was} — ${guess} was accepted, kept for the session`, + }); + yield { + kind: "info", + text: `this endpoint refused a ${was}-token reply reserve — asking for at most ${guess} output tokens instead.`, + }; + } else { + this.fittedCompletion = before.fitted; + this.fittedGuessed = before.guessed; + } + res = retry; + body = res.ok ? "" : (await res.text().catch(() => "")); + } + } + // A provider that will not take `max_tokens` at all (OpenAI's // reasoning models want max_completion_tokens). Try once without; // only believe the field was the problem if that works. @@ -7191,20 +7308,29 @@ export class Engine { // sentence. if (finishReason === "length" && truncatedTurns < TRUNCATED_TURN_RETRIES) { truncatedTurns += 1; + // The cap this reply hit, before any change below: the note, the + // nudge and the info line report what happened, not what comes next. + const hit = this.maxTokensFor(); // A reply that actually hit the default cap earns a larger one; a cap - // the person set, or one the server said is all that fits, stays. - if (this.cfg.maxTokens === undefined && this.outputCap < OUTPUT_CAP_MAX && this.maxTokensFor() === this.outputCap) { + // the person set, or one the server said is all that fits, stays. A + // fitted cap that was only a guess is dropped; if the full one is + // refused again, the refusal fits it again. + if (this.cfg.maxTokens === undefined && this.outputCap < OUTPUT_CAP_MAX && hit === this.outputCap) { this.outputCap = Math.min(OUTPUT_CAP_MAX, this.outputCap * 2); + } else if (this.fittedGuessed && hit === this.fittedCompletion) { + this.fittedCompletion = undefined; + this.fittedGuessed = false; } + const next = this.maxTokensFor(); log?.append("note", { - text: `reply cut off at the ${this.maxTokensFor()}-token output ceiling`, + text: `reply cut off at the ${hit}-token output ceiling` + (next !== hit ? `; the next request asks for ${next}` : ""), step, attempt: truncatedTurns, }); this.transcript.push({ role: "user", content: - `[molt: that reply hit the output ceiling of ${this.maxTokensFor()} tokens and stopped ` + + `[molt: that reply hit the output ceiling of ${hit} tokens and stopped ` + `part-way through, so it is not being read as a finished answer. Continue from where ` + `it stops, and keep what remains short.]`, molt: { nudge: true }, @@ -7213,7 +7339,7 @@ export class Engine { yield { kind: "info", text: - `the reply was cut off at the ${this.maxTokensFor()}-token output ceiling — asking it ` + + `the reply was cut off at the ${hit}-token output ceiling — asking it ` + `to continue rather than running the bar on half a sentence` + (this.cfg.maxTokens === undefined ? " (raise it with --max-tokens)" : ""), }; diff --git a/test/completion-reserve.test.ts b/test/completion-reserve.test.ts index ebea43c..98e0750 100644 --- a/test/completion-reserve.test.ts +++ b/test/completion-reserve.test.ts @@ -11,7 +11,7 @@ import assert from "node:assert/strict"; import { writeFileSync } from "node:fs"; import { join } from "node:path"; import { describe, it } from "node:test"; -import { completionOverflow, contextOverflow, Engine, fittedCompletion, refusedMaxTokens } from "../src/engine.js"; +import { completionOverflow, contextOverflow, Engine, FALLBACK_COMPLETION, fittedCompletion, refusedMaxTokens, reserveFallback } from "../src/engine.js"; import { outputCeiling } from "../src/anthropic.js"; import { allowAll, drain, workspace } from "./helpers.js"; @@ -65,6 +65,182 @@ describe("reading a completion-reserve overflow", () => { }); }); +/** Each provider's own refusal for a 1000-token prompt and a 32768 cap on a 32768 window. */ +const OPENROUTER = (input: number, completion: number) => + `{"error":{"message":"This endpoint's maximum context length is 32768 tokens. However, you requested about ${input + completion} tokens ` + + `(${input - 100} of text input, 100 of tool input, ${completion} in the output). Please reduce the length of either one, ` + + `or use the \"middle-out\" transform to compress your prompt automatically.","code":400}}`; +const VLLM_NEW = (input: number, completion: number) => + `{"object":"error","message":"This model's maximum context length is 32768 tokens. However, you requested ${completion} output tokens ` + + `and your prompt contains ${input} input tokens, for a total of ${input + completion} tokens. Please reduce the length of the input ` + + `prompt or the number of requested output tokens.","type":"BadRequestError","param":null,"code":400}`; +const VLLM_OLD = (input: number, completion: number) => + `{"object":"error","message":"'max_tokens' or 'max_completion_tokens' is too large: ${completion}. This model's maximum context ` + + `length is 32768 tokens and your request has ${input} input tokens (${completion} > 32768 - ${input}).","type":"BadRequestError","param":null,"code":400}`; +const OPENAI = (input: number, completion: number) => + `{"error":{"message":"This model's maximum context length is 32768 tokens. However, you requested ${input + completion} tokens ` + + `(${input - 200} in the messages, 200 in the functions, ${completion} in the completion). Please reduce the length of the messages, functions, or completion.",` + + `"type":"invalid_request_error","param":"messages","code":"context_length_exceeded"}}`; +const ANTHROPIC = (input: number, completion: number) => + `{"type":"error","error":{"type":"invalid_request_error","message":"input length and \`max_tokens\` exceed context limit: ${input} + ${completion} > 32768, ` + + `decrease input length or \`max_tokens\` and try again"}}`; +const LLAMACPP = (input: number) => + `{"error":{"code":400,"message":"the request exceeds the available context size, try increasing it","type":"exceed_context_size_error",` + + `"n_prompt_tokens":${input},"n_ctx":32768}}`; + +describe("each provider's refusal wording", () => { + for (const [name, make] of [ + ["OpenRouter", OPENROUTER], + ["vLLM (current)", VLLM_NEW], + ["vLLM 0.8-0.16", VLLM_OLD], + ["OpenAI", OPENAI], + ["Anthropic", ANTHROPIC], + ] as const) { + it(`${name}: reads the window, the prompt and the completion, and the prompt is what contextOverflow reports`, () => { + const body = make(1000, 32768); + assert.deepEqual(completionOverflow(body), { window: 32768, input: 1000, completion: 32768 }); + assert.deepEqual(contextOverflow(body), { window: 32768, sent: 1000 }, "never the output count as sent"); + assert.equal(refusedMaxTokens(body), false, "the field itself was not refused"); + const fit = fittedCompletion(completionOverflow(body)!); + assert.ok(fit !== null && fit <= 32768 - 1000 - 256 && fit >= 1024, String(fit)); + }); + } + + it("TGI: reads `inputs` + `max_new_tokens` must be <= N", () => { + const body = "Input validation error: `inputs` tokens + `max_new_tokens` must be <= 4096. Given: 1000 `inputs` tokens and 32768 `max_new_tokens`"; + assert.deepEqual(completionOverflow(body), { window: 4096, input: 1000, completion: 32768 }); + }); + + it("llama.cpp: a prompt overflow, not a reserve one — shed, no smaller cap", () => { + const body = LLAMACPP(40000); + assert.equal(completionOverflow(body), null); + assert.deepEqual(contextOverflow(body), { window: 32768, sent: 40000 }); + assert.equal(reserveFallback(body, 32768, 100), null, "the prompt alone does not fit"); + }); + + it("Anthropic: a prompt that is too long by itself is read as a prompt overflow", () => { + const body = `{"type":"error","error":{"type":"invalid_request_error","message":"prompt is too long: 210000 tokens > 200000 maximum"}}`; + assert.equal(completionOverflow(body), null); + assert.deepEqual(contextOverflow(body), { window: 200000, sent: 210000 }); + assert.equal(reserveFallback(body, 32768, 100), null); + }); + + it("an unknown window wording with a large cap gets one smaller cap to try; a small cap or an unrelated 400 does not", () => { + assert.equal(reserveFallback("Error: prompt + max_tokens exceeds the context length of this model", 32768, 1000), FALLBACK_COMPLETION); + // A window it named, with a prompt count it did not: fitted from molt's own estimate. + const g = reserveFallback("maximum context length is 32768 tokens; reduce your request", 32768, 1000); + assert.ok(g !== null && g <= 16384 && g >= 1024, String(g)); + assert.equal(reserveFallback("Error: prompt + max_tokens exceeds the context length of this model", 4096, 1000), null); + assert.equal(reserveFallback('{"error":{"message":"invalid tool_choice"}}', 32768, 1000), null); + }); + + it("refusedMaxTokens is as narrow as its comment: naming both fields is not a refusal of the field", () => { + assert.equal(refusedMaxTokens("max_tokens / max_completion_tokens must be <= 4096"), false); + assert.equal(refusedMaxTokens(VLLM_OLD(1000, 32768)), false); + assert.equal(refusedMaxTokens(`{"error":{"message":"Unrecognized request argument supplied: max_tokens"}}`), true); + }); +}); + +/** A server with a 32768 window that counts the reserve against it, refusing in `wording`. */ +function reserveServer(wording: (input: number, completion: number) => string, input = 1000) { + const bodies: Body[] = []; + let refusals = 0; + const fetchFn = (async (_u: string, init?: RequestInit) => { + const b = JSON.parse(String(init?.body ?? "{}")) as Body; + bodies.push(b); + const asked = b.max_tokens ?? 32768 - input; + if (asked + input > 32768) { + refusals++; + return refused(wording(input, asked)); + } + return okReply({ role: "assistant", content: "answered." }); + }) as unknown as typeof fetch; + return { fetchFn, bodies, refusals: () => refusals }; +} + +describe("turn 1 against each provider's wording", () => { + for (const [name, make] of [ + ["OpenRouter", OPENROUTER], + ["vLLM (current)", VLLM_NEW], + ["vLLM 0.8-0.16", VLLM_OLD], + ["OpenAI", OPENAI], + ["Anthropic wording", ANTHROPIC], + ["an unknown wording", (i: number, c: number) => `{"error":{"message":"prompt (${i}) + max_tokens (${c}) exceeds the context length of this model"}}`], + ] as const) { + it(`${name}: answers on turn 1 with a smaller cap, no shed, max_tokens still sent`, async () => { + const ws = workspace(); + try { + const srv = reserveServer(make); + const engine = engineWith(ws.dir, srv.fetchFn); + const events = await drain(engine.run("say hi", allowAll)); + assert.ok(events.some((e) => e.kind === "assistant_text" && e.text.includes("answered.")), "the turn answered"); + assert.equal(events.filter((e) => e.kind === "shed").length, 0, "nothing was shed"); + assert.equal(srv.refusals(), 1); + assert.equal(srv.bodies[0]!.max_tokens, 32768); + const second = srv.bodies[1]!.max_tokens; + assert.ok(second !== undefined && second < 32768 - 1000 && second >= 1024, String(second)); + assert.deepEqual(srv.bodies[1]!.messages, srv.bodies[0]!.messages, "the same history went again"); + // The next turn does not pay the refusal again. + await drain(engine.run("again", allowAll)); + assert.equal(srv.refusals(), 1); + } finally { + ws.cleanup(); + } + }); + } + + it("llama.cpp: a prompt overflow sheds, and is not answered with a smaller cap", async () => { + const ws = workspace(); + try { + writeFileSync(join(ws.dir, "big.txt"), "x".repeat(6000)); + const bodies: Body[] = []; + let refusals = 0; + const fetchFn = (async (_u: string, init?: RequestInit) => { + bodies.push(JSON.parse(String(init?.body ?? "{}")) as Body); + if (bodies.length <= 5) { + return okReply( + { role: "assistant", content: null, tool_calls: [{ id: `c${bodies.length}`, type: "function", function: { name: "read_file", arguments: '{"path":"big.txt"}' } }] }, + "tool_calls", + ); + } + if (refusals < 1) { + refusals++; + return refused(LLAMACPP(40000)); + } + return okReply({ role: "assistant", content: "fitted." }); + }) as unknown as typeof fetch; + const events = await drain(engineWith(ws.dir, fetchFn).run("read it a few times then answer", allowAll)); + assert.equal(refusals, 1); + assert.ok(events.some((e) => e.kind === "shed"), "the history was shed"); + assert.equal(bodies.at(-1)!.max_tokens, 32768, "max_tokens unchanged"); + assert.ok(events.some((e) => e.kind === "assistant_text" && e.text.includes("fitted."))); + } finally { + ws.cleanup(); + } + }); + + it("a smaller cap that is refused too is forgotten, and the refusal goes on to the shed path", async () => { + const ws = workspace(); + try { + const bodies: Body[] = []; + const fetchFn = (async (_u: string, init?: RequestInit) => { + bodies.push(JSON.parse(String(init?.body ?? "{}")) as Body); + if (bodies.length <= 2) return refused(`{"error":{"message":"prompt + max_tokens exceeds the context length of this model"}}`); + return okReply({ role: "assistant", content: "ok." }); + }) as unknown as typeof fetch; + const engine = engineWith(ws.dir, fetchFn); + await drain(engine.run("hi", allowAll)); + assert.equal(bodies[1]!.max_tokens, FALLBACK_COMPLETION, "one smaller cap was tried"); + assert.equal(bodies.length, 2, "then the shed path had nothing to shed on turn 1"); + const events = await drain(engine.run("again", allowAll)); + assert.equal(bodies[2]!.max_tokens, 32768, "the guess was not kept"); + assert.ok(events.some((e) => e.kind === "assistant_text" && e.text.includes("ok."))); + } finally { + ws.cleanup(); + } + }); +}); + describe("a request whose completion reserve overflows", () => { it("is sent again with a smaller max_tokens, the same history and no shed, and the cap stays for the session", async () => { const ws = workspace(); @@ -143,9 +319,17 @@ describe("the default output cap", () => { : okReply({ role: "assistant", content: "word. Done." }); }) as unknown as typeof fetch; const engine = engineWith(ws.dir, fetchFn); - await drain(engine.run("write", allowAll)); + const events = await drain(engine.run("write", allowAll)); assert.equal(bodies[0]!.max_tokens, 32768); assert.equal(bodies[1]!.max_tokens, 65536); + // The cap the reply hit, not the doubled one, in the nudge the model reads and the info line. + assert.ok(JSON.stringify(bodies[1]!.messages).includes("output ceiling of 32768 tokens")); + assert.ok(!JSON.stringify(bodies[1]!.messages).includes("output ceiling of 65536")); + assert.ok(events.some((e) => e.kind === "info" && e.text.includes("cut off at the 32768-token output ceiling"))); + // A new model starts from the default again. + engine.setModel("other"); + await drain(engine.run("again", allowAll)); + assert.equal(bodies.at(-1)!.max_tokens, 32768); } finally { ws.cleanup(); } @@ -162,9 +346,34 @@ describe("the default output cap", () => { ? refused(`{"error":{"message":"Unsupported parameter: 'max_tokens' is not supported with this model. Use 'max_completion_tokens' instead."}}`) : okReply({ role: "assistant", content: "ok." }); }) as unknown as typeof fetch; - const events = await drain(engineWith(ws.dir, fetchFn).run("hi", allowAll)); + const engine = engineWith(ws.dir, fetchFn); + const events = await drain(engine.run("hi", allowAll)); assert.ok(events.some((e) => e.kind === "assistant_text" && e.text.includes("ok."))); assert.equal("max_tokens" in bodies.at(-1)!, false); + // The next turn does not pay the refusal again. + const before = bodies.length; + await drain(engine.run("again", allowAll)); + assert.equal(bodies.length, before + 1); + assert.equal("max_tokens" in bodies.at(-1)!, false); + } finally { + ws.cleanup(); + } + }); + + it("a retry without the field that also fails puts max_tokens back", async () => { + const ws = workspace(); + try { + const bodies: Body[] = []; + const fetchFn = (async (_u: string, init?: RequestInit) => { + bodies.push(JSON.parse(String(init?.body ?? "{}")) as Body); + if (bodies.length <= 2) return refused(`{"error":{"message":"Unrecognized request argument supplied: max_tokens"}}`); + return okReply({ role: "assistant", content: "ok." }); + }) as unknown as typeof fetch; + const engine = engineWith(ws.dir, fetchFn); + await drain(engine.run("hi", allowAll)); + assert.equal("max_tokens" in bodies[1]!, false, "tried once without"); + await drain(engine.run("again", allowAll)); + assert.equal(bodies.at(-1)!.max_tokens, 32768, "and put it back when that failed too"); } finally { ws.cleanup(); }