From 2a720b558189907a99e96d573df4d06f1b94a450 Mon Sep 17 00:00:00 2001 From: "Mars.P" Date: Wed, 23 Sep 2026 20:23:19 +0800 Subject: [PATCH 01/16] Interpolate an object into text without HTML-escaping it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A PR-review template binds {{nodes.fetch_diff.outputs.files}} into prose. VariableResolver wrote an object or array into surrounding text with the default HTML-safe encoder, so every CJK character and every + < > & ' of the diff reached the model as a six-character \uXXXX: "修复" arrived as 修复 and every added line began +, at up to twice the bytes. The model was reviewing escape codes. The string branch two lines above already inserted all of those characters raw, so the escaping never protected anything; it only made objects and strings disagree. WorkflowJson.InterpolatedText leaves characters as themselves and still escapes what JSON requires — the quote, the backslash, control characters — so a value embedded in a JSON body stays parseable. MapResultsPrompt, whose under-budget text must stay byte-identical to the resolver's, uses the same options; its identity pin gains a CJK branch, since plain ASCII could not tell two encoders apart. Characters outside the Basic Multilingual Plane are still escaped: no built-in encoder lifts that. --- .../Workflows/Runtime/MapResultsPrompt.cs | 6 ++-- .../Workflows/Runtime/VariableResolver.cs | 2 +- .../Services/Workflows/WorkflowJson.cs | 13 ++++++++ .../Workflows/MapResultsPromptTests.cs | 14 ++++++--- .../Workflows/VariableResolverTests.cs | 31 +++++++++++++++++++ 5 files changed, 57 insertions(+), 9 deletions(-) diff --git a/backend/src/CodeSpace.Core/Services/Workflows/Runtime/MapResultsPrompt.cs b/backend/src/CodeSpace.Core/Services/Workflows/Runtime/MapResultsPrompt.cs index 48ab1f68a..d35c3e9bf 100644 --- a/backend/src/CodeSpace.Core/Services/Workflows/Runtime/MapResultsPrompt.cs +++ b/backend/src/CodeSpace.Core/Services/Workflows/Runtime/MapResultsPrompt.cs @@ -33,7 +33,7 @@ namespace CodeSpace.Core.Services.Workflows.Runtime; /// that want more. One pathological branch therefore cannot consume the budget and starve its siblings. /// /// Under budget it is byte-identical to the unbounded binding. The within-budget case returns exactly -/// JsonSerializer.Serialize(resultsArray) as its text — the same call VariableResolver's array arm +/// JsonSerializer.Serialize(resultsArray, WorkflowJson.InterpolatedText) as its text — the same call VariableResolver's array arm /// makes on the same element — so the ordinary small fan-out reaches the model unchanged, character for character, /// and its coverage reads complete. /// @@ -53,13 +53,13 @@ public static class MapResultsPrompt /// public static MapResultsProjection Project(JsonElement resultsArray, int budgetChars) { - var whole = JsonSerializer.Serialize(resultsArray); + var whole = JsonSerializer.Serialize(resultsArray, WorkflowJson.InterpolatedText); var total = resultsArray.ValueKind == JsonValueKind.Array ? resultsArray.GetArrayLength() : 0; if (budgetChars <= 0 || whole.Length <= budgetChars) return Whole(whole, total); if (resultsArray.ValueKind != JsonValueKind.Array) return NothingIncluded(Cut(whole, budgetChars), total); - var branches = resultsArray.EnumerateArray().Select(element => JsonSerializer.Serialize(element)).ToList(); + var branches = resultsArray.EnumerateArray().Select(element => JsonSerializer.Serialize(element, WorkflowJson.InterpolatedText)).ToList(); if (branches.Count == 0) return NothingIncluded(Cut(whole, budgetChars), total); diff --git a/backend/src/CodeSpace.Core/Services/Workflows/Runtime/VariableResolver.cs b/backend/src/CodeSpace.Core/Services/Workflows/Runtime/VariableResolver.cs index b8eebba51..20906345e 100644 --- a/backend/src/CodeSpace.Core/Services/Workflows/Runtime/VariableResolver.cs +++ b/backend/src/CodeSpace.Core/Services/Workflows/Runtime/VariableResolver.cs @@ -134,7 +134,7 @@ public static IReadOnlyDictionary ResolveBag(JsonElement so JsonValueKind.String => value.GetString() ?? "", JsonValueKind.Number or JsonValueKind.True or JsonValueKind.False => value.ToString(), JsonValueKind.Null or JsonValueKind.Undefined => "", - _ => JsonSerializer.Serialize(value) + _ => JsonSerializer.Serialize(value, WorkflowJson.InterpolatedText) }; }); } diff --git a/backend/src/CodeSpace.Core/Services/Workflows/WorkflowJson.cs b/backend/src/CodeSpace.Core/Services/Workflows/WorkflowJson.cs index 33a40b18e..0731121ed 100644 --- a/backend/src/CodeSpace.Core/Services/Workflows/WorkflowJson.cs +++ b/backend/src/CodeSpace.Core/Services/Workflows/WorkflowJson.cs @@ -1,3 +1,4 @@ +using System.Text.Encodings.Web; using System.Text.Json; using System.Text.Json.Serialization; @@ -14,6 +15,18 @@ public static class WorkflowJson { public static JsonSerializerOptions Options { get; } = BuildOptions(); + /// + /// How an object or array is written into surrounding TEXT — a prompt, a message body — by a {{...}} + /// template, and by any projection that must stay byte-identical to one (MapResultsPrompt). Characters are + /// left as themselves: the HTML-safe default turned every CJK character and every + < > & ' into a + /// six-character \uXXXX, so a pull-request diff bound into a review prompt reached the model as escape + /// codes at up to twice its size — while the string branch beside it already inserted all of those raw, so the + /// escaping protected nothing. What JSON itself requires is still escaped (the quote, the backslash, control + /// characters), so a value embedded in a JSON body stays parseable. One exception no built-in encoder lifts: a + /// character outside the Basic Multilingual Plane (an emoji) is still written as its surrogate-pair escape. + /// + public static JsonSerializerOptions InterpolatedText { get; } = new() { Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping }; + private static JsonSerializerOptions BuildOptions() { var options = new JsonSerializerOptions(JsonSerializerDefaults.Web); diff --git a/backend/tests/CodeSpace.UnitTests/Workflows/MapResultsPromptTests.cs b/backend/tests/CodeSpace.UnitTests/Workflows/MapResultsPromptTests.cs index 6c0204cbb..45e99c898 100644 --- a/backend/tests/CodeSpace.UnitTests/Workflows/MapResultsPromptTests.cs +++ b/backend/tests/CodeSpace.UnitTests/Workflows/MapResultsPromptTests.cs @@ -2,6 +2,7 @@ using System.Text.RegularExpressions; using CodeSpace.Core.Services.Workflows.Engine; using CodeSpace.Core.Services.Workflows.Runtime; +using WorkflowJson = CodeSpace.Core.Services.Workflows.WorkflowJson; using CodeSpace.Messages.Constants; using CodeSpace.Messages.Dtos.Workflows; using Shouldly; @@ -42,7 +43,7 @@ public void Within_budget_the_projection_is_the_unbounded_serialization_characte // The ordinary fan-out: the model must see EXACTLY what the raw-array binding produced — the same call // VariableResolver's array arm makes on the same element. Not "equivalent JSON": the same characters. - projected.ShouldBe(JsonSerializer.Serialize(results), + projected.ShouldBe(JsonSerializer.Serialize(results, WorkflowJson.InterpolatedText), customMessage: "a fan-out inside the budget must not be reshaped at all — the bound may only bind when it binds"); } @@ -51,7 +52,7 @@ public void A_zero_budget_disables_the_bound_entirely() { var results = Results(new string('x', 50_000)); - MapResultsPrompt.Project(results, 0).Text.ShouldBe(JsonSerializer.Serialize(results), + MapResultsPrompt.Project(results, 0).Text.ShouldBe(JsonSerializer.Serialize(results, WorkflowJson.InterpolatedText), customMessage: "budget <= 0 means no bound — every map that declares none keeps its pre-existing output"); } @@ -107,7 +108,7 @@ public void A_single_pathological_branch_cannot_spend_the_budget_its_siblings_ne var projected = MapResultsPrompt.Project(results, Budget).Text; foreach (var (element, i) in results.EnumerateArray().Select((e, i) => (e, i)).Where(x => x.i != pathologicalIndex)) - projected.ShouldContain(JsonSerializer.Serialize(element), + projected.ShouldContain(JsonSerializer.Serialize(element, WorkflowJson.InterpolatedText), customMessage: $"small sibling {i} must survive in FULL — one oversized branch may not evict it, wherever the oversized branch sits"); CountBranchMarkers(projected).ShouldBe(1, @@ -151,6 +152,9 @@ public void The_within_budget_prompt_resolves_identically_to_the_raw_array_bindi { Obj("""{"status":"Succeeded","summary":"renamed the module","changedFiles":["a.cs","b.cs"]}"""), Obj("""{"status":"Succeeded","summary":"added the tests","changedFiles":["c.cs"]}"""), + // Characters the resolver and the projection must write the same way. With two different encoders these + // two strings agree on plain ASCII and diverge on the first CJK character, '+', '<', '&' or quote. + Obj("""{"status":"Succeeded","summary":"修复 List & 'x' a+b","changedFiles":["d.cs"]}"""), }; var outputs = WorkflowEngine.BuildMapOutputs("results", results, failed: 0, promptBudgetChars: Budget); @@ -245,8 +249,8 @@ public void Complete_is_recorded_true_exactly_when_nothing_was_dropped(int branc var projection = MapResultsPrompt.Project(results, budget); - projection.Coverage.Complete.ShouldBe(projection.Text == JsonSerializer.Serialize(results), - customMessage: $"{branches} branches of {branchChars} chars at a {budget}-char budget recorded Complete={projection.Coverage.Complete} over a text that is {(projection.Text == JsonSerializer.Serialize(results) ? "" : "NOT ")}the whole serialization"); + projection.Coverage.Complete.ShouldBe(projection.Text == JsonSerializer.Serialize(results, WorkflowJson.InterpolatedText), + customMessage: $"{branches} branches of {branchChars} chars at a {budget}-char budget recorded Complete={projection.Coverage.Complete} over a text that is {(projection.Text == JsonSerializer.Serialize(results, WorkflowJson.InterpolatedText) ? "" : "NOT ")}the whole serialization"); } /// diff --git a/backend/tests/CodeSpace.UnitTests/Workflows/VariableResolverTests.cs b/backend/tests/CodeSpace.UnitTests/Workflows/VariableResolverTests.cs index 30baddbf5..e0f73b70d 100644 --- a/backend/tests/CodeSpace.UnitTests/Workflows/VariableResolverTests.cs +++ b/backend/tests/CodeSpace.UnitTests/Workflows/VariableResolverTests.cs @@ -44,6 +44,37 @@ public void Sole_template_preserves_native_type() resolved.GetInt32().ShouldBe(42); } + [Fact] + public void An_object_interpolated_into_text_keeps_its_characters_readable() + { + // A PR-review goal binds {{nodes.fetch_diff.outputs.files}} into prose. The object branch used to serialize with + // the HTML-safe default encoder, so every CJK character and every + < > & ' of the diff reached the model as a + // six-character \uXXXX — "修复" became \u4FEE\u590D, every added line began \u002B — at up to twice the size. The + // string branch beside it already inserts all of those raw, so the escaping protected nothing. (A character + // outside the Basic Multilingual Plane — an emoji — is still escaped: every built-in encoder does that.) + var scope = new NodeRunScope { Trigger = ParseDict("""{ "files": [ { "path": "src/a.cs", "patch": "+ // 修复：List & 'x' a+b" } ] }""") }; + + var resolved = VariableResolver.Resolve(ParseElement("\"请审查：\\n{{trigger.files}}\""), scope).GetString()!; + + resolved.ShouldContain("修复", customMessage: "the model must read the diff's own text, not escape codes"); + resolved.ShouldContain("+ // "); + resolved.ShouldContain("List & 'x' a+b"); + resolved.ShouldNotContain("\\u", Case.Insensitive, "no character of the interpolated value may reach the prompt as a \\uXXXX escape"); + } + + [Fact] + public void An_object_interpolated_into_text_is_still_valid_json_that_round_trips() + { + // The relaxed encoder still escapes what JSON itself requires — the quote, the backslash and control + // characters — so a template that embeds a value in a JSON body keeps a parseable body. + var scope = new NodeRunScope { Trigger = ParseDict("""{ "value": { "text": "a \"quoted\" \\ path\nsecond line 修复" } }""") }; + + var resolved = VariableResolver.Resolve(ParseElement("\"{\\\"wrapped\\\": {{trigger.value}}}\""), scope).GetString()!; + + using var document = JsonDocument.Parse(resolved); + document.RootElement.GetProperty("wrapped").GetProperty("text").GetString().ShouldBe("a \"quoted\" \\ path\nsecond line 修复"); + } + [Fact] public void Multiple_templates_concatenate_into_string() { From 5336e5cf8ed51c8a31daeebfe5c66dd9e3e9267d Mon Sep 17 00:00:00 2001 From: "Mars.P" Date: Wed, 23 Sep 2026 20:34:54 +0800 Subject: [PATCH 02/16] Stop respawning an agent whose goal overflows the model's context MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the model refused a request as larger than its context window, the run failed as an ordinary non-zero exit and the agent.run node respawned it. A respawn warm-resumes the conversation, so its request carries the goal again and overflows harder; a task-launched agent node, which defaults to three attempts, ended on three identical billed refusals. The text the author saw was the CLI's advice to its interactive user — trim your tools, start a new thread — which a workflow author cannot act on. AgentRetryCauses gains ContextWindowExceeded, matched against the phrases the CLIs themselves print, each observed from the real binary answered with the provider's own error body: Claude's "Prompt is too long", a gateway's "maximum context length is", OpenAI's context_length_exceeded and its message, Codex's rewording of a streaming failure, and Codex's own input_too_large. The tests run those real lines through ParseEvents and BuildResult before classifying, so the markers are pinned to what a failed run actually carries. The node treats the cause as deterministic unless a stronger model is on offer, and names it with the author's remedy. The cause carries no mitigation. The escalation trigger only escalates after a failed acceptance grade, which an overflowing run never reaches, and every other consumer keys on the format-fault cause specifically. --- .../Services/Supervisor/AgentRetryCauses.cs | 28 ++++++ .../Workflows/Nodes/Builtin/AgentCodeNode.cs | 36 ++++--- .../Agents/AgentContextWindowRetryTests.cs | 96 +++++++++++++++++++ .../Workflows/AgentCodeNodeTests.cs | 17 ++++ 4 files changed, 164 insertions(+), 13 deletions(-) create mode 100644 backend/tests/CodeSpace.UnitTests/Agents/AgentContextWindowRetryTests.cs diff --git a/backend/src/CodeSpace.Core/Services/Supervisor/AgentRetryCauses.cs b/backend/src/CodeSpace.Core/Services/Supervisor/AgentRetryCauses.cs index db676df02..8a548fd5e 100644 --- a/backend/src/CodeSpace.Core/Services/Supervisor/AgentRetryCauses.cs +++ b/backend/src/CodeSpace.Core/Services/Supervisor/AgentRetryCauses.cs @@ -31,9 +31,34 @@ public static class AgentRetryCauses /// public const string ModelAccessLost = "model-access-lost"; + /// + /// The model refused the request as larger than its context window — deterministic on replay. A retry warm-resumes + /// the conversation, so its request carries the goal AGAIN and is longer still; a fresh one carries the same goal. + /// Either way the same refusal comes back, billed, and buries the one fact the author needs: the goal is too big + /// for this model. No mitigation, like — its only consumer that matters is + /// AgentCodeNode, which stops respawning it. + /// + public const string ContextWindowExceeded = "context-window-exceeded"; + /// Seen live 2026-08-30 (run wedge postmortem): the gateway's Anthropic-compat layer broke thinking-block continuation and killed the agent tail with exactly this text. private static readonly string[] FormatFaultMarkers = { "is not a thinking block" }; + /// + /// What the CLIs themselves print for an over-long prompt — each observed from the real binary (Claude Code + /// 2.1.226, Codex 0.147.0 and 0.142.2) answered with the provider's own error body, and pinned through the real + /// harness folds by AgentContextWindowRetryTests. Provider- and CLI-authored phrases, not words an agent's + /// own prose is likely to end on; the vocabulary stays closed like the one above. + /// + private static readonly string[] ContextWindowMarkers = + { + "Prompt is too long", // Claude Code, for either Anthropic overflow body (terminal_reason=prompt_too_long) + "maximum context length is", // OpenAI-compatible gateways (vLLM, LiteLLM), passed through verbatim + "context_length_exceeded", // OpenAI error code, which Codex passes through + "exceeds the context window", // OpenAI's message for the same code + "out of room in the model's context window", // Codex's rewording of a streaming response.failed + "input_too_large", // Codex refusing an input past its own 1,048,576-character cap + }; + /// /// The prior attempt's retry-relevant cause, reading its DECLARED exit reason before any text. A typed code is this /// codebase's own diagnosis and can never be prose about one, so it settles the question outright; only an attempt @@ -50,6 +75,9 @@ public static class AgentRetryCauses foreach (var marker in FormatFaultMarkers) if (error.Contains(marker, StringComparison.OrdinalIgnoreCase)) return GatewayFormatFault; + foreach (var marker in ContextWindowMarkers) + if (error.Contains(marker, StringComparison.OrdinalIgnoreCase)) return ContextWindowExceeded; + return null; } diff --git a/backend/src/CodeSpace.Core/Services/Workflows/Nodes/Builtin/AgentCodeNode.cs b/backend/src/CodeSpace.Core/Services/Workflows/Nodes/Builtin/AgentCodeNode.cs index 6a6e629e9..734dddb54 100644 --- a/backend/src/CodeSpace.Core/Services/Workflows/Nodes/Builtin/AgentCodeNode.cs +++ b/backend/src/CodeSpace.Core/Services/Workflows/Nodes/Builtin/AgentCodeNode.cs @@ -373,16 +373,22 @@ private static NodeResult MapResult(JsonElement payload, decimal? maxCostUsd) // that ALREADY ran mitigated (`thinkingDisabled`, projected from its own dispatched envelope) and died // of the SAME fault has proven the repair does not hold here, so a second identical respawn would only // re-bill a broken gateway and bury the one fact the operator needs. - var formatFault = Supervisor.AgentRetryCauses.Classify(error) == Supervisor.AgentRetryCauses.GatewayFormatFault; + var cause = Supervisor.AgentRetryCauses.Classify(error); + var formatFault = cause == Supervisor.AgentRetryCauses.GatewayFormatFault; var mitigationSpent = formatFault && ReadFlag(payload, "thinkingDisabled"); - var deterministic = ((status is nameof(AgentRunStatus.NeedsReview) or nameof(AgentRunStatus.Cancelled) || acceptanceFailed || resourceExhausted || argumentTooLong) + // The model refused the request as larger than its context window. A respawn warm-resumes, so it re-sends + // the goal inside a LONGER request and is refused again, harder — every attempt after the first is an + // identical, billed refusal. Deterministic unless a stronger model is on offer, which may have a larger window. + var contextWindowExceeded = cause == Supervisor.AgentRetryCauses.ContextWindowExceeded; + + var deterministic = ((status is nameof(AgentRunStatus.NeedsReview) or nameof(AgentRunStatus.Cancelled) || acceptanceFailed || resourceExhausted || argumentTooLong || contextWindowExceeded) && !acceptanceInfraFault && !stalled && !escalationAvailable) || mitigationSpent; - return NodeResult.Fail($"Agent run did not succeed: {(string.IsNullOrEmpty(error) ? status : error)}{FailureCauseSuffix(formatFault, mitigationSpent)}", retryable: !deterministic); + return NodeResult.Fail($"Agent run did not succeed: {(string.IsNullOrEmpty(error) ? status : error)}{FailureCauseSuffix(cause, mitigationSpent)}", retryable: !deterministic); } var outputs = new Dictionary { ["status"] = JsonSerializer.SerializeToElement(nameof(AgentRunStatus.Succeeded)) }; @@ -400,17 +406,21 @@ private static NodeResult MapResult(JsonElement payload, decimal? maxCostUsd) } /// - /// Name the CAUSE on a gateway format fault's failure message — the text the engine persists as this attempt's - /// attempt.failed / node.failed record, so the run's timeline says the gateway mangled the wire - /// instead of only echoing an opaque CLI line. Deliberately states no more than is true at that instant: it - /// never promises a respawn (whether one is bought is the node's retry budget, which this node cannot see), - /// and on the spent-mitigation arm it says the repair already ran. Empty for every other failure, so every - /// existing message stays byte-identical. + /// Name the CAUSE on a classified failure's message — the text the engine persists as this attempt's + /// attempt.failed / node.failed record, so the run's timeline says what happened instead of only + /// echoing an opaque CLI line. Deliberately states no more than is true at that instant: it never promises a + /// respawn (whether one is bought is the node's retry budget, which this node cannot see), and on the + /// spent-mitigation arm it says the repair already ran. A context overflow gets the author's remedy, because the + /// CLI's own text advises its interactive user (trim your tools, start a new thread), which a workflow author + /// cannot act on. Empty for every other failure, so every existing message stays byte-identical. /// - private static string FailureCauseSuffix(bool formatFault, bool mitigationSpent) => - !formatFault ? "" - : mitigationSpent ? $" ({Supervisor.AgentRetryCauses.GatewayFormatFault}: a fresh conversation with extended thinking disabled hit the same fault)" - : $" ({Supervisor.AgentRetryCauses.GatewayFormatFault})"; + private static string FailureCauseSuffix(string? cause, bool mitigationSpent) => cause switch + { + Supervisor.AgentRetryCauses.GatewayFormatFault when mitigationSpent => $" ({Supervisor.AgentRetryCauses.GatewayFormatFault}: a fresh conversation with extended thinking disabled hit the same fault)", + Supervisor.AgentRetryCauses.GatewayFormatFault => $" ({Supervisor.AgentRetryCauses.GatewayFormatFault})", + Supervisor.AgentRetryCauses.ContextWindowExceeded => $" ({Supervisor.AgentRetryCauses.ContextWindowExceeded}: the goal is larger than this model's context window, so every attempt is refused the same way — shorten the goal, or choose a model with a larger window)", + _ => "", + }; /// /// P2.3: stamp the retry-resume hint from the RETIRING prior attempt's own resume payload (the same diff --git a/backend/tests/CodeSpace.UnitTests/Agents/AgentContextWindowRetryTests.cs b/backend/tests/CodeSpace.UnitTests/Agents/AgentContextWindowRetryTests.cs new file mode 100644 index 000000000..98b913869 --- /dev/null +++ b/backend/tests/CodeSpace.UnitTests/Agents/AgentContextWindowRetryTests.cs @@ -0,0 +1,96 @@ +using CodeSpace.Core.Services.Agents; +using CodeSpace.Core.Services.Agents.Harnesses.Claude; +using CodeSpace.Core.Services.Agents.Harnesses.Codex; +using CodeSpace.Core.Services.Supervisor; +using CodeSpace.Messages.Agents; +using CodeSpace.Messages.Enums; +using Shouldly; + +namespace CodeSpace.UnitTests.Agents; + +/// +/// A context-window overflow is recognised from what the CLIs actually print, through the real harness folds. +/// +/// Every shape below is a terminal line a real CLI printed (Claude Code 2.1.226, Codex 0.147.0) when a local +/// endpoint answered its request with the provider's own over-long-prompt error body, trimmed to the fields the +/// fold reads. They are run through ParseEvents and BuildResult — the reader production uses — and only +/// then classified, so the markers are pinned against the text a failed run really carries, not against strings +/// written to match them. +/// +/// Why it matters: an overflow used to be an ordinary retryable failure. A retry warm-resumes, so the next +/// request carries the goal twice and overflows harder; a task-launched agent node spends all three default +/// attempts that way and ends on three identical refusals. +/// +[Trait("Category", "Unit")] +public sealed class AgentContextWindowRetryTests +{ + public static TheoryData ClaudeOverflowLines => new() + { + // Anthropic "prompt is too long: N tokens > M maximum" — the CLI's own rewording, stamped terminal_reason=prompt_too_long. + """{"type":"result","subtype":"success","is_error":true,"num_turns":1,"terminal_reason":"prompt_too_long","api_error_status":400,"session_id":"8d014f31-0881-46c2-8311-2ad17d852c4f","result":"Prompt is too long · the request is ~215000 tokens (limit 200000) but this conversation is only ~30286 tokens — the rest is system prompt, tool definitions, and attachment content. A single-exchange conversation cannot be compacted; reduce attached files/tools or start with less context."}""", + // Anthropic "input length and max_tokens exceed context limit". + """{"type":"result","subtype":"success","is_error":true,"num_turns":1,"terminal_reason":"prompt_too_long","api_error_status":400,"session_id":"58110d54-f8b2-43f3-9153-dcd32a2d58dc","result":"Prompt is too long · this conversation is a single exchange and cannot be compacted — the request size comes mostly from system prompt, tool definitions, or attachments."}""", + // An OpenAI-compatible gateway (vLLM/LiteLLM) body the CLI passes through verbatim as terminal_reason=api_error. + """{"type":"result","subtype":"success","is_error":true,"num_turns":1,"terminal_reason":"api_error","api_error_status":400,"session_id":"6406b570-40de-4274-a586-ab044dbf5d48","result":"API Error: 400 This model's maximum context length is 131072 tokens. However, you requested 161234 tokens. Please reduce the length of the messages."}""", + }; + + public static TheoryData CodexOverflowLines => new() + { + // OpenAI Responses API 400 context_length_exceeded. + """{"type":"turn.failed","error":{"message":"{\"error\": {\"message\": \"Your input exceeds the context window of this model. Please adjust your input and try again.\", \"type\": \"invalid_request_error\", \"param\": \"input\", \"code\": \"context_length_exceeded\"}}"}}""", + // The same refusal delivered as a streaming response.failed, which Codex rewords. + """{"type":"turn.failed","error":{"message":"Codex ran out of room in the model's context window. Start a new thread or clear earlier history before retrying."}}""", + }; + + [Theory] + [MemberData(nameof(ClaudeOverflowLines))] + public void A_claude_context_overflow_folds_to_an_error_classified_as_one(string terminalLine) + { + var harness = new ClaudeCodeHarness(); + + var result = harness.BuildResult(harness.ParseEvents(terminalLine), exitCode: 1, diagnostics: ""); + + result.Status.ShouldBe(AgentRunStatus.Failed); + AgentRetryCauses.Classify(result.Error).ShouldBe(AgentRetryCauses.ContextWindowExceeded, customMessage: $"the folded error was: {result.Error}"); + } + + [Theory] + [MemberData(nameof(CodexOverflowLines))] + public void A_codex_context_overflow_folds_to_an_error_classified_as_one(string terminalLine) + { + var harness = new CodexHarness(); + + var result = harness.BuildResult(harness.ParseEvents(terminalLine), exitCode: 1, diagnostics: ""); + + result.Status.ShouldBe(AgentRunStatus.Failed); + AgentRetryCauses.Classify(result.Error).ShouldBe(AgentRetryCauses.ContextWindowExceeded, customMessage: $"the folded error was: {result.Error}"); + } + + [Fact] + public void Codex_refusing_an_input_past_its_own_character_cap_is_classified_the_same_way() + { + // Codex refuses before any request, on stderr only — so the only carrier is the diagnostics excerpt the fold + // appends to a bare exit. Pinned against codex 0.142.2 (the worker's version) as well as 0.147.0. + const string stderr = """Error: turn/start: turn/start failed: Input exceeds the maximum length of 1048576 characters. (code -32602), data: {"input_error_code":"input_too_large","max_chars":1048576,"actual_chars":1100000}"""; + + var result = new CodexHarness().BuildResult(Array.Empty(), exitCode: 1, diagnostics: stderr); + + AgentRetryCauses.Classify(result.Error).ShouldBe(AgentRetryCauses.ContextWindowExceeded, customMessage: $"the folded error was: {result.Error}"); + } + + [Theory] + [InlineData("claude exited with code 1")] + [InlineData("patch did not apply")] + [InlineData("API Error: 529 Overloaded")] + [InlineData("codex exited with code 1 — stderr: npm WARN deprecated glob@7")] + public void An_ordinary_failure_is_not_mistaken_for_an_overflow(string error) + { + AgentRetryCauses.Classify(error).ShouldBeNull(customMessage: "an ordinary death keeps the default resume-and-retry semantics"); + } + + [Fact] + public void The_format_fault_keeps_its_own_cause() + { + AgentRetryCauses.Classify("API Error: 400 messages.3.content.0: Content block is not a thinking block").ShouldBe(AgentRetryCauses.GatewayFormatFault); + } +} diff --git a/backend/tests/CodeSpace.UnitTests/Workflows/AgentCodeNodeTests.cs b/backend/tests/CodeSpace.UnitTests/Workflows/AgentCodeNodeTests.cs index b0f34a4f3..32b37992d 100644 --- a/backend/tests/CodeSpace.UnitTests/Workflows/AgentCodeNodeTests.cs +++ b/backend/tests/CodeSpace.UnitTests/Workflows/AgentCodeNodeTests.cs @@ -547,6 +547,23 @@ public async Task A_resumed_failure_carries_the_retry_verdict_for_the_engine(str result.Retryable.ShouldBe(expectedRetryable, "the node's verdict tells the retry policy whether a fresh agent could change the outcome"); } + [Theory] + [InlineData("Prompt is too long · the request is ~215000 tokens (limit 200000) but this conversation is only ~30286 tokens")] + [InlineData("API Error: 400 This model's maximum context length is 131072 tokens. However, you requested 161234 tokens.")] + [InlineData("Codex ran out of room in the model's context window. Start a new thread or clear earlier history before retrying.")] + public async Task A_context_window_overflow_is_not_respawned(string error) + { + // A respawn warm-resumes the conversation, so it re-sends the goal inside a LONGER request — it overflows again, + // harder. Every attempt after the first is an identical, billed refusal that buries the one fact the author needs. + var resume = JsonDocument.Parse(JsonSerializer.Serialize(new { status = "Failed", error, exitReason = "non-zero-exit" })).RootElement; + + var result = await new AgentCodeNode().RunAsync(BuildContext(new(), resume), CancellationToken.None); + + result.Status.ShouldBe(NodeStatus.Failure); + result.Retryable.ShouldBeFalse(); + result.Error.ShouldContain("context window", Case.Insensitive, customMessage: "the node's own message must name the cause, whatever advice the CLI's text gives"); + } + [Fact] public void The_two_watchdogs_are_classified_alike_because_neither_can_see_why_the_process_went_quiet() { From 26a1d018ff563a747d41d0035dc492dd007c2eeb Mon Sep 17 00:00:00 2001 From: "Mars.P" Date: Wed, 23 Sep 2026 20:34:54 +0800 Subject: [PATCH 03/16] Refuse a goal past Codex's own input cap before launching it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit codex-rs refuses a turn/start input longer than 1,048,576 characters with input_too_large before any model request, and says so on stderr only — verified against 0.142.2, the worker's pin, and 0.147.0. The platform provisioned a sandbox and cloned the repository for a goal Codex would reject in the first second, and the stdin preflight allows up to 8 MiB encoded, well past that cap. The limit belongs to this CLI, so CodexHarness owns it (Rule 7): BuildInvocation refuses a longer goal with the same terminal SandboxArgumentTooLongException an argument the kernel cannot take gets, naming the goal's size and the cap and never the goal. Counted as Unicode scalar values, the way Rust counts a string's characters, so an emoji-heavy goal Codex accepts is not refused on its UTF-16 length. The cap is pinned by a test and moves by PR with the CLI; the context-window classifier also recognises the CLI's own refusal as the backstop. --- .../Agents/Harnesses/Codex/CodexHarness.cs | 24 +++++++++++ .../SandboxArgumentTooLongException.cs | 3 +- .../Workflows/CodexHarnessTests.cs | 43 +++++++++++++++++++ 3 files changed, 69 insertions(+), 1 deletion(-) diff --git a/backend/src/CodeSpace.Core/Services/Agents/Harnesses/Codex/CodexHarness.cs b/backend/src/CodeSpace.Core/Services/Agents/Harnesses/Codex/CodexHarness.cs index e52c55c52..ba877da98 100644 --- a/backend/src/CodeSpace.Core/Services/Agents/Harnesses/Codex/CodexHarness.cs +++ b/backend/src/CodeSpace.Core/Services/Agents/Harnesses/Codex/CodexHarness.cs @@ -1,6 +1,7 @@ using System.Text.Json; using CodeSpace.Core.DependencyInjection; using CodeSpace.Core.Services.Agents.Mcp; +using CodeSpace.Core.Services.Agents.Sandbox.Exceptions; using CodeSpace.Core.Services.Agents.Skills; using CodeSpace.Messages.Agents; using CodeSpace.Messages.Enums; @@ -136,8 +137,18 @@ public sealed class CodexHarness : IAgentHarness, IAgentHarnessBinary, IAgentHar public IReadOnlyList Models { get; } = new[] { "gpt-5.3-codex", "gpt-5.4", "gpt-5.4-codex" }; + /// + /// The most characters Codex accepts in one input. codex-rs refuses a longer turn/start with + /// input_too_large before any model request — verified against 0.142.2 (the worker's pin) and 0.147.0 — and + /// says so on stderr only. Counted as Unicode scalar values, the way Rust counts a string's characters. Pinned by a + /// test; it moves by PR with the CLI. + /// + public const int MaxInputCharacters = 1_048_576; + public SandboxSpec BuildInvocation(AgentTask task) { + EnsureWithinInputCap(task.Goal); + // P3.2: a CONTINUE re-stage rewrites the `exec --json` seed to `exec resume --json` so Codex picks up the // prior thread. The subcommand must follow `exec` directly; --model, the `-c` overrides (incl. the sandbox on // the resume path — see AppendSandbox), and the stdin `-` positional follow. Null (a fresh run) → the plain seed. @@ -287,6 +298,19 @@ private static bool TryReadModelKey(JsonElement obj, out string model) return model.Length > 0; } + /// + /// Refuse a goal Codex would refuse, before a sandbox is provisioned for it. The limit belongs to this CLI, not to + /// the launch (Rule 7), so it lives on this harness; the refusal is the same terminal, non-retried one an argument + /// the kernel cannot take gets, because every attempt meets the identical cap. + /// + private static void EnsureWithinInputCap(string goal) + { + var characters = goal.EnumerateRunes().Count(); + + if (characters > MaxInputCharacters) + throw new SandboxArgumentTooLongException($"the agent's goal is {characters} characters; codex accepts at most {MaxInputCharacters} in one input and refuses anything longer before any model request. This is an input-size limit of the codex CLI, not a memory limit — shorten the goal, or run this agent on a harness without that cap."); + } + /// /// The config-home files the runner materializes: (1) B1 — AGENTS.md carrying the persona + the always-on /// operating contract (Codex's native instruction channel, since exec has no system-prompt flag; codex loads diff --git a/backend/src/CodeSpace.Core/Services/Agents/Sandbox/Exceptions/SandboxArgumentTooLongException.cs b/backend/src/CodeSpace.Core/Services/Agents/Sandbox/Exceptions/SandboxArgumentTooLongException.cs index 100548fd4..1273cceef 100644 --- a/backend/src/CodeSpace.Core/Services/Agents/Sandbox/Exceptions/SandboxArgumentTooLongException.cs +++ b/backend/src/CodeSpace.Core/Services/Agents/Sandbox/Exceptions/SandboxArgumentTooLongException.cs @@ -4,7 +4,8 @@ namespace CodeSpace.Core.Services.Agents.Sandbox.Exceptions; /// /// The invocation cannot be executed as written: one of its argv or environment strings is past the kernel's -/// per-string ceiling, or its standard input is past what the launch pipe can carry. Its own exit scenario rather than a reason, because the two +/// per-string ceiling, its standard input is past what the launch pipe can carry, or its goal is past the harness +/// CLI's own input cap (Codex's, which refuses before any model request). Its own exit scenario rather than a reason, because the two /// differ in the one way a caller acts on. A launch-slot refusal is about THIS host — a foreign boot, a binding /// conflict, a missing bootstrap — and another worker may well admit it, so it stays retryable. These bytes are /// refused by every kernel on every host, so a retry is N identical refusals, each one burning budget and burying the diff --git a/backend/tests/CodeSpace.UnitTests/Workflows/CodexHarnessTests.cs b/backend/tests/CodeSpace.UnitTests/Workflows/CodexHarnessTests.cs index 832af2c8b..be0c8a2ca 100644 --- a/backend/tests/CodeSpace.UnitTests/Workflows/CodexHarnessTests.cs +++ b/backend/tests/CodeSpace.UnitTests/Workflows/CodexHarnessTests.cs @@ -1,3 +1,6 @@ +using CodeSpace.Messages.Failures; +using CodeSpace.Core.Services.Agents.Sandbox.Exceptions; +using System.Globalization; using CodeSpace.Core.Services.Agents.Sandbox; using CodeSpace.Core.Services.Agents; using CodeSpace.Core.Services.Agents.Harnesses.Codex; @@ -51,6 +54,46 @@ public void A_resumed_thread_takes_its_prompt_from_stdin_behind_the_dash() spec.StandardInput.ShouldBe("Fix the failing billing tests"); } + [Fact] + public void Codexs_own_input_cap_is_pinned() + { + // codex-rs refuses a turn/start input past this many characters (input_too_large) before any model request — + // verified against 0.142.2, the worker's pin, and 0.147.0. If a Codex bump moves it, this moves by PR. + CodexHarness.MaxInputCharacters.ShouldBe(1_048_576); + } + + [Fact] + public void A_goal_at_codexs_input_cap_builds_an_invocation() + { + Should.NotThrow(() => Harness.BuildInvocation(Task(goal: new string('x', CodexHarness.MaxInputCharacters)))); + } + + [Fact] + public void A_goal_past_codexs_input_cap_is_refused_before_launch_as_deterministic() + { + // Without this the launch succeeds, `codex exec -` reads stdin, and exits 1 before any request with the refusal + // on stderr only — a run that clones, provisions and launches only to be told no, and that a retry reproduces. + var goal = new string('x', CodexHarness.MaxInputCharacters + 1); + + var refusal = Should.Throw(() => Harness.BuildInvocation(Task(goal: goal))); + + ((IFailure)refusal).Code.ShouldBe(FailureCodes.SandboxArgumentTooLong, customMessage: "every attempt is refused identically, so it must not be retried"); + refusal.Message.ShouldContain(CodexHarness.MaxInputCharacters.ToString(CultureInfo.InvariantCulture)); + refusal.Message.ShouldContain((CodexHarness.MaxInputCharacters + 1).ToString(CultureInfo.InvariantCulture), customMessage: "the author needs to know how far over the goal is"); + refusal.Message.ShouldNotContain("xxxx", Case.Sensitive, "a refusal is host metadata — never the goal itself"); + } + + [Fact] + public void Codexs_input_cap_counts_characters_not_utf16_units() + { + // Codex is Rust: its "characters" are Unicode scalar values. An emoji is one of them and two UTF-16 units, so a + // UTF-16 count would refuse a goal Codex accepts. + var emoji = string.Concat(Enumerable.Repeat("🚀", CodexHarness.MaxInputCharacters / 2 + 1)); + emoji.Length.ShouldBeGreaterThan(CodexHarness.MaxInputCharacters, "fixture check: over the cap in UTF-16 units"); + + Should.NotThrow(() => Harness.BuildInvocation(Task(goal: emoji))); + } + [Fact] public void A_goal_past_the_argv_ceiling_builds_an_invocation_the_kernel_accepts() { From 2bc33ffbe8b18f76da92e575a49a23edae0f1f8e Mon Sep 17 00:00:00 2001 From: "Mars.P" Date: Thu, 24 Sep 2026 06:49:32 +0800 Subject: [PATCH 04/16] Type a context overflow from the CLI's own fields, not its prose The overflow cause was a substring scan over the run's error text. A fail-closed acceptance verdict overwrites that text with the rubric's own requirement and the judge's evidence, so a review whose rubric is about context windows classified as an overflow - and the escalation trigger, which stands down for any classified cause, stopped offering the stronger model the failed grade was evidence for. A crash whose last agent message quoted an overflow read the same way. Each harness folder now decides from what its CLI wrote: Claude's terminal_reason (or a 400 api_error relaying a gateway overflow body), Codex's terminal turn.failed (the provider's typed error code, or its own streamed-refusal sentence). The fold stamps a dedicated exit reason, and the classifier reads it off the exit reason only; text never classifies an overflow. A 5xx whose body mentions context length stays an ordinary, retryable failure. Codex refusing an input past its own character cap is not the model's window, so it folds to sandbox_argument_too_long - the code the harness preflight already throws - instead of advice to pick a model with a larger window, which could not help. --- .../Agents/AgentTerminalOutcomeReader.cs | 9 +++ .../Claude/ClaudeCodeResultFolder.cs | 44 ++++++++++- .../Harnesses/Codex/CodexResultFolder.cs | 52 ++++++++++++- .../Services/Supervisor/AgentRetryCauses.cs | 43 ++++------- .../Workflows/Nodes/Builtin/AgentCodeNode.cs | 12 +-- .../Agents/AgentContextWindowRetryTests.cs | 77 ++++++++++++++++--- .../Workflows/AgentCodeNodeTests.cs | 2 +- 7 files changed, 192 insertions(+), 47 deletions(-) diff --git a/backend/src/CodeSpace.Core/Services/Agents/AgentTerminalOutcomeReader.cs b/backend/src/CodeSpace.Core/Services/Agents/AgentTerminalOutcomeReader.cs index e4ef53398..9c55d3b13 100644 --- a/backend/src/CodeSpace.Core/Services/Agents/AgentTerminalOutcomeReader.cs +++ b/backend/src/CodeSpace.Core/Services/Agents/AgentTerminalOutcomeReader.cs @@ -13,6 +13,15 @@ namespace CodeSpace.Core.Services.Agents; /// public static class AgentTerminalOutcomeReader { + /// + /// The a harness's folder stamps when its CLI's OWN terminal event says the + /// model refused the request as larger than its context window — a field or status the CLI wrote, never a phrase in + /// the agent's prose. Harness-agnostic, like the rest of this reader: each folder knows its CLI's shape, and every + /// consumer (the retry-cause classifier, and through it the agent.run node's retry verdict) keys on this one code. + /// Pinned by a unit test (Rule 8) so the producers and the verdict cannot drift apart. + /// + public const string ContextWindowExceededExitReason = "context-window-exceeded"; + /// /// True when the last Completed-or-Error event in the stream is an Error — i.e. the harness itself reported /// the run failed, even if the OS exit code was 0. False when no such event exists (nothing to reconcile diff --git a/backend/src/CodeSpace.Core/Services/Agents/Harnesses/Claude/ClaudeCodeResultFolder.cs b/backend/src/CodeSpace.Core/Services/Agents/Harnesses/Claude/ClaudeCodeResultFolder.cs index 313d4601c..e9cf8f525 100644 --- a/backend/src/CodeSpace.Core/Services/Agents/Harnesses/Claude/ClaudeCodeResultFolder.cs +++ b/backend/src/CodeSpace.Core/Services/Agents/Harnesses/Claude/ClaudeCodeResultFolder.cs @@ -1,3 +1,4 @@ +using System.Text.Json; using CodeSpace.Core.Services.Agents.Sandbox; using CodeSpace.Messages.Agents; using CodeSpace.Messages.Enums; @@ -12,9 +13,22 @@ namespace CodeSpace.Core.Services.Agents.Harnesses.Claude; /// internal sealed class ClaudeCodeResultFolder : IAgentEventFolder { + /// + /// What an OpenAI-compatible gateway (vLLM, LiteLLM) answers an over-long request with, which the CLI passes + /// through verbatim as terminal_reason: api_error — the one overflow shape it does not stamp as its own. + /// Observed from Claude Code 2.1.226 answered with each body; read only off a 400 refusal on the result line. + /// + private static readonly string[] GatewayOverflowMarkers = { "maximum context length is", "context_length_exceeded" }; + private readonly AgentResultFold _fold = new(); + private JsonElement? _lastErrorLine; + + public void Add(AgentEvent normalized) + { + _fold.Add(normalized); - public void Add(AgentEvent normalized) => _fold.Add(normalized); + if (normalized.Kind == AgentEventKind.Error) _lastErrorLine = normalized.Data; + } public AgentRunResult BuildResult(AgentRunFacts facts, int exitCode, string diagnostics) { @@ -52,8 +66,34 @@ public AgentRunResult BuildResult(AgentRunFacts facts, int exitCode, string diag ?? (string.IsNullOrWhiteSpace(summary) ? null : summary) ?? AgentDiagnosticExcerpt.Explain($"claude exited with code {SandboxExitCode.Describe(exitCode)}", diagnostics); - var exitReason = exitCode != 0 ? "non-zero-exit" : "harness-reported-failure"; + var exitReason = _fold.ReportedFailure && RefusedAsOverContextWindow(_lastErrorLine) ? AgentTerminalOutcomeReader.ContextWindowExceededExitReason + : exitCode != 0 ? "non-zero-exit" : "harness-reported-failure"; return new AgentRunResult { Status = AgentRunStatus.Failed, ExitReason = exitReason, Summary = summary, ChangedFiles = changedFiles, Error = error, TokenUsage = usage, SessionId = sessionId, Model = model }; } + + /// + /// Whether the CLI's own terminal result line says the model refused the request as larger than its context + /// window. Read off fields the CLI writes (terminal_reason, api_error_status) and, for the gateway + /// shape, off the refusal body it relays — never off the agent's prose, which is how a crash's last message or a + /// rubric's wording would otherwise pass for a diagnosis. A 5xx is the gateway failing, not the model refusing, + /// so it stays an ordinary, retryable failure whatever its body says. + /// + private static bool RefusedAsOverContextWindow(JsonElement? line) + { + if (line is not { ValueKind: JsonValueKind.Object } result) return false; + + var terminalReason = ReadString(result, "terminal_reason"); + + if (terminalReason == "prompt_too_long") return true; + + if (terminalReason != "api_error" || !result.TryGetProperty("api_error_status", out var status) || !status.TryGetInt32(out var code) || code != 400) return false; + + var body = ReadString(result, "result"); + + return GatewayOverflowMarkers.Any(marker => body.Contains(marker, StringComparison.OrdinalIgnoreCase)); + } + + private static string ReadString(JsonElement root, string key) => + root.TryGetProperty(key, out var value) && value.ValueKind == JsonValueKind.String ? value.GetString() ?? "" : ""; } diff --git a/backend/src/CodeSpace.Core/Services/Agents/Harnesses/Codex/CodexResultFolder.cs b/backend/src/CodeSpace.Core/Services/Agents/Harnesses/Codex/CodexResultFolder.cs index da7781d65..32ca067a3 100644 --- a/backend/src/CodeSpace.Core/Services/Agents/Harnesses/Codex/CodexResultFolder.cs +++ b/backend/src/CodeSpace.Core/Services/Agents/Harnesses/Codex/CodexResultFolder.cs @@ -1,6 +1,8 @@ +using System.Text.Json; using CodeSpace.Core.Services.Agents.Sandbox; using CodeSpace.Messages.Agents; using CodeSpace.Messages.Enums; +using CodeSpace.Messages.Failures; namespace CodeSpace.Core.Services.Agents.Harnesses.Codex; @@ -12,9 +14,21 @@ namespace CodeSpace.Core.Services.Agents.Harnesses.Codex; /// internal sealed class CodexResultFolder : IAgentEventFolder { + /// Codex's own sentence for a model refusing a streamed request as over its window (a response.failed, reworded) — observed from Codex 0.147.0. + private const string StreamedOverflowSentence = "ran out of room in the model's context window"; + + /// The JSON-RPC error data Codex writes to stderr when it refuses an input past its own character cap, before any request (observed from 0.142.2 and 0.147.0). + private const string InputCapRefusal = "\"input_error_code\":\"input_too_large\""; + private readonly AgentResultFold _fold = new(); + private JsonElement? _lastErrorLine; + + public void Add(AgentEvent normalized) + { + _fold.Add(normalized); - public void Add(AgentEvent normalized) => _fold.Add(normalized); + if (normalized.Kind == AgentEventKind.Error) _lastErrorLine = normalized.Data; + } public AgentRunResult BuildResult(AgentRunFacts facts, int exitCode, string diagnostics) { @@ -48,8 +62,42 @@ public AgentRunResult BuildResult(AgentRunFacts facts, int exitCode, string diag ?? (string.IsNullOrWhiteSpace(summary) ? null : summary) ?? AgentDiagnosticExcerpt.Explain($"codex exited with code {SandboxExitCode.Describe(exitCode)}", diagnostics); - var exitReason = exitCode != 0 ? "non-zero-exit" : "harness-reported-failure"; + var exitReason = _fold.ReportedFailure && RefusedAsOverContextWindow(_lastErrorLine) ? AgentTerminalOutcomeReader.ContextWindowExceededExitReason + : diagnostics.Contains(InputCapRefusal, StringComparison.Ordinal) ? FailureCodes.SandboxArgumentTooLong + : exitCode != 0 ? "non-zero-exit" : "harness-reported-failure"; return new AgentRunResult { Status = AgentRunStatus.Failed, ExitReason = exitReason, Summary = summary, ChangedFiles = changedFiles, Error = error, TokenUsage = usage, SessionId = sessionId, Model = model }; } + + /// + /// Whether Codex's own terminal turn.failed says the model refused the request as larger than its window: + /// the provider's typed error code it relays, or its own sentence for the streamed refusal. Only that event — the + /// turn's verdict, which the agent cannot author — is read, so an agent message about context windows never is. + /// + private static bool RefusedAsOverContextWindow(JsonElement? line) + { + if (line is not { ValueKind: JsonValueKind.Object } failed || !failed.TryGetProperty("type", out var type) || type.ValueKind != JsonValueKind.String || type.GetString() != "turn.failed") return false; + + if (!failed.TryGetProperty("error", out var error) || error.ValueKind != JsonValueKind.Object || !error.TryGetProperty("message", out var message) || message.ValueKind != JsonValueKind.String) return false; + + var text = message.GetString() ?? ""; + + return ProviderErrorCode(text) == "context_length_exceeded" || text.Contains(StreamedOverflowSentence, StringComparison.Ordinal); + } + + /// The error.code of a provider error body Codex relays as its message verbatim, or null when the message is not one. + private static string? ProviderErrorCode(string message) + { + try + { + using var body = JsonDocument.Parse(message); + + return body.RootElement.ValueKind == JsonValueKind.Object && body.RootElement.TryGetProperty("error", out var error) && error.ValueKind == JsonValueKind.Object + && error.TryGetProperty("code", out var code) && code.ValueKind == JsonValueKind.String ? code.GetString() : null; + } + catch (JsonException) + { + return null; + } + } } diff --git a/backend/src/CodeSpace.Core/Services/Supervisor/AgentRetryCauses.cs b/backend/src/CodeSpace.Core/Services/Supervisor/AgentRetryCauses.cs index 8a548fd5e..1c1dd899b 100644 --- a/backend/src/CodeSpace.Core/Services/Supervisor/AgentRetryCauses.cs +++ b/backend/src/CodeSpace.Core/Services/Supervisor/AgentRetryCauses.cs @@ -32,40 +32,34 @@ public static class AgentRetryCauses public const string ModelAccessLost = "model-access-lost"; /// - /// The model refused the request as larger than its context window — deterministic on replay. A retry warm-resumes - /// the conversation, so its request carries the goal AGAIN and is longer still; a fresh one carries the same goal. - /// Either way the same refusal comes back, billed, and buries the one fact the author needs: the goal is too big - /// for this model. No mitigation, like — its only consumer that matters is - /// AgentCodeNode, which stops respawning it. + /// The model refused the request as larger than its context window + /// () — deterministic on replay. A retry + /// warm-resumes the conversation, so its request carries the goal AGAIN and is longer still; a fresh one carries + /// the same goal. Either way the same refusal comes back, billed, and buries the one fact the author needs: the + /// agent is being given more than this model can read. No mitigation, like — its + /// only consumer that matters is AgentCodeNode, which stops respawning it. + /// + /// Typed by the harness that folded the run, from a field its CLI wrote, and read here off the exit reason + /// only. Never from text: a fail-closed acceptance verdict overwrites the error with the rubric's own words, and a + /// review whose rubric is ABOUT context windows would otherwise classify — which switches the escalation trigger + /// off for exactly the failed grade it exists to act on. /// public const string ContextWindowExceeded = "context-window-exceeded"; /// Seen live 2026-08-30 (run wedge postmortem): the gateway's Anthropic-compat layer broke thinking-block continuation and killed the agent tail with exactly this text. private static readonly string[] FormatFaultMarkers = { "is not a thinking block" }; - /// - /// What the CLIs themselves print for an over-long prompt — each observed from the real binary (Claude Code - /// 2.1.226, Codex 0.147.0 and 0.142.2) answered with the provider's own error body, and pinned through the real - /// harness folds by AgentContextWindowRetryTests. Provider- and CLI-authored phrases, not words an agent's - /// own prose is likely to end on; the vocabulary stays closed like the one above. - /// - private static readonly string[] ContextWindowMarkers = - { - "Prompt is too long", // Claude Code, for either Anthropic overflow body (terminal_reason=prompt_too_long) - "maximum context length is", // OpenAI-compatible gateways (vLLM, LiteLLM), passed through verbatim - "context_length_exceeded", // OpenAI error code, which Codex passes through - "exceeds the context window", // OpenAI's message for the same code - "out of room in the model's context window", // Codex's rewording of a streaming response.failed - "input_too_large", // Codex refusing an input past its own 1,048,576-character cap - }; - /// /// The prior attempt's retry-relevant cause, reading its DECLARED exit reason before any text. A typed code is this /// codebase's own diagnosis and can never be prose about one, so it settles the question outright; only an attempt /// that declared nothing falls through to the marker scan. /// - public static string? Classify(string? exitReason, string? error) => - exitReason == Messages.Failures.FailureCodes.ModelCredentialLeaseLost ? ModelAccessLost : Classify(error); + public static string? Classify(string? exitReason, string? error) => exitReason switch + { + Messages.Failures.FailureCodes.ModelCredentialLeaseLost => ModelAccessLost, + Agents.AgentTerminalOutcomeReader.ContextWindowExceededExitReason => ContextWindowExceeded, + _ => Classify(error), + }; /// The prior attempt's retry-relevant cause read from its error TEXT alone, or null for every ordinary failure (default resume semantics stand unchanged). Prefer the overload above wherever the exit reason is in hand. public static string? Classify(string? error) @@ -75,9 +69,6 @@ public static class AgentRetryCauses foreach (var marker in FormatFaultMarkers) if (error.Contains(marker, StringComparison.OrdinalIgnoreCase)) return GatewayFormatFault; - foreach (var marker in ContextWindowMarkers) - if (error.Contains(marker, StringComparison.OrdinalIgnoreCase)) return ContextWindowExceeded; - return null; } diff --git a/backend/src/CodeSpace.Core/Services/Workflows/Nodes/Builtin/AgentCodeNode.cs b/backend/src/CodeSpace.Core/Services/Workflows/Nodes/Builtin/AgentCodeNode.cs index 734dddb54..42ca7882e 100644 --- a/backend/src/CodeSpace.Core/Services/Workflows/Nodes/Builtin/AgentCodeNode.cs +++ b/backend/src/CodeSpace.Core/Services/Workflows/Nodes/Builtin/AgentCodeNode.cs @@ -373,13 +373,15 @@ private static NodeResult MapResult(JsonElement payload, decimal? maxCostUsd) // that ALREADY ran mitigated (`thinkingDisabled`, projected from its own dispatched envelope) and died // of the SAME fault has proven the repair does not hold here, so a second identical respawn would only // re-bill a broken gateway and bury the one fact the operator needs. - var cause = Supervisor.AgentRetryCauses.Classify(error); + var cause = Supervisor.AgentRetryCauses.Classify(exitReason, error); var formatFault = cause == Supervisor.AgentRetryCauses.GatewayFormatFault; var mitigationSpent = formatFault && ReadFlag(payload, "thinkingDisabled"); - // The model refused the request as larger than its context window. A respawn warm-resumes, so it re-sends - // the goal inside a LONGER request and is refused again, harder — every attempt after the first is an - // identical, billed refusal. Deterministic unless a stronger model is on offer, which may have a larger window. + // The model refused the request as larger than its context window — typed by the harness from its CLI's own + // fields, never read from the error text. A respawn warm-resumes, so it re-sends the goal inside a LONGER + // request, and a fresh one sends the same goal: either way it sends at least as much and is refused the + // same way. The escalation escape below never opens for it: the trigger proposes a stronger model only on a + // failed grade, and stands down for any classified cause, so an overflowing attempt carries no proposal. var contextWindowExceeded = cause == Supervisor.AgentRetryCauses.ContextWindowExceeded; var deterministic = ((status is nameof(AgentRunStatus.NeedsReview) or nameof(AgentRunStatus.Cancelled) || acceptanceFailed || resourceExhausted || argumentTooLong || contextWindowExceeded) @@ -418,7 +420,7 @@ private static NodeResult MapResult(JsonElement payload, decimal? maxCostUsd) { Supervisor.AgentRetryCauses.GatewayFormatFault when mitigationSpent => $" ({Supervisor.AgentRetryCauses.GatewayFormatFault}: a fresh conversation with extended thinking disabled hit the same fault)", Supervisor.AgentRetryCauses.GatewayFormatFault => $" ({Supervisor.AgentRetryCauses.GatewayFormatFault})", - Supervisor.AgentRetryCauses.ContextWindowExceeded => $" ({Supervisor.AgentRetryCauses.ContextWindowExceeded}: the goal is larger than this model's context window, so every attempt is refused the same way — shorten the goal, or choose a model with a larger window)", + Supervisor.AgentRetryCauses.ContextWindowExceeded => $" ({Supervisor.AgentRetryCauses.ContextWindowExceeded}: the model refused the request as larger than its context window, and a respawn would send at least as much — give the agent less, or choose a model with a larger window)", _ => "", }; diff --git a/backend/tests/CodeSpace.UnitTests/Agents/AgentContextWindowRetryTests.cs b/backend/tests/CodeSpace.UnitTests/Agents/AgentContextWindowRetryTests.cs index 98b913869..b6368eac0 100644 --- a/backend/tests/CodeSpace.UnitTests/Agents/AgentContextWindowRetryTests.cs +++ b/backend/tests/CodeSpace.UnitTests/Agents/AgentContextWindowRetryTests.cs @@ -4,6 +4,7 @@ using CodeSpace.Core.Services.Supervisor; using CodeSpace.Messages.Agents; using CodeSpace.Messages.Enums; +using CodeSpace.Messages.Failures; using Shouldly; namespace CodeSpace.UnitTests.Agents; @@ -13,9 +14,9 @@ namespace CodeSpace.UnitTests.Agents; /// /// Every shape below is a terminal line a real CLI printed (Claude Code 2.1.226, Codex 0.147.0) when a local /// endpoint answered its request with the provider's own over-long-prompt error body, trimmed to the fields the -/// fold reads. They are run through ParseEvents and BuildResult — the reader production uses — and only -/// then classified, so the markers are pinned against the text a failed run really carries, not against strings -/// written to match them. +/// fold reads. They are run through ParseEvents and BuildResult — the reader production uses — and the +/// cause is read off the exit reason the fold typed from the CLI's own fields, never off the error text: a text scan +/// also matched a rubric that talks about context windows, and switched model escalation off for its failed grade. /// /// Why it matters: an overflow used to be an ordinary retryable failure. A retry warm-resumes, so the next /// request carries the goal twice and overflows harder; a task-launched agent node spends all three default @@ -51,7 +52,8 @@ public void A_claude_context_overflow_folds_to_an_error_classified_as_one(string var result = harness.BuildResult(harness.ParseEvents(terminalLine), exitCode: 1, diagnostics: ""); result.Status.ShouldBe(AgentRunStatus.Failed); - AgentRetryCauses.Classify(result.Error).ShouldBe(AgentRetryCauses.ContextWindowExceeded, customMessage: $"the folded error was: {result.Error}"); + result.ExitReason.ShouldBe(AgentTerminalOutcomeReader.ContextWindowExceededExitReason, customMessage: "the CLI's own terminal_reason / status is the evidence — typed on the exit reason, never re-read from prose"); + AgentRetryCauses.Classify(result.ExitReason, result.Error).ShouldBe(AgentRetryCauses.ContextWindowExceeded, customMessage: $"the folded error was: {result.Error}"); } [Theory] @@ -63,19 +65,61 @@ public void A_codex_context_overflow_folds_to_an_error_classified_as_one(string var result = harness.BuildResult(harness.ParseEvents(terminalLine), exitCode: 1, diagnostics: ""); result.Status.ShouldBe(AgentRunStatus.Failed); - AgentRetryCauses.Classify(result.Error).ShouldBe(AgentRetryCauses.ContextWindowExceeded, customMessage: $"the folded error was: {result.Error}"); + result.ExitReason.ShouldBe(AgentTerminalOutcomeReader.ContextWindowExceededExitReason); + AgentRetryCauses.Classify(result.ExitReason, result.Error).ShouldBe(AgentRetryCauses.ContextWindowExceeded, customMessage: $"the folded error was: {result.Error}"); } [Fact] - public void Codex_refusing_an_input_past_its_own_character_cap_is_classified_the_same_way() + public void Codex_refusing_an_input_past_its_own_character_cap_is_that_refusal_not_a_context_overflow() { - // Codex refuses before any request, on stderr only — so the only carrier is the diagnostics excerpt the fold - // appends to a bare exit. Pinned against codex 0.142.2 (the worker's version) as well as 0.147.0. + // Codex refuses before any request, on stderr only. It is the CLI's own character cap — the condition the + // harness preflight refuses up front — not the model's window, so "choose a larger-window model" would be + // advice that cannot help. It folds to the same terminal code as the preflight. const string stderr = """Error: turn/start: turn/start failed: Input exceeds the maximum length of 1048576 characters. (code -32602), data: {"input_error_code":"input_too_large","max_chars":1048576,"actual_chars":1100000}"""; var result = new CodexHarness().BuildResult(Array.Empty(), exitCode: 1, diagnostics: stderr); - AgentRetryCauses.Classify(result.Error).ShouldBe(AgentRetryCauses.ContextWindowExceeded, customMessage: $"the folded error was: {result.Error}"); + result.ExitReason.ShouldBe(FailureCodes.SandboxArgumentTooLong); + result.Error.ShouldContain("Input exceeds the maximum length of 1048576 characters", customMessage: "the CLI's own sentence is the cause the author reads"); + } + + [Fact] + public void A_rubric_that_talks_about_context_windows_does_not_switch_escalation_off() + { + // The regression a text scan made: a fail-closed acceptance verdict overwrites Error with the rubric's own + // requirement and the judge's evidence. A review whose rubric is ABOUT context windows then read as an + // overflow, and the escalation trigger — which stands down for any classified cause — stopped offering the + // stronger model the failed grade was evidence for. A cause is typed by the harness now; prose never is one. + const string graderProse = "The acceptance check did not pass: requirement 'returns context_length_exceeded when the input exceeds the context window' — evidence: the handler throws instead."; + + AgentRetryCauses.Classify(AgentAcceptanceContract.FailClosedExitReason, graderProse).ShouldBeNull(); + AgentModelEscalationTrigger.Reason(AgentContradiction.OverClaim, acceptanceFailed: true, acceptanceDetail: "requirement not met", workPresent: true, error: graderProse, exitReason: AgentAcceptanceContract.FailClosedExitReason) + .ShouldNotBeNull(customMessage: "a failed grade on a claimed success is escalation evidence, whatever words the rubric uses"); + } + + [Fact] + public void A_crashed_run_whose_last_words_mention_an_overflow_is_not_one() + { + // No result line: the harness reports the agent's own last message as the error. That is the agent's prose. + var harness = new ClaudeCodeHarness(); + var lastWords = """{"type":"assistant","message":{"content":[{"type":"text","text":"The docs say a request fails with 'Prompt is too long' past the context window; let me check."}]}}"""; + + var result = harness.BuildResult(harness.ParseEvents(lastWords), exitCode: 137, diagnostics: ""); + + result.ExitReason.ShouldNotBe(AgentTerminalOutcomeReader.ContextWindowExceededExitReason); + AgentRetryCauses.Classify(result.ExitReason, result.Error).ShouldBeNull(customMessage: "a crash stays a retryable crash"); + } + + [Fact] + public void A_transient_gateway_error_whose_body_mentions_context_length_is_not_an_overflow() + { + // Only the model REFUSING the request (a 4xx) is an overflow; a 5xx is the gateway, and it is worth a retry. + var harness = new ClaudeCodeHarness(); + var line = """{"type":"result","subtype":"success","is_error":true,"num_turns":1,"terminal_reason":"api_error","api_error_status":503,"session_id":"s","result":"API Error: 503 upstream timed out after prefilling; maximum context length is 131072 tokens"}"""; + + var result = harness.BuildResult(harness.ParseEvents(line), exitCode: 1, diagnostics: ""); + + result.ExitReason.ShouldNotBe(AgentTerminalOutcomeReader.ContextWindowExceededExitReason); } [Theory] @@ -83,9 +127,20 @@ public void Codex_refusing_an_input_past_its_own_character_cap_is_classified_the [InlineData("patch did not apply")] [InlineData("API Error: 529 Overloaded")] [InlineData("codex exited with code 1 — stderr: npm WARN deprecated glob@7")] - public void An_ordinary_failure_is_not_mistaken_for_an_overflow(string error) + [InlineData("Prompt is too long")] + [InlineData("the handler must return context_length_exceeded when the input exceeds the context window")] + public void Text_alone_is_never_an_overflow(string error) + { + AgentRetryCauses.Classify(error).ShouldBeNull(customMessage: "only a harness-typed exit reason is an overflow; words are not"); + AgentRetryCauses.Classify("non-zero-exit", error).ShouldBeNull(); + } + + [Fact] + public void The_overflow_exit_reason_is_pinned() { - AgentRetryCauses.Classify(error).ShouldBeNull(customMessage: "an ordinary death keeps the default resume-and-retry semantics"); + // Both harness folders stamp it and the node's retry verdict keys on it; a rename that missed either side + // would silently turn an overflow back into three identical, billed refusals. + AgentTerminalOutcomeReader.ContextWindowExceededExitReason.ShouldBe("context-window-exceeded"); } [Fact] diff --git a/backend/tests/CodeSpace.UnitTests/Workflows/AgentCodeNodeTests.cs b/backend/tests/CodeSpace.UnitTests/Workflows/AgentCodeNodeTests.cs index 32b37992d..fd01fb7df 100644 --- a/backend/tests/CodeSpace.UnitTests/Workflows/AgentCodeNodeTests.cs +++ b/backend/tests/CodeSpace.UnitTests/Workflows/AgentCodeNodeTests.cs @@ -555,7 +555,7 @@ public async Task A_context_window_overflow_is_not_respawned(string error) { // A respawn warm-resumes the conversation, so it re-sends the goal inside a LONGER request — it overflows again, // harder. Every attempt after the first is an identical, billed refusal that buries the one fact the author needs. - var resume = JsonDocument.Parse(JsonSerializer.Serialize(new { status = "Failed", error, exitReason = "non-zero-exit" })).RootElement; + var resume = JsonDocument.Parse(JsonSerializer.Serialize(new { status = "Failed", error, exitReason = AgentTerminalOutcomeReader.ContextWindowExceededExitReason })).RootElement; var result = await new AgentCodeNode().RunAsync(BuildContext(new(), resume), CancellationToken.None); From 6b917b0a4509dcd3ef4001f098bd3fd3f4c73eec Mon Sep 17 00:00:00 2001 From: "Mars.P" Date: Thu, 24 Sep 2026 06:49:41 +0800 Subject: [PATCH 05/16] Keep a failed agent's own cause on a cost-capped node On a node with maxCostUsd, any attempt without a cumulative spend failed as "cannot be priced ... cumulative spend is missing", whatever killed it. A launch refused for its size never starts a process, so it never has a spend to read, and that sentence replaced the one telling the author to shorten the goal. Only a succeeded run still fails closed on its price, since its output would otherwise escape the cap unaccounted. A failed one keeps its own message and is never retried - which the retry's own prior-spend check would decide one attempt later anyway - and says so only when that is what stopped a failure a respawn could otherwise change. --- .../Workflows/Nodes/Builtin/AgentCodeNode.cs | 28 ++++++++++++---- .../Workflows/AgentCodeNodeTests.cs | 32 +++++++++++++++++++ 2 files changed, 54 insertions(+), 6 deletions(-) diff --git a/backend/src/CodeSpace.Core/Services/Workflows/Nodes/Builtin/AgentCodeNode.cs b/backend/src/CodeSpace.Core/Services/Workflows/Nodes/Builtin/AgentCodeNode.cs index 42ca7882e..98d84c83c 100644 --- a/backend/src/CodeSpace.Core/Services/Workflows/Nodes/Builtin/AgentCodeNode.cs +++ b/backend/src/CodeSpace.Core/Services/Workflows/Nodes/Builtin/AgentCodeNode.cs @@ -299,20 +299,32 @@ private static bool TryReadAcceptance(IReadOnlyDictionary c private static NodeResult MapResult(JsonElement payload, decimal? maxCostUsd) { var status = ReadString(payload, "status"); + var succeeded = status == nameof(AgentRunStatus.Succeeded); + var unpriced = false; if (maxCostUsd is { } cap) { - if (ReadFlag(payload, "costIndeterminate")) + var indeterminate = ReadFlag(payload, "costIndeterminate"); + var spendMissing = !TryReadNonNegativeDecimal(payload, "cumulativeCostUsd", out var cumulative) || cumulative is null; + + // A SUCCEEDED run that cannot be priced fails closed on its price: its output would otherwise escape the + // cap unaccounted. A FAILED one has already failed, and its own cause is the sentence the author can act + // on — a launch refused for its size started no process, so it never had a spend to read, and the + // missing price used to stand in for "shorten the goal". It keeps its cause and is only never retried + // (below), which is what the retry's own prior-spend check would decide one attempt later anyway. + if (indeterminate && succeeded) return NodeResult.Fail($"Agent run cannot be priced under the monitored ${cap.ToString(CultureInfo.InvariantCulture)} cost cap.", retryable: false); - if (!TryReadNonNegativeDecimal(payload, "cumulativeCostUsd", out var cumulative) || cumulative is null) + if (spendMissing && succeeded) return NodeResult.Fail($"Agent run cannot be priced under the monitored ${cap.ToString(CultureInfo.InvariantCulture)} cost cap because cumulative spend is missing.", retryable: false); - if (cumulative > cap || (cumulative == cap && status != nameof(AgentRunStatus.Succeeded))) - return NodeResult.Fail($"Agent run stopped: {Supervisor.SupervisorStopReasons.CostCapReached} (${cumulative.Value.ToString(CultureInfo.InvariantCulture)} of ${cap.ToString(CultureInfo.InvariantCulture)} observed).", retryable: false); + unpriced = indeterminate || spendMissing; + + if (!unpriced && (cumulative > cap || (cumulative == cap && !succeeded))) + return NodeResult.Fail($"Agent run stopped: {Supervisor.SupervisorStopReasons.CostCapReached} (${cumulative!.Value.ToString(CultureInfo.InvariantCulture)} of ${cap.ToString(CultureInfo.InvariantCulture)} observed).", retryable: false); } - if (status != nameof(AgentRunStatus.Succeeded)) + if (!succeeded) { var error = ReadString(payload, "error"); @@ -390,7 +402,7 @@ private static NodeResult MapResult(JsonElement payload, decimal? maxCostUsd) && !escalationAvailable) || mitigationSpent; - return NodeResult.Fail($"Agent run did not succeed: {(string.IsNullOrEmpty(error) ? status : error)}{FailureCauseSuffix(cause, mitigationSpent)}", retryable: !deterministic); + return NodeResult.Fail($"Agent run did not succeed: {(string.IsNullOrEmpty(error) ? status : error)}{FailureCauseSuffix(cause, mitigationSpent)}{UnpricedRetrySuffix(unpriced && !deterministic, maxCostUsd)}", retryable: !deterministic && !unpriced); } var outputs = new Dictionary { ["status"] = JsonSerializer.SerializeToElement(nameof(AgentRunStatus.Succeeded)) }; @@ -424,6 +436,10 @@ private static NodeResult MapResult(JsonElement payload, decimal? maxCostUsd) _ => "", }; + /// Why a failure a respawn could otherwise change is not respawned: its spend is unknown, so a retry cannot be bought under the cap. Empty whenever that is not the deciding fact, so the cause stays the whole message. + private static string UnpricedRetrySuffix(bool decides, decimal? maxCostUsd) => + decides ? $"; not retried: its spend cannot be priced under the monitored ${maxCostUsd!.Value.ToString(CultureInfo.InvariantCulture)} cost cap" : ""; + /// /// P2.3: stamp the retry-resume hint from the RETIRING prior attempt's own resume payload (the same /// sessionId/transcript triple RealSupervisorActionExecutor.ApplyRetryResumeHintAsync reads from a DB diff --git a/backend/tests/CodeSpace.UnitTests/Workflows/AgentCodeNodeTests.cs b/backend/tests/CodeSpace.UnitTests/Workflows/AgentCodeNodeTests.cs index fd01fb7df..1c97e5546 100644 --- a/backend/tests/CodeSpace.UnitTests/Workflows/AgentCodeNodeTests.cs +++ b/backend/tests/CodeSpace.UnitTests/Workflows/AgentCodeNodeTests.cs @@ -564,6 +564,38 @@ public async Task A_context_window_overflow_is_not_respawned(string error) result.Error.ShouldContain("context window", Case.Insensitive, customMessage: "the node's own message must name the cause, whatever advice the CLI's text gives"); } + [Fact] + public async Task A_launch_refused_for_its_size_on_a_cost_capped_node_says_why_instead_of_cannot_be_priced() + { + // Refused before any CLI started, so nothing was spent and there is nothing to price. The cost cap's + // "cannot be priced — cumulative spend is missing" used to replace the one sentence that says what to shorten. + var config = new Dictionary { ["maxCostUsd"] = Num(5) }; + var resume = JsonDocument.Parse(JsonSerializer.Serialize(new { status = "Failed", error = "the agent's goal is 1100000 characters; codex accepts at most 1048576", exitReason = FailureCodes.SandboxArgumentTooLong })).RootElement; + + var result = await new AgentCodeNode().RunAsync(BuildContext(config, resume), CancellationToken.None); + + result.Status.ShouldBe(NodeStatus.Failure); + result.Retryable.ShouldBeFalse(); + result.Error.ShouldContain("codex accepts at most 1048576"); + result.Error.ShouldNotContain("cannot be priced", Case.Insensitive); + } + + [Fact] + public async Task An_unpriced_failure_on_a_cost_capped_node_keeps_its_cause_and_is_not_retried() + { + // A crash is ordinarily worth a respawn, but under a cap an unpriced attempt cannot buy one. The node says so + // AFTER the cause, rather than instead of it. + var config = new Dictionary { ["maxCostUsd"] = Num(5) }; + var resume = JsonDocument.Parse(JsonSerializer.Serialize(new { status = "Failed", error = "claude exited with code 1", exitReason = "non-zero-exit" })).RootElement; + + var result = await new AgentCodeNode().RunAsync(BuildContext(config, resume), CancellationToken.None); + + result.Status.ShouldBe(NodeStatus.Failure); + result.Retryable.ShouldBeFalse(); + result.Error.ShouldStartWith("Agent run did not succeed: claude exited with code 1"); + result.Error.ShouldEndWith("; not retried: its spend cannot be priced under the monitored $5 cost cap"); + } + [Fact] public void The_two_watchdogs_are_classified_alike_because_neither_can_see_why_the_process_went_quiet() { From ad2f4ab133d68942107e6bf942c88ee5b0c2fbe6 Mon Sep 17 00:00:00 2001 From: "Mars.P" Date: Thu, 24 Sep 2026 06:49:42 +0800 Subject: [PATCH 06/16] Read a 401 in a Room failure only as a status of its own The auth heuristic matched "401" as a substring, so a size refusal reporting a 1401234-character goal rendered as "Authentication failed" with a Fix credentials action, sending the author to rotate a working key. The digits now have to stand alone. --- .../Services/Sessions/Room/RoomNarrative.cs | 8 +++++- .../Sessions/Room/RoomNarrativeTests.cs | 28 +++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/backend/src/CodeSpace.Core/Services/Sessions/Room/RoomNarrative.cs b/backend/src/CodeSpace.Core/Services/Sessions/Room/RoomNarrative.cs index a94c9dbee..9c20085c7 100644 --- a/backend/src/CodeSpace.Core/Services/Sessions/Room/RoomNarrative.cs +++ b/backend/src/CodeSpace.Core/Services/Sessions/Room/RoomNarrative.cs @@ -1,4 +1,5 @@ using System.Globalization; +using System.Text.RegularExpressions; using CodeSpace.Core.Services.Agents; using CodeSpace.Core.Services.Tasks.Phases.Sources.Nodes; using CodeSpace.Core.Services.Tasks.Phases.Sources.Supervisor; @@ -853,10 +854,15 @@ private static bool IsAuthError(string? error) if (error.Contains(Agents.Credentials.ModelCredentialLeaseLostException.Explanation, StringComparison.Ordinal)) return false; - return new[] { "401", "unauthorized", "authentication", "api key", "api-key", "credential", "invalid_api_key" } + if (UnauthorizedStatus.IsMatch(error)) return true; + + return new[] { "unauthorized", "authentication", "api key", "api-key", "credential", "invalid_api_key" } .Any(m => error.Contains(m, StringComparison.OrdinalIgnoreCase)); } + /// An HTTP 401 as a status on its own — never the digits inside a longer number, such as the length a size refusal reports (a 1401234-character goal is not a rejected key). + private static readonly Regex UnauthorizedStatus = new(@"(? narrativePhases, string? error) { if (status == WorkflowRunStatus.Cancelled) return "This turn was cancelled."; diff --git a/backend/tests/CodeSpace.UnitTests/Sessions/Room/RoomNarrativeTests.cs b/backend/tests/CodeSpace.UnitTests/Sessions/Room/RoomNarrativeTests.cs index 9e0a44179..bbea39a07 100644 --- a/backend/tests/CodeSpace.UnitTests/Sessions/Room/RoomNarrativeTests.cs +++ b/backend/tests/CodeSpace.UnitTests/Sessions/Room/RoomNarrativeTests.cs @@ -1,4 +1,6 @@ using CodeSpace.Core.Services.Agents.Credentials; +using CodeSpace.Core.Services.Agents.Harnesses.Codex; +using CodeSpace.Core.Services.Agents.Sandbox.Exceptions; using CodeSpace.Core.Services.Sessions.Journal.FactsSources; using CodeSpace.Core.Services.Sessions.Room; using CodeSpace.Core.Services.Tasks.Phases; @@ -236,6 +238,32 @@ public void A_lost_model_credential_lease_is_NOT_dressed_up_as_a_rejected_creden diag.Text.ShouldContain("Retry", Case.Insensitive, "and what to do about it"); } + [Fact] + public void A_size_refusal_whose_count_contains_401_is_not_dressed_up_as_a_rejected_credential() + { + // The refusal names how long the goal is, and a length such as 1401234 carries "401" inside it. Read as a + // substring, that sent the author to rotate a working key for a goal that was simply too long. + var refusal = Should.Throw(() => new CodexHarness().BuildInvocation(new AgentTask { Goal = new string('x', 1_401_234), Harness = CodexHarness.HarnessKind, WorkspaceDirectory = "/tmp/ws" })); + var facts = new RoomTurnFacts { RawError = $"Agent run did not succeed: {refusal.Message}" }; + + var diag = Build(Array.Empty(), WorkflowRunStatus.Failure, facts: facts).Blocks.OfType().ShouldHaveSingleItem(); + + refusal.Message.ShouldContain("1401234", customMessage: "fixture check: the producer's own text carries the digits"); + diag.Title.ShouldNotBe("Authentication failed"); + diag.Actions.ShouldNotContain(a => a.Kind == RoomActionKind.FixCredentials); + } + + [Theory] + [InlineData("OpenAI API error: 401 Unauthorized")] + [InlineData("API Error: 401 {\"type\":\"error\"}")] + [InlineData("request failed with status code 401")] + public void A_401_status_still_reads_as_a_rejected_credential(string error) + { + var diag = Build(Array.Empty(), WorkflowRunStatus.Failure, facts: new RoomTurnFacts { RawError = error }).Blocks.OfType().ShouldHaveSingleItem(); + + diag.Title.ShouldBe("Authentication failed"); + } + [Fact] public void A_non_auth_failure_humanizes_the_engine_error() { From 381ff2b14a136bd73540792b6bb74f05f8dc5f66 Mon Sep 17 00:00:00 2001 From: "Mars.P" Date: Thu, 24 Sep 2026 06:49:42 +0800 Subject: [PATCH 07/16] Say what interpolation escaping did and did not protect The doc claimed the HTML-safe escaping protected nothing. In a URL query, a header or a single-quoted sh -c string it did keep an object's & or ' from reading as syntax - protection a string value there never had, so no template could rely on it. State that instead. --- .../Services/Workflows/WorkflowJson.cs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/backend/src/CodeSpace.Core/Services/Workflows/WorkflowJson.cs b/backend/src/CodeSpace.Core/Services/Workflows/WorkflowJson.cs index 0731121ed..265e64f19 100644 --- a/backend/src/CodeSpace.Core/Services/Workflows/WorkflowJson.cs +++ b/backend/src/CodeSpace.Core/Services/Workflows/WorkflowJson.cs @@ -20,10 +20,14 @@ public static class WorkflowJson /// template, and by any projection that must stay byte-identical to one (MapResultsPrompt). Characters are /// left as themselves: the HTML-safe default turned every CJK character and every + < > & ' into a /// six-character \uXXXX, so a pull-request diff bound into a review prompt reached the model as escape - /// codes at up to twice its size — while the string branch beside it already inserted all of those raw, so the - /// escaping protected nothing. What JSON itself requires is still escaped (the quote, the backslash, control - /// characters), so a value embedded in a JSON body stays parseable. One exception no built-in encoder lifts: a - /// character outside the Basic Multilingual Plane (an emoji) is still written as its surrogate-pair escape. + /// codes at up to twice its size. An object now behaves like the string branch beside it, which has always inserted + /// those characters raw. That is not a new exposure, but it is not "the escaping protected nothing" either: in a + /// URL query, a header or a single-quoted sh -c string the escape codes did keep an object's & or + /// ' from reading as syntax — protection a string value there never had, so no template could rely on it, + /// and a value bound into such a context needs that context's own encoding. What JSON itself requires is still + /// escaped (the quote, the backslash, control characters), so a value embedded in a JSON body stays parseable. + /// One exception no built-in encoder lifts: a character outside the Basic Multilingual Plane (an emoji) is still + /// written as its surrogate-pair escape. /// public static JsonSerializerOptions InterpolatedText { get; } = new() { Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping }; From ec25d03c996b0140cd1ebc348be24ce79232b713 Mon Sep 17 00:00:00 2001 From: "Mars.P" Date: Thu, 24 Sep 2026 11:42:56 +0800 Subject: [PATCH 08/16] Keep a Claude connection failure's own cause in the fold The overflow predicate read api_error_status with TryGetInt32, which throws on a non-number. Claude writes "api_error_status": null on every connection-level failure - ConnectionRefused, ECONNRESET, a dead gateway, a broker listener that went away - so BuildResult threw on exactly those runs. The run landed as executor-error with a .NET message, and its diff, transcript, usage and session id were never captured; the real-model gates read that code as a code fault. Guard the kind first. Pinned with the pinned 2.1.263's own lines. --- .../Claude/ClaudeCodeResultFolder.cs | 2 +- .../Agents/AgentContextWindowRetryTests.cs | 23 +++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/backend/src/CodeSpace.Core/Services/Agents/Harnesses/Claude/ClaudeCodeResultFolder.cs b/backend/src/CodeSpace.Core/Services/Agents/Harnesses/Claude/ClaudeCodeResultFolder.cs index e9cf8f525..ff5f3d205 100644 --- a/backend/src/CodeSpace.Core/Services/Agents/Harnesses/Claude/ClaudeCodeResultFolder.cs +++ b/backend/src/CodeSpace.Core/Services/Agents/Harnesses/Claude/ClaudeCodeResultFolder.cs @@ -87,7 +87,7 @@ private static bool RefusedAsOverContextWindow(JsonElement? line) if (terminalReason == "prompt_too_long") return true; - if (terminalReason != "api_error" || !result.TryGetProperty("api_error_status", out var status) || !status.TryGetInt32(out var code) || code != 400) return false; + if (terminalReason != "api_error" || !result.TryGetProperty("api_error_status", out var status) || status.ValueKind != JsonValueKind.Number || !status.TryGetInt32(out var code) || code != 400) return false; var body = ReadString(result, "result"); diff --git a/backend/tests/CodeSpace.UnitTests/Agents/AgentContextWindowRetryTests.cs b/backend/tests/CodeSpace.UnitTests/Agents/AgentContextWindowRetryTests.cs index b6368eac0..02f2efbfd 100644 --- a/backend/tests/CodeSpace.UnitTests/Agents/AgentContextWindowRetryTests.cs +++ b/backend/tests/CodeSpace.UnitTests/Agents/AgentContextWindowRetryTests.cs @@ -35,6 +35,13 @@ public sealed class AgentContextWindowRetryTests """{"type":"result","subtype":"success","is_error":true,"num_turns":1,"terminal_reason":"api_error","api_error_status":400,"session_id":"6406b570-40de-4274-a586-ab044dbf5d48","result":"API Error: 400 This model's maximum context length is 131072 tokens. However, you requested 161234 tokens. Please reduce the length of the messages."}""", }; + public static TheoryData ClaudeConnectionFailureLines => new() + { + // Pinned 2.1.263 against a dead port and a peer that resets: the status is null because no answer ever came. + { """{"type":"result","subtype":"success","is_error":true,"num_turns":1,"terminal_reason":"api_error","api_error_status":null,"session_id":"54818d4e-66e4-4f1b-b3ee-73de94f3b9cc","result":"API Error: Connection refused — a firewall or proxy may be blocking it (ConnectionRefused)"}""", "API Error: Connection refused — a firewall or proxy may be blocking it (ConnectionRefused)" }, + { """{"type":"result","subtype":"success","is_error":true,"num_turns":1,"terminal_reason":"api_error","api_error_status":null,"session_id":"f01a6528-1c83-4400-af5d-4ac6d322c7f9","result":"API Error: Connection dropped (ECONNRESET)"}""", "API Error: Connection dropped (ECONNRESET)" }, + }; + public static TheoryData CodexOverflowLines => new() { // OpenAI Responses API 400 context_length_exceeded. @@ -69,6 +76,22 @@ public void A_codex_context_overflow_folds_to_an_error_classified_as_one(string AgentRetryCauses.Classify(result.ExitReason, result.Error).ShouldBe(AgentRetryCauses.ContextWindowExceeded, customMessage: $"the folded error was: {result.Error}"); } + [Theory] + [MemberData(nameof(ClaudeConnectionFailureLines))] + public void A_claude_connection_failure_folds_to_its_own_cause(string terminalLine, string cliError) + { + // The status is null when no answer came. Reading it as a number threw out of the fold, so the run landed as + // executor-error with a .NET message, and its diff, transcript and session were never captured. + var harness = new ClaudeCodeHarness(); + + var result = harness.BuildResult(harness.ParseEvents(terminalLine), exitCode: 1, diagnostics: ""); + + result.Status.ShouldBe(AgentRunStatus.Failed); + result.ExitReason.ShouldBe("non-zero-exit"); + result.Error.ShouldBe(cliError); + result.SessionId.ShouldNotBeNullOrEmpty(customMessage: "the fold completed, so the session a retry resumes is still there"); + } + [Fact] public void Codex_refusing_an_input_past_its_own_character_cap_is_that_refusal_not_a_context_overflow() { From ef1999ce5cbce2509a024b1de3480457f8b7ea34 Mon Sep 17 00:00:00 2001 From: "Mars.P" Date: Thu, 24 Sep 2026 11:43:08 +0800 Subject: [PATCH 09/16] Type the overflow shapes the folds missed Claude refuses locally, sending nothing, once its own token estimate is past the window: terminal_reason "blocking_limit", result "Prompt is too long" (the pinned 2.1.263 does this from ~760 KB on a 200K model). It folded to non-zero-exit and was respawned with no cause named. Codex relays a vLLM or LiteLLM refusal verbatim, with the code as 400 or "400" and the reason only in the message, so it matched neither OpenAI's typed code nor Codex's own sentence - while the Claude fold typed the same gateway body. The gateway marker list now lives beside the exit reason and both folds read it, only off a relayed provider body. A 5xx stays retryable on both. --- .../Agents/AgentTerminalOutcomeReader.cs | 13 +++++++ .../Claude/ClaudeCodeResultFolder.cs | 34 +++++++++---------- .../Harnesses/Codex/CodexResultFolder.cs | 22 ++++++++---- .../Agents/AgentContextWindowRetryTests.cs | 18 ++++++++++ 4 files changed, 62 insertions(+), 25 deletions(-) diff --git a/backend/src/CodeSpace.Core/Services/Agents/AgentTerminalOutcomeReader.cs b/backend/src/CodeSpace.Core/Services/Agents/AgentTerminalOutcomeReader.cs index 9c55d3b13..8aec846d4 100644 --- a/backend/src/CodeSpace.Core/Services/Agents/AgentTerminalOutcomeReader.cs +++ b/backend/src/CodeSpace.Core/Services/Agents/AgentTerminalOutcomeReader.cs @@ -22,6 +22,19 @@ public static class AgentTerminalOutcomeReader /// public const string ContextWindowExceededExitReason = "context-window-exceeded"; + /// + /// What an OpenAI-compatible gateway (vLLM, LiteLLM) says when it refuses a request as over the model's window — + /// the overflow shape neither CLI stamps as its own, because the gateway speaks in its own words and codes (vLLM + /// sends code: 400, LiteLLM code: "400", with the reason only in the message). Read ONLY off a + /// provider refusal body a CLI relays (Claude's 400 api_error result, Codex's turn.failed) — never + /// off an agent's prose, which is how a crash or a rubric used to pass for a diagnosis. Shared so the two folds + /// cannot disagree about the same gateway. + /// + public static bool NamesAContextOverflow(string providerMessage) => + GatewayOverflowMarkers.Any(marker => providerMessage.Contains(marker, StringComparison.OrdinalIgnoreCase)); + + private static readonly string[] GatewayOverflowMarkers = { "maximum context length is", "context_length_exceeded" }; + /// /// True when the last Completed-or-Error event in the stream is an Error — i.e. the harness itself reported /// the run failed, even if the OS exit code was 0. False when no such event exists (nothing to reconcile diff --git a/backend/src/CodeSpace.Core/Services/Agents/Harnesses/Claude/ClaudeCodeResultFolder.cs b/backend/src/CodeSpace.Core/Services/Agents/Harnesses/Claude/ClaudeCodeResultFolder.cs index ff5f3d205..c4180336e 100644 --- a/backend/src/CodeSpace.Core/Services/Agents/Harnesses/Claude/ClaudeCodeResultFolder.cs +++ b/backend/src/CodeSpace.Core/Services/Agents/Harnesses/Claude/ClaudeCodeResultFolder.cs @@ -13,13 +13,6 @@ namespace CodeSpace.Core.Services.Agents.Harnesses.Claude; /// internal sealed class ClaudeCodeResultFolder : IAgentEventFolder { - /// - /// What an OpenAI-compatible gateway (vLLM, LiteLLM) answers an over-long request with, which the CLI passes - /// through verbatim as terminal_reason: api_error — the one overflow shape it does not stamp as its own. - /// Observed from Claude Code 2.1.226 answered with each body; read only off a 400 refusal on the result line. - /// - private static readonly string[] GatewayOverflowMarkers = { "maximum context length is", "context_length_exceeded" }; - private readonly AgentResultFold _fold = new(); private JsonElement? _lastErrorLine; @@ -73,11 +66,16 @@ public AgentRunResult BuildResult(AgentRunFacts facts, int exitCode, string diag } /// - /// Whether the CLI's own terminal result line says the model refused the request as larger than its context - /// window. Read off fields the CLI writes (terminal_reason, api_error_status) and, for the gateway - /// shape, off the refusal body it relays — never off the agent's prose, which is how a crash's last message or a - /// rubric's wording would otherwise pass for a diagnosis. A 5xx is the gateway failing, not the model refusing, - /// so it stays an ordinary, retryable failure whatever its body says. + /// Whether the CLI's own terminal result line says the request is larger than the model's context window. Read off + /// fields the CLI writes and, for the gateway shape, off the refusal body it relays — never off the agent's prose, + /// which is how a crash's last message or a rubric's wording would otherwise pass for a diagnosis. + /// + /// Two verdicts are the CLI's own: prompt_too_long, the provider refused the request; and + /// blocking_limit, the CLI's own token estimate is past the window so it refused locally and sent nothing + /// (observed from 2.1.226 and the pinned 2.1.263 for a goal past roughly 760 KB on a 200K-token model). Either way a + /// respawn sends at least as much. The third shape is a gateway's 400 relayed as api_error. A 5xx is the + /// gateway failing, not the model refusing, and a connection-level failure carries api_error_status: null + /// — both stay ordinary, retryable failures whatever their text says. /// private static bool RefusedAsOverContextWindow(JsonElement? line) { @@ -85,15 +83,15 @@ private static bool RefusedAsOverContextWindow(JsonElement? line) var terminalReason = ReadString(result, "terminal_reason"); - if (terminalReason == "prompt_too_long") return true; + if (terminalReason is "prompt_too_long" or "blocking_limit") return true; - if (terminalReason != "api_error" || !result.TryGetProperty("api_error_status", out var status) || status.ValueKind != JsonValueKind.Number || !status.TryGetInt32(out var code) || code != 400) return false; - - var body = ReadString(result, "result"); - - return GatewayOverflowMarkers.Any(marker => body.Contains(marker, StringComparison.OrdinalIgnoreCase)); + return terminalReason == "api_error" && IsClientRefusal(result) && AgentTerminalOutcomeReader.NamesAContextOverflow(ReadString(result, "result")); } + /// The result line's api_error_status is a 400. Null (a connection that never got an answer), absent, or any other kind is not. + private static bool IsClientRefusal(JsonElement result) => + result.TryGetProperty("api_error_status", out var status) && status.ValueKind == JsonValueKind.Number && status.TryGetInt32(out var code) && code == 400; + private static string ReadString(JsonElement root, string key) => root.TryGetProperty(key, out var value) && value.ValueKind == JsonValueKind.String ? value.GetString() ?? "" : ""; } diff --git a/backend/src/CodeSpace.Core/Services/Agents/Harnesses/Codex/CodexResultFolder.cs b/backend/src/CodeSpace.Core/Services/Agents/Harnesses/Codex/CodexResultFolder.cs index 32ca067a3..e45a83c93 100644 --- a/backend/src/CodeSpace.Core/Services/Agents/Harnesses/Codex/CodexResultFolder.cs +++ b/backend/src/CodeSpace.Core/Services/Agents/Harnesses/Codex/CodexResultFolder.cs @@ -71,8 +71,10 @@ public AgentRunResult BuildResult(AgentRunFacts facts, int exitCode, string diag /// /// Whether Codex's own terminal turn.failed says the model refused the request as larger than its window: - /// the provider's typed error code it relays, or its own sentence for the streamed refusal. Only that event — the - /// turn's verdict, which the agent cannot author — is read, so an agent message about context windows never is. + /// the provider refusal body Codex relays verbatim — OpenAI's typed context_length_exceeded code, or a + /// gateway's overflow message () — or Codex's own + /// sentence for a streamed refusal. Only that event, the turn's verdict, is read, so an agent message about context + /// windows never is; and a body whose code is a 5xx is the gateway failing, which stays retryable. /// private static bool RefusedAsOverContextWindow(JsonElement? line) { @@ -82,18 +84,24 @@ private static bool RefusedAsOverContextWindow(JsonElement? line) var text = message.GetString() ?? ""; - return ProviderErrorCode(text) == "context_length_exceeded" || text.Contains(StreamedOverflowSentence, StringComparison.Ordinal); + if (text.Contains(StreamedOverflowSentence, StringComparison.Ordinal)) return true; + + return ProviderRefusal(text) is { } refusal && !refusal.ServerError && (refusal.Code == "context_length_exceeded" || AgentTerminalOutcomeReader.NamesAContextOverflow(refusal.Message)); } - /// The error.code of a provider error body Codex relays as its message verbatim, or null when the message is not one. - private static string? ProviderErrorCode(string message) + /// The error of a provider body Codex relays as its message verbatim — its code (a string, or a number as text) and message — or null when the message is not one. + private static (string? Code, string Message, bool ServerError)? ProviderRefusal(string message) { try { using var body = JsonDocument.Parse(message); - return body.RootElement.ValueKind == JsonValueKind.Object && body.RootElement.TryGetProperty("error", out var error) && error.ValueKind == JsonValueKind.Object - && error.TryGetProperty("code", out var code) && code.ValueKind == JsonValueKind.String ? code.GetString() : null; + if (body.RootElement.ValueKind != JsonValueKind.Object || !body.RootElement.TryGetProperty("error", out var error) || error.ValueKind != JsonValueKind.Object) return null; + + var code = error.TryGetProperty("code", out var c) ? c.ValueKind switch { JsonValueKind.String => c.GetString(), JsonValueKind.Number => c.GetRawText(), _ => null } : null; + var text = error.TryGetProperty("message", out var m) && m.ValueKind == JsonValueKind.String ? m.GetString() ?? "" : ""; + + return (code, text, int.TryParse(code, out var status) && status >= 500); } catch (JsonException) { diff --git a/backend/tests/CodeSpace.UnitTests/Agents/AgentContextWindowRetryTests.cs b/backend/tests/CodeSpace.UnitTests/Agents/AgentContextWindowRetryTests.cs index 02f2efbfd..8e53a348b 100644 --- a/backend/tests/CodeSpace.UnitTests/Agents/AgentContextWindowRetryTests.cs +++ b/backend/tests/CodeSpace.UnitTests/Agents/AgentContextWindowRetryTests.cs @@ -33,6 +33,8 @@ public sealed class AgentContextWindowRetryTests """{"type":"result","subtype":"success","is_error":true,"num_turns":1,"terminal_reason":"prompt_too_long","api_error_status":400,"session_id":"58110d54-f8b2-43f3-9153-dcd32a2d58dc","result":"Prompt is too long · this conversation is a single exchange and cannot be compacted — the request size comes mostly from system prompt, tool definitions, or attachments."}""", // An OpenAI-compatible gateway (vLLM/LiteLLM) body the CLI passes through verbatim as terminal_reason=api_error. """{"type":"result","subtype":"success","is_error":true,"num_turns":1,"terminal_reason":"api_error","api_error_status":400,"session_id":"6406b570-40de-4274-a586-ab044dbf5d48","result":"API Error: 400 This model's maximum context length is 131072 tokens. However, you requested 161234 tokens. Please reduce the length of the messages."}""", + // The CLI's own local refusal: its token estimate is past the window, so it sent no request at all (pinned 2.1.263, a 1.2 MB goal). + """{"type":"result","subtype":"success","is_error":true,"num_turns":1,"terminal_reason":"blocking_limit","api_error_status":null,"session_id":"ff8b4b95-8351-4462-ada3-c302b4f526d8","result":"Prompt is too long"}""", }; public static TheoryData ClaudeConnectionFailureLines => new() @@ -48,6 +50,10 @@ public sealed class AgentContextWindowRetryTests """{"type":"turn.failed","error":{"message":"{\"error\": {\"message\": \"Your input exceeds the context window of this model. Please adjust your input and try again.\", \"type\": \"invalid_request_error\", \"param\": \"input\", \"code\": \"context_length_exceeded\"}}"}}""", // The same refusal delivered as a streaming response.failed, which Codex rewords. """{"type":"turn.failed","error":{"message":"Codex ran out of room in the model's context window. Start a new thread or clear earlier history before retrying."}}""", + // A vLLM gateway's refusal, relayed verbatim by the pinned 0.142.2: the code is the number 400, the reason only in the message. + """{"type":"turn.failed","error":{"message":"{\"error\": {\"message\": \"This model's maximum context length is 131072 tokens. However, you requested 161234 tokens (161234 in the messages, 0 in the completion). Please reduce the length of the messages or completion.\", \"type\": \"BadRequestError\", \"param\": null, \"code\": 400}}"}}""", + // A LiteLLM proxy's refusal, relayed verbatim by the pinned 0.142.2: the code is the string "400". + """{"type":"turn.failed","error":{"message":"{\"error\": {\"message\": \"litellm.ContextWindowExceededError: litellm.BadRequestError: ContextWindowExceededError: OpenAIException - Error code: 400 - {'error': {'message': \\\"This model's maximum context length is 128000 tokens. However, your messages resulted in 161234 tokens.\\\", 'type': 'invalid_request_error', 'param': 'messages', 'code': 'context_length_exceeded'}}\", \"type\": null, \"param\": null, \"code\": \"400\"}}"}}""", }; [Theory] @@ -92,6 +98,18 @@ public void A_claude_connection_failure_folds_to_its_own_cause(string terminalLi result.SessionId.ShouldNotBeNullOrEmpty(customMessage: "the fold completed, so the session a retry resumes is still there"); } + [Fact] + public void A_codex_gateway_error_whose_body_mentions_context_length_is_not_an_overflow() + { + // The same rule as Claude's: a 5xx is the gateway failing, and it is worth a retry. + var harness = new CodexHarness(); + var line = """{"type":"turn.failed","error":{"message":"{\"error\": {\"message\": \"upstream timed out after prefilling; maximum context length is 131072 tokens\", \"type\": \"ServiceUnavailableError\", \"code\": 503}}"}}"""; + + var result = harness.BuildResult(harness.ParseEvents(line), exitCode: 1, diagnostics: ""); + + result.ExitReason.ShouldBe("non-zero-exit"); + } + [Fact] public void Codex_refusing_an_input_past_its_own_character_cap_is_that_refusal_not_a_context_overflow() { From f746d7963202892665cc1ef4bda9337d5ae1abdb Mon Sep 17 00:00:00 2001 From: "Mars.P" Date: Thu, 24 Sep 2026 11:43:08 +0800 Subject: [PATCH 10/16] Name the cost cap only on a node that would have retried An unpriced failure is never retried under a cap, and the message said so - on every node, including one whose own policy allows a single attempt, where the cap decided nothing and sent the operator after pricing for a retry that could not have happened. The note now needs the node's retry policy too; the verdict is unchanged. --- .../Workflows/Nodes/Builtin/AgentCodeNode.cs | 8 ++++---- .../Workflows/AgentCodeNodeTests.cs | 14 ++++++++++++++ 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/backend/src/CodeSpace.Core/Services/Workflows/Nodes/Builtin/AgentCodeNode.cs b/backend/src/CodeSpace.Core/Services/Workflows/Nodes/Builtin/AgentCodeNode.cs index 98d84c83c..525aa29ac 100644 --- a/backend/src/CodeSpace.Core/Services/Workflows/Nodes/Builtin/AgentCodeNode.cs +++ b/backend/src/CodeSpace.Core/Services/Workflows/Nodes/Builtin/AgentCodeNode.cs @@ -134,7 +134,7 @@ public Task RunAsync(NodeRunContext context, CancellationToken cance if (!TryReadCostCap(context.Config, out var maxCostUsd)) return Fail("Config 'maxCostUsd' must be a positive USD amount when set."); // Resumed: the agent run finished. ResumePayload = { status, summary, changedFiles, branch, error }. - if (context.ResumePayload.HasValue) return Task.FromResult(MapResult(context.ResumePayload.Value, maxCostUsd)); + if (context.ResumePayload.HasValue) return Task.FromResult(MapResult(context.ResumePayload.Value, maxCostUsd, context.RetriesOnFailure)); if (!TryReadPriorSpend(context.PriorAttemptPayload, maxCostUsd, out var budgetSpentUsd, out var budgetError)) return Fail(budgetError!); @@ -296,7 +296,7 @@ private static bool TryReadAcceptance(IReadOnlyDictionary c } /// Map the resumed agent-run outcome onto this node's result. Succeeded → outputs; anything else → a clean node failure, marked retryable only when a fresh respawn could change the outcome. - private static NodeResult MapResult(JsonElement payload, decimal? maxCostUsd) + private static NodeResult MapResult(JsonElement payload, decimal? maxCostUsd, bool retriesOnFailure) { var status = ReadString(payload, "status"); var succeeded = status == nameof(AgentRunStatus.Succeeded); @@ -402,7 +402,7 @@ private static NodeResult MapResult(JsonElement payload, decimal? maxCostUsd) && !escalationAvailable) || mitigationSpent; - return NodeResult.Fail($"Agent run did not succeed: {(string.IsNullOrEmpty(error) ? status : error)}{FailureCauseSuffix(cause, mitigationSpent)}{UnpricedRetrySuffix(unpriced && !deterministic, maxCostUsd)}", retryable: !deterministic && !unpriced); + return NodeResult.Fail($"Agent run did not succeed: {(string.IsNullOrEmpty(error) ? status : error)}{FailureCauseSuffix(cause, mitigationSpent)}{UnpricedRetrySuffix(unpriced && !deterministic && retriesOnFailure, maxCostUsd)}", retryable: !deterministic && !unpriced); } var outputs = new Dictionary { ["status"] = JsonSerializer.SerializeToElement(nameof(AgentRunStatus.Succeeded)) }; @@ -436,7 +436,7 @@ private static NodeResult MapResult(JsonElement payload, decimal? maxCostUsd) _ => "", }; - /// Why a failure a respawn could otherwise change is not respawned: its spend is unknown, so a retry cannot be bought under the cap. Empty whenever that is not the deciding fact, so the cause stays the whole message. + /// Why a failure a respawn could otherwise change is not respawned: its spend is unknown, so a retry cannot be bought under the cap. Empty whenever that is not the deciding fact — including on a node whose own policy never retries — so the cause stays the whole message. private static string UnpricedRetrySuffix(bool decides, decimal? maxCostUsd) => decides ? $"; not retried: its spend cannot be priced under the monitored ${maxCostUsd!.Value.ToString(CultureInfo.InvariantCulture)} cost cap" : ""; diff --git a/backend/tests/CodeSpace.UnitTests/Workflows/AgentCodeNodeTests.cs b/backend/tests/CodeSpace.UnitTests/Workflows/AgentCodeNodeTests.cs index 1c97e5546..c9e76edc1 100644 --- a/backend/tests/CodeSpace.UnitTests/Workflows/AgentCodeNodeTests.cs +++ b/backend/tests/CodeSpace.UnitTests/Workflows/AgentCodeNodeTests.cs @@ -596,6 +596,20 @@ public async Task An_unpriced_failure_on_a_cost_capped_node_keeps_its_cause_and_ result.Error.ShouldEndWith("; not retried: its spend cannot be priced under the monitored $5 cost cap"); } + [Fact] + public async Task An_unpriced_failure_on_a_single_attempt_node_does_not_blame_the_cost_cap() + { + // With no retry policy there was never a retry to refuse, so naming the cap as the reason would send the + // operator after pricing for a retry that could not have happened either way. Still not retryable. + var config = new Dictionary { ["maxCostUsd"] = Num(5) }; + var resume = JsonDocument.Parse(JsonSerializer.Serialize(new { status = "Failed", error = "claude exited with code 1", exitReason = "non-zero-exit" })).RootElement; + + var result = await new AgentCodeNode().RunAsync(BuildContext(config, resume, retriesOnFailure: false), CancellationToken.None); + + result.Retryable.ShouldBeFalse(); + result.Error.ShouldBe("Agent run did not succeed: claude exited with code 1"); + } + [Fact] public void The_two_watchdogs_are_classified_alike_because_neither_can_see_why_the_process_went_quiet() { From f94df88f19aff670e08f20da28c36d789d816e95 Mon Sep 17 00:00:00 2001 From: "Mars.P" Date: Thu, 24 Sep 2026 11:43:09 +0800 Subject: [PATCH 11/16] Classify Anthropic's overflow bodies as a context-length error Neither "prompt is too long: N tokens > M maximum" nor "input length and max_tokens exceed context limit" matched a needle, so both were BadRequest. The supervisor brain's reactive compaction keys on ContextLengthExceeded and never ran for an Anthropic model whose window the operator had not declared, and every llm.complete overflow was labelled a malformed request. --- .../CodeSpace.Core/Services/Workflows/Llm/LlmApiException.cs | 2 +- .../tests/CodeSpace.UnitTests/Workflows/LlmApiExceptionTests.cs | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/src/CodeSpace.Core/Services/Workflows/Llm/LlmApiException.cs b/backend/src/CodeSpace.Core/Services/Workflows/Llm/LlmApiException.cs index 0ab344569..664107432 100644 --- a/backend/src/CodeSpace.Core/Services/Workflows/Llm/LlmApiException.cs +++ b/backend/src/CodeSpace.Core/Services/Workflows/Llm/LlmApiException.cs @@ -98,7 +98,7 @@ 400 or 422 when MentionsContentFilter(body) => LlmErrorCategory.ContentFiltered, // The category is only a refined LABEL on a 400/422 anyway (the degrade decision is status-based), so a miss merely // leaves the generic BadRequest — it can never disable the progressive fallback. private static bool MentionsContextLength(string? body) => - ContainsAny(body, "context length", "context_length", "context window", "maximum context", "maximum_tokens", "reduce the length", "string too long", "too many tokens", "exceeds the maximum"); + ContainsAny(body, "context length", "context_length", "context window", "maximum context", "maximum_tokens", "reduce the length", "string too long", "too many tokens", "exceeds the maximum", "prompt is too long", "exceed context limit"); private static bool MentionsContentFilter(string? body) => ContainsAny(body, "content filter", "content_filter", "content policy", "content_policy", "content was blocked", "blocked by safety", "safety policy", "flagged"); diff --git a/backend/tests/CodeSpace.UnitTests/Workflows/LlmApiExceptionTests.cs b/backend/tests/CodeSpace.UnitTests/Workflows/LlmApiExceptionTests.cs index af0a595b3..571cfc5d9 100644 --- a/backend/tests/CodeSpace.UnitTests/Workflows/LlmApiExceptionTests.cs +++ b/backend/tests/CodeSpace.UnitTests/Workflows/LlmApiExceptionTests.cs @@ -31,6 +31,8 @@ public void Classify_maps_status_to_category(int status, LlmErrorCategory expect [InlineData(400, "This model's maximum context length is 8192 tokens")] [InlineData(413, "input is too long for the context window")] [InlineData(422, "reduce the length of the messages")] + [InlineData(400, """{"type":"error","error":{"type":"invalid_request_error","message":"prompt is too long: 215000 tokens > 200000 maximum"}}""")] + [InlineData(400, """{"type":"error","error":{"type":"invalid_request_error","message":"input length and `max_tokens` exceed context limit: 197000 + 32000 > 200000, decrease input length or `max_tokens` and try again"}}""")] public void Classify_refines_a_4xx_to_context_length_on_a_matching_body(int status, string body) { LlmApiException.Classify(status, body).ShouldBe(LlmErrorCategory.ContextLengthExceeded); From cbbaaea953ac5d95794f233c2074bffd2694b569 Mon Sep 17 00:00:00 2001 From: "Mars.P" Date: Thu, 24 Sep 2026 18:21:12 +0800 Subject: [PATCH 12/16] Never let a relayed body that is not valid text throw out of a fold A gateway's error text can be well-formed JSON that is not valid text: an unpaired surrogate escape (a preview cut in the middle of an emoji) parses, then throws on read. Codex relays such a 400 body verbatim, and the fold read its message to look for an overflow, so BuildResult threw on every such line, overflow or not. The run landed as executor-error and its session, diff and transcript were dropped - the same shape as the null-status throw, one field over. An unreadable body is now simply not a refusal the fold can type. The Claude fold's string reads get the same guard. Pinned with the pinned 0.142.2's own relayed lines. --- .../Claude/ClaudeCodeResultFolder.cs | 16 +++++++++++-- .../Harnesses/Codex/CodexResultFolder.cs | 9 ++++++-- .../Agents/AgentContextWindowRetryTests.cs | 23 +++++++++++++++++++ 3 files changed, 44 insertions(+), 4 deletions(-) diff --git a/backend/src/CodeSpace.Core/Services/Agents/Harnesses/Claude/ClaudeCodeResultFolder.cs b/backend/src/CodeSpace.Core/Services/Agents/Harnesses/Claude/ClaudeCodeResultFolder.cs index c4180336e..4d42d604b 100644 --- a/backend/src/CodeSpace.Core/Services/Agents/Harnesses/Claude/ClaudeCodeResultFolder.cs +++ b/backend/src/CodeSpace.Core/Services/Agents/Harnesses/Claude/ClaudeCodeResultFolder.cs @@ -92,6 +92,18 @@ private static bool RefusedAsOverContextWindow(JsonElement? line) private static bool IsClientRefusal(JsonElement result) => result.TryGetProperty("api_error_status", out var status) && status.ValueKind == JsonValueKind.Number && status.TryGetInt32(out var code) && code == 400; - private static string ReadString(JsonElement root, string key) => - root.TryGetProperty(key, out var value) && value.ValueKind == JsonValueKind.String ? value.GetString() ?? "" : ""; + /// A string field of the CLI's line, or "" when it is absent, another kind, or not valid text (an unpaired surrogate escape a relayed gateway body can carry parses, then throws on read) — the fold must never be where a run's work is dropped. + private static string ReadString(JsonElement root, string key) + { + if (!root.TryGetProperty(key, out var value) || value.ValueKind != JsonValueKind.String) return ""; + + try + { + return value.GetString() ?? ""; + } + catch (InvalidOperationException) + { + return ""; + } + } } diff --git a/backend/src/CodeSpace.Core/Services/Agents/Harnesses/Codex/CodexResultFolder.cs b/backend/src/CodeSpace.Core/Services/Agents/Harnesses/Codex/CodexResultFolder.cs index e45a83c93..b83b267b8 100644 --- a/backend/src/CodeSpace.Core/Services/Agents/Harnesses/Codex/CodexResultFolder.cs +++ b/backend/src/CodeSpace.Core/Services/Agents/Harnesses/Codex/CodexResultFolder.cs @@ -89,7 +89,12 @@ private static bool RefusedAsOverContextWindow(JsonElement? line) return ProviderRefusal(text) is { } refusal && !refusal.ServerError && (refusal.Code == "context_length_exceeded" || AgentTerminalOutcomeReader.NamesAContextOverflow(refusal.Message)); } - /// The error of a provider body Codex relays as its message verbatim — its code (a string, or a number as text) and message — or null when the message is not one. + /// + /// The error of a provider body Codex relays as its message verbatim — its code (a string, or a number as + /// text) and message — or null when the message is not one. A body is the gateway's own text, so it can be + /// well-formed JSON that is not valid text: an unpaired surrogate escape (\ud83d) parses, then throws on + /// read. That is "not a refusal we can read", never a reason for the fold to throw and drop the run's work. + /// private static (string? Code, string Message, bool ServerError)? ProviderRefusal(string message) { try @@ -103,7 +108,7 @@ private static (string? Code, string Message, bool ServerError)? ProviderRefusal return (code, text, int.TryParse(code, out var status) && status >= 500); } - catch (JsonException) + catch (Exception ex) when (ex is JsonException or InvalidOperationException) { return null; } diff --git a/backend/tests/CodeSpace.UnitTests/Agents/AgentContextWindowRetryTests.cs b/backend/tests/CodeSpace.UnitTests/Agents/AgentContextWindowRetryTests.cs index 8e53a348b..61185b4cb 100644 --- a/backend/tests/CodeSpace.UnitTests/Agents/AgentContextWindowRetryTests.cs +++ b/backend/tests/CodeSpace.UnitTests/Agents/AgentContextWindowRetryTests.cs @@ -110,6 +110,29 @@ public void A_codex_gateway_error_whose_body_mentions_context_length_is_not_an_o result.ExitReason.ShouldBe("non-zero-exit"); } + public static TheoryData CodexUnreadableBodyLines => new() + { + // The pinned 0.142.2 relaying a gateway 400 whose message holds an unpaired surrogate escape (a preview cut in + // the middle of an emoji) — once a non-overflow, once an overflow. The body parses and then throws on read. + """{"type":"turn.failed","error":{"message":"{\"error\": {\"message\": \"Invalid value for input[0]: preview \\ud83d ... (truncated)\", \"type\": \"BadRequestError\", \"param\": null, \"code\": 400}}"}}""", + """{"type":"turn.failed","error":{"message":"{\"error\": {\"message\": \"This model's maximum context length is 131072 tokens. However, you requested 161234 tokens. Prompt starts: \\ud83d\", \"type\": \"BadRequestError\", \"param\": null, \"code\": 400}}"}}""", + }; + + [Theory] + [MemberData(nameof(CodexUnreadableBodyLines))] + public void A_relayed_body_that_is_not_valid_text_never_throws_out_of_the_fold(string failedLine) + { + // A throw here lands the run as executor-error and drops its session, diff and transcript — the fold is never + // the place that happens. The body is unreadable, so it is not a refusal this fold can type. + var harness = new CodexHarness(); + var events = harness.ParseEvents("""{"type":"thread.started","thread_id":"01a0d18d-0000-7000-8000-000000000001"}""").Concat(harness.ParseEvents(failedLine)).ToList(); + + var result = harness.BuildResult(events, exitCode: 1, diagnostics: ""); + + result.ExitReason.ShouldBe("non-zero-exit"); + result.SessionId.ShouldNotBeNullOrEmpty(customMessage: "the fold completed, so the thread a retry resumes is kept"); + } + [Fact] public void Codex_refusing_an_input_past_its_own_character_cap_is_that_refusal_not_a_context_overflow() { From cf724e30acc0b26f450ae78b09dc49d6e17fde9a Mon Sep 17 00:00:00 2001 From: "Mars.P" Date: Thu, 24 Sep 2026 18:21:12 +0800 Subject: [PATCH 13/16] Type a LiteLLM refusal in front of a non-OpenAI model The shared gateway markers matched a LiteLLM refusal only when the upstream was OpenAI-shaped. In front of a Claude or Bedrock model LiteLLM carries its own class name and the upstream's words, so a Codex agent on such a proxy was respawned into the same refusal. The list gains LiteLLM's ContextWindowExceededError and Anthropic's two refusals, still read only off a relayed 400 body. The Codex negative that stood for a 5xx was a line the pinned 0.142.2 never prints: it rewraps a 503, 413 or 422 as prose. The real lines are pinned now, and stay retryable whatever they mention. --- .../Agents/AgentTerminalOutcomeReader.cs | 9 +++++++- .../Agents/AgentContextWindowRetryTests.cs | 21 +++++++++++++++---- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/backend/src/CodeSpace.Core/Services/Agents/AgentTerminalOutcomeReader.cs b/backend/src/CodeSpace.Core/Services/Agents/AgentTerminalOutcomeReader.cs index 8aec846d4..40c46f3b7 100644 --- a/backend/src/CodeSpace.Core/Services/Agents/AgentTerminalOutcomeReader.cs +++ b/backend/src/CodeSpace.Core/Services/Agents/AgentTerminalOutcomeReader.cs @@ -33,7 +33,14 @@ public static class AgentTerminalOutcomeReader public static bool NamesAContextOverflow(string providerMessage) => GatewayOverflowMarkers.Any(marker => providerMessage.Contains(marker, StringComparison.OrdinalIgnoreCase)); - private static readonly string[] GatewayOverflowMarkers = { "maximum context length is", "context_length_exceeded" }; + private static readonly string[] GatewayOverflowMarkers = + { + "maximum context length is", // vLLM, and OpenAI's own wording that gateways pass through + "context_length_exceeded", // OpenAI's error code, embedded in a gateway's message + "ContextWindowExceededError", // LiteLLM's class name, stamped only on a window refusal whatever the upstream + "prompt is too long", // Anthropic's refusal, relayed by a gateway in front of a Claude model + "exceed context limit", // Anthropic's other refusal (input length and max_tokens) + }; /// /// True when the last Completed-or-Error event in the stream is an Error — i.e. the harness itself reported diff --git a/backend/tests/CodeSpace.UnitTests/Agents/AgentContextWindowRetryTests.cs b/backend/tests/CodeSpace.UnitTests/Agents/AgentContextWindowRetryTests.cs index 61185b4cb..e96984c45 100644 --- a/backend/tests/CodeSpace.UnitTests/Agents/AgentContextWindowRetryTests.cs +++ b/backend/tests/CodeSpace.UnitTests/Agents/AgentContextWindowRetryTests.cs @@ -52,6 +52,9 @@ public sealed class AgentContextWindowRetryTests """{"type":"turn.failed","error":{"message":"Codex ran out of room in the model's context window. Start a new thread or clear earlier history before retrying."}}""", // A vLLM gateway's refusal, relayed verbatim by the pinned 0.142.2: the code is the number 400, the reason only in the message. """{"type":"turn.failed","error":{"message":"{\"error\": {\"message\": \"This model's maximum context length is 131072 tokens. However, you requested 161234 tokens (161234 in the messages, 0 in the completion). Please reduce the length of the messages or completion.\", \"type\": \"BadRequestError\", \"param\": null, \"code\": 400}}"}}""", + // A LiteLLM proxy in front of a Claude model: its class name and Anthropic's own words, neither OpenAI-shaped. + // Representative of LiteLLM's exception mapping (not byte-captured from a live proxy), relayed the way 0.142.2 relays any 400 body. + """{"type":"turn.failed","error":{"message":"{\"error\": {\"message\": \"litellm.ContextWindowExceededError: litellm.BadRequestError: AnthropicException - {\\\"type\\\":\\\"error\\\",\\\"error\\\":{\\\"type\\\":\\\"invalid_request_error\\\",\\\"message\\\":\\\"prompt is too long: 215000 tokens > 200000 maximum\\\"}}\", \"type\": null, \"param\": null, \"code\": \"400\"}}"}}""", // A LiteLLM proxy's refusal, relayed verbatim by the pinned 0.142.2: the code is the string "400". """{"type":"turn.failed","error":{"message":"{\"error\": {\"message\": \"litellm.ContextWindowExceededError: litellm.BadRequestError: ContextWindowExceededError: OpenAIException - Error code: 400 - {'error': {'message': \\\"This model's maximum context length is 128000 tokens. However, your messages resulted in 161234 tokens.\\\", 'type': 'invalid_request_error', 'param': 'messages', 'code': 'context_length_exceeded'}}\", \"type\": null, \"param\": null, \"code\": \"400\"}}"}}""", }; @@ -98,12 +101,22 @@ public void A_claude_connection_failure_folds_to_its_own_cause(string terminalLi result.SessionId.ShouldNotBeNullOrEmpty(customMessage: "the fold completed, so the session a retry resumes is still there"); } - [Fact] - public void A_codex_gateway_error_whose_body_mentions_context_length_is_not_an_overflow() + public static TheoryData CodexNonRefusalLines => new() + { + // What the pinned 0.142.2 really prints for a status it does not relay verbatim: a 503, 413 and 422 whose bodies + // name the window. It rewraps them as prose, so they are not a relayed refusal, and a 5xx is worth a retry. + """{"type":"turn.failed","error":{"message":"unexpected status 503 Service Unavailable: upstream prefill timed out; maximum context length is 131072 tokens, url: http://127.0.0.1:19703/v1/responses"}}""", + """{"type":"turn.failed","error":{"message":"unexpected status 413 Payload Too Large: This model's maximum context length is 131072 tokens. However, you requested 161234 tokens., url: http://127.0.0.1:19719/v1/responses"}}""", + """{"type":"turn.failed","error":{"message":"unexpected status 422 Unprocessable Entity: This model's maximum context length is 131072 tokens. However, you requested 161234 tokens., url: http://127.0.0.1:19891/v1/responses"}}""", + // Belt and braces for the 5xx guard: a JSON body with a 5xx code, which the pin does not print today. + """{"type":"turn.failed","error":{"message":"{\"error\": {\"message\": \"upstream timed out after prefilling; maximum context length is 131072 tokens\", \"type\": \"ServiceUnavailableError\", \"code\": 503}}"}}""", + }; + + [Theory] + [MemberData(nameof(CodexNonRefusalLines))] + public void A_codex_failure_that_is_not_a_relayed_refusal_is_not_an_overflow_whatever_it_mentions(string line) { - // The same rule as Claude's: a 5xx is the gateway failing, and it is worth a retry. var harness = new CodexHarness(); - var line = """{"type":"turn.failed","error":{"message":"{\"error\": {\"message\": \"upstream timed out after prefilling; maximum context length is 131072 tokens\", \"type\": \"ServiceUnavailableError\", \"code\": 503}}"}}"""; var result = harness.BuildResult(harness.ParseEvents(line), exitCode: 1, diagnostics: ""); From bba62843e5933d603a2f632800f9d49e4ed1cc63 Mon Sep 17 00:00:00 2001 From: "Mars.P" Date: Thu, 24 Sep 2026 18:21:19 +0800 Subject: [PATCH 14/16] Describe a context overflow without claiming the model refused it blocking_limit shares the exit reason with a provider's refusal, but the CLI raises it before sending anything, measured against the window it believes. The node's cause suffix said the model had refused the request. It now says the request is larger than the window the CLI or the model allows, and the classifier's and exit reason's docs say the same. --- .../Services/Agents/AgentTerminalOutcomeReader.cs | 4 ++-- .../Services/Supervisor/AgentRetryCauses.cs | 8 ++++---- .../Services/Workflows/Nodes/Builtin/AgentCodeNode.cs | 10 +++++----- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/backend/src/CodeSpace.Core/Services/Agents/AgentTerminalOutcomeReader.cs b/backend/src/CodeSpace.Core/Services/Agents/AgentTerminalOutcomeReader.cs index 40c46f3b7..12f49afa9 100644 --- a/backend/src/CodeSpace.Core/Services/Agents/AgentTerminalOutcomeReader.cs +++ b/backend/src/CodeSpace.Core/Services/Agents/AgentTerminalOutcomeReader.cs @@ -15,8 +15,8 @@ public static class AgentTerminalOutcomeReader { /// /// The a harness's folder stamps when its CLI's OWN terminal event says the - /// model refused the request as larger than its context window — a field or status the CLI wrote, never a phrase in - /// the agent's prose. Harness-agnostic, like the rest of this reader: each folder knows its CLI's shape, and every + /// request is larger than the model's context window — the provider refused it, or the CLI did before sending it: + /// a field or status the CLI wrote, never a phrase in the agent's prose. Harness-agnostic, like the rest of this reader: each folder knows its CLI's shape, and every /// consumer (the retry-cause classifier, and through it the agent.run node's retry verdict) keys on this one code. /// Pinned by a unit test (Rule 8) so the producers and the verdict cannot drift apart. /// diff --git a/backend/src/CodeSpace.Core/Services/Supervisor/AgentRetryCauses.cs b/backend/src/CodeSpace.Core/Services/Supervisor/AgentRetryCauses.cs index 1c1dd899b..35e6a80f2 100644 --- a/backend/src/CodeSpace.Core/Services/Supervisor/AgentRetryCauses.cs +++ b/backend/src/CodeSpace.Core/Services/Supervisor/AgentRetryCauses.cs @@ -32,10 +32,10 @@ public static class AgentRetryCauses public const string ModelAccessLost = "model-access-lost"; /// - /// The model refused the request as larger than its context window - /// () — deterministic on replay. A retry - /// warm-resumes the conversation, so its request carries the goal AGAIN and is longer still; a fresh one carries - /// the same goal. Either way the same refusal comes back, billed, and buries the one fact the author needs: the + /// The request is larger than the model's context window — the provider refused it, or the CLI did before sending + /// it () — deterministic on replay. A + /// retry warm-resumes the conversation, so its request carries the goal AGAIN and is longer still; a fresh one + /// carries the same goal. Either way the same refusal comes back and buries the one fact the author needs: the /// agent is being given more than this model can read. No mitigation, like — its /// only consumer that matters is AgentCodeNode, which stops respawning it. /// diff --git a/backend/src/CodeSpace.Core/Services/Workflows/Nodes/Builtin/AgentCodeNode.cs b/backend/src/CodeSpace.Core/Services/Workflows/Nodes/Builtin/AgentCodeNode.cs index 525aa29ac..9bb4629b7 100644 --- a/backend/src/CodeSpace.Core/Services/Workflows/Nodes/Builtin/AgentCodeNode.cs +++ b/backend/src/CodeSpace.Core/Services/Workflows/Nodes/Builtin/AgentCodeNode.cs @@ -389,10 +389,10 @@ private static NodeResult MapResult(JsonElement payload, decimal? maxCostUsd, bo var formatFault = cause == Supervisor.AgentRetryCauses.GatewayFormatFault; var mitigationSpent = formatFault && ReadFlag(payload, "thinkingDisabled"); - // The model refused the request as larger than its context window — typed by the harness from its CLI's own - // fields, never read from the error text. A respawn warm-resumes, so it re-sends the goal inside a LONGER - // request, and a fresh one sends the same goal: either way it sends at least as much and is refused the - // same way. The escalation escape below never opens for it: the trigger proposes a stronger model only on a + // The request is larger than the context window — the model refused it, or the CLI did before sending it — + // typed by the harness from its CLI's own fields, never read from the error text. A respawn warm-resumes, + // so it re-sends the goal inside a LONGER request, and a fresh one sends the same goal: either way it + // sends at least as much and is refused the same way. The escalation escape below never opens for it: the trigger proposes a stronger model only on a // failed grade, and stands down for any classified cause, so an overflowing attempt carries no proposal. var contextWindowExceeded = cause == Supervisor.AgentRetryCauses.ContextWindowExceeded; @@ -432,7 +432,7 @@ private static NodeResult MapResult(JsonElement payload, decimal? maxCostUsd, bo { Supervisor.AgentRetryCauses.GatewayFormatFault when mitigationSpent => $" ({Supervisor.AgentRetryCauses.GatewayFormatFault}: a fresh conversation with extended thinking disabled hit the same fault)", Supervisor.AgentRetryCauses.GatewayFormatFault => $" ({Supervisor.AgentRetryCauses.GatewayFormatFault})", - Supervisor.AgentRetryCauses.ContextWindowExceeded => $" ({Supervisor.AgentRetryCauses.ContextWindowExceeded}: the model refused the request as larger than its context window, and a respawn would send at least as much — give the agent less, or choose a model with a larger window)", + Supervisor.AgentRetryCauses.ContextWindowExceeded => $" ({Supervisor.AgentRetryCauses.ContextWindowExceeded}: the request is larger than the context window the CLI or the model allows, and a respawn would send at least as much — give the agent less, or choose a model with a larger window)", _ => "", }; From d43c779dd93d39be67b340a0f8e4153403d5e1d7 Mon Sep 17 00:00:00 2001 From: "Mars.P" Date: Thu, 24 Sep 2026 20:29:58 +0800 Subject: [PATCH 15/16] Keep a readable refusal readable beside one bad character Guarding the unreadable-message throw dropped the whole relayed body, including a code already read. A gateway refusal carrying OpenAI's typed context_length_exceeded beside a message with an unpaired surrogate escape was typed as an ordinary, retryable failure, and so was a vLLM overflow whose ASCII wording was intact. An unreadable string now stands in as its escaped text. Every word the folds look for is ASCII, which an escape cannot split, so one bad character neither throws out of the fold nor hides what the rest of the body states. A lookup that cannot unescape a gateway-authored key is still "not a refusal", never a throw. --- .../Claude/ClaudeCodeResultFolder.cs | 4 +-- .../Harnesses/Codex/CodexResultFolder.cs | 28 +++++++++++++++---- .../Agents/AgentContextWindowRetryTests.cs | 23 ++++++++------- 3 files changed, 38 insertions(+), 17 deletions(-) diff --git a/backend/src/CodeSpace.Core/Services/Agents/Harnesses/Claude/ClaudeCodeResultFolder.cs b/backend/src/CodeSpace.Core/Services/Agents/Harnesses/Claude/ClaudeCodeResultFolder.cs index 4d42d604b..6bbe563d3 100644 --- a/backend/src/CodeSpace.Core/Services/Agents/Harnesses/Claude/ClaudeCodeResultFolder.cs +++ b/backend/src/CodeSpace.Core/Services/Agents/Harnesses/Claude/ClaudeCodeResultFolder.cs @@ -92,7 +92,7 @@ private static bool RefusedAsOverContextWindow(JsonElement? line) private static bool IsClientRefusal(JsonElement result) => result.TryGetProperty("api_error_status", out var status) && status.ValueKind == JsonValueKind.Number && status.TryGetInt32(out var code) && code == 400; - /// A string field of the CLI's line, or "" when it is absent, another kind, or not valid text (an unpaired surrogate escape a relayed gateway body can carry parses, then throws on read) — the fold must never be where a run's work is dropped. + /// A string field of the CLI's line, or "" when it is absent or another kind. When it is not valid text (an unpaired surrogate escape a relayed gateway body can carry parses, then throws on read), its escaped text stands in: the words the fold looks for are ASCII, and the fold must never be where a run's work is dropped. private static string ReadString(JsonElement root, string key) { if (!root.TryGetProperty(key, out var value) || value.ValueKind != JsonValueKind.String) return ""; @@ -103,7 +103,7 @@ private static string ReadString(JsonElement root, string key) } catch (InvalidOperationException) { - return ""; + return value.GetRawText(); } } } diff --git a/backend/src/CodeSpace.Core/Services/Agents/Harnesses/Codex/CodexResultFolder.cs b/backend/src/CodeSpace.Core/Services/Agents/Harnesses/Codex/CodexResultFolder.cs index b83b267b8..6d2408ecb 100644 --- a/backend/src/CodeSpace.Core/Services/Agents/Harnesses/Codex/CodexResultFolder.cs +++ b/backend/src/CodeSpace.Core/Services/Agents/Harnesses/Codex/CodexResultFolder.cs @@ -91,9 +91,7 @@ private static bool RefusedAsOverContextWindow(JsonElement? line) /// /// The error of a provider body Codex relays as its message verbatim — its code (a string, or a number as - /// text) and message — or null when the message is not one. A body is the gateway's own text, so it can be - /// well-formed JSON that is not valid text: an unpaired surrogate escape (\ud83d) parses, then throws on - /// read. That is "not a refusal we can read", never a reason for the fold to throw and drop the run's work. + /// text) and message — or null when the message is not one. /// private static (string? Code, string Message, bool ServerError)? ProviderRefusal(string message) { @@ -103,14 +101,34 @@ private static (string? Code, string Message, bool ServerError)? ProviderRefusal if (body.RootElement.ValueKind != JsonValueKind.Object || !body.RootElement.TryGetProperty("error", out var error) || error.ValueKind != JsonValueKind.Object) return null; - var code = error.TryGetProperty("code", out var c) ? c.ValueKind switch { JsonValueKind.String => c.GetString(), JsonValueKind.Number => c.GetRawText(), _ => null } : null; - var text = error.TryGetProperty("message", out var m) && m.ValueKind == JsonValueKind.String ? m.GetString() ?? "" : ""; + var code = error.TryGetProperty("code", out var c) ? c.ValueKind switch { JsonValueKind.String => TextOf(c), JsonValueKind.Number => c.GetRawText(), _ => null } : null; + var text = error.TryGetProperty("message", out var m) && m.ValueKind == JsonValueKind.String ? TextOf(m) : ""; return (code, text, int.TryParse(code, out var status) && status >= 500); } catch (Exception ex) when (ex is JsonException or InvalidOperationException) { + // Unparseable, or a gateway-authored key that is not valid text (a lookup unescapes keys): not a refusal + // this fold can read, and never a reason to throw. return null; } } + + /// + /// A string of the relayed body. A body is the gateway's own text, so it can be well-formed JSON that is not valid + /// text: an unpaired surrogate escape (\ud83d) parses, then throws on read. Then the escaped text stands in — + /// every word the fold looks for is ASCII, which an escape cannot split — so one bad character neither throws out of + /// the fold (dropping the run's work) nor hides a refusal the rest of the body states plainly. + /// + private static string TextOf(JsonElement value) + { + try + { + return value.GetString() ?? ""; + } + catch (InvalidOperationException) + { + return value.GetRawText(); + } + } } diff --git a/backend/tests/CodeSpace.UnitTests/Agents/AgentContextWindowRetryTests.cs b/backend/tests/CodeSpace.UnitTests/Agents/AgentContextWindowRetryTests.cs index e96984c45..7634c5cb7 100644 --- a/backend/tests/CodeSpace.UnitTests/Agents/AgentContextWindowRetryTests.cs +++ b/backend/tests/CodeSpace.UnitTests/Agents/AgentContextWindowRetryTests.cs @@ -12,9 +12,9 @@ namespace CodeSpace.UnitTests.Agents; /// /// A context-window overflow is recognised from what the CLIs actually print, through the real harness folds. /// -/// Every shape below is a terminal line a real CLI printed (Claude Code 2.1.226, Codex 0.147.0) when a local -/// endpoint answered its request with the provider's own over-long-prompt error body, trimmed to the fields the -/// fold reads. They are run through ParseEvents and BuildResult — the reader production uses — and the +/// The shapes below are terminal lines real CLIs printed (Claude Code 2.1.226 and the pinned 2.1.263, Codex +/// 0.147.0 and the pinned 0.142.2) when a local endpoint answered with a provider's own error body, trimmed to the +/// fields the fold reads — except where a case says it is representative or synthesized. They are run through ParseEvents and BuildResult — the reader production uses — and the /// cause is read off the exit reason the fold typed from the CLI's own fields, never off the error text: a text scan /// also matched a rubric that talks about context windows, and switched model escalation off for its failed grade. /// @@ -123,26 +123,29 @@ public void A_codex_failure_that_is_not_a_relayed_refusal_is_not_an_overflow_wha result.ExitReason.ShouldBe("non-zero-exit"); } - public static TheoryData CodexUnreadableBodyLines => new() + public static TheoryData CodexUnreadableBodyLines => new() { // The pinned 0.142.2 relaying a gateway 400 whose message holds an unpaired surrogate escape (a preview cut in - // the middle of an emoji) — once a non-overflow, once an overflow. The body parses and then throws on read. - """{"type":"turn.failed","error":{"message":"{\"error\": {\"message\": \"Invalid value for input[0]: preview \\ud83d ... (truncated)\", \"type\": \"BadRequestError\", \"param\": null, \"code\": 400}}"}}""", - """{"type":"turn.failed","error":{"message":"{\"error\": {\"message\": \"This model's maximum context length is 131072 tokens. However, you requested 161234 tokens. Prompt starts: \\ud83d\", \"type\": \"BadRequestError\", \"param\": null, \"code\": 400}}"}}""", + // the middle of an emoji). The body parses and then throws on read; the rest of it still says what it says. + { """{"type":"turn.failed","error":{"message":"{\"error\": {\"message\": \"Invalid value for input[0]: preview \\ud83d ... (truncated)\", \"type\": \"BadRequestError\", \"param\": null, \"code\": 400}}"}}""", "non-zero-exit" }, + { """{"type":"turn.failed","error":{"message":"{\"error\": {\"message\": \"This model's maximum context length is 131072 tokens. However, you requested 161234 tokens. Prompt starts: \\ud83d\", \"type\": \"BadRequestError\", \"param\": null, \"code\": 400}}"}}""", AgentTerminalOutcomeReader.ContextWindowExceededExitReason }, + // OpenAI's typed code beside an unreadable message, as the pin relays it: the code alone settles it. + { """{"type":"turn.failed","error":{"message":"{\"error\": {\"message\": \"Your input exceeds the context window of this model. Input preview: \\ud83d\", \"type\": \"invalid_request_error\", \"param\": \"input\", \"code\": \"context_length_exceeded\"}}"}}""", AgentTerminalOutcomeReader.ContextWindowExceededExitReason }, + }; [Theory] [MemberData(nameof(CodexUnreadableBodyLines))] - public void A_relayed_body_that_is_not_valid_text_never_throws_out_of_the_fold(string failedLine) + public void A_relayed_body_that_is_not_valid_text_never_throws_and_still_says_what_it_says(string failedLine, string expectedExitReason) { // A throw here lands the run as executor-error and drops its session, diff and transcript — the fold is never - // the place that happens. The body is unreadable, so it is not a refusal this fold can type. + // the place that happens. And one bad character must not hide a refusal the rest of the body states plainly. var harness = new CodexHarness(); var events = harness.ParseEvents("""{"type":"thread.started","thread_id":"01a0d18d-0000-7000-8000-000000000001"}""").Concat(harness.ParseEvents(failedLine)).ToList(); var result = harness.BuildResult(events, exitCode: 1, diagnostics: ""); - result.ExitReason.ShouldBe("non-zero-exit"); + result.ExitReason.ShouldBe(expectedExitReason); result.SessionId.ShouldNotBeNullOrEmpty(customMessage: "the fold completed, so the thread a retry resumes is kept"); } From 6d49c8c3d6e12cbc0189ece2db89321daa9ddefc Mon Sep 17 00:00:00 2001 From: "Mars.P" Date: Fri, 25 Sep 2026 00:55:34 +0800 Subject: [PATCH 16/16] Stop revising, not the run, when a revision crosses Codex's cap A cold revise goal restates the whole contract, so a goal just under Codex's 1,048,576-character cap produces a revision past it. The harness refused that revision while the executor built it, and the throw left the revise loop: the generic catch replaced round 0's graded result - its diff, summary and verdict - with a bare size refusal, as if nothing had run. The refusal is about the round, so it now ends the revise loop the way a spend refusal does: a timeline note says why, and the last graded round's result stands. Nothing needs settling, because the round was never admitted. --- .../Services/Agents/AgentRunExecutor.cs | 45 ++++++++++---- .../Workflows/AgentRunReviseLoopFlowTests.cs | 60 +++++++++++++++++++ 2 files changed, 92 insertions(+), 13 deletions(-) diff --git a/backend/src/CodeSpace.Core/Services/Agents/AgentRunExecutor.cs b/backend/src/CodeSpace.Core/Services/Agents/AgentRunExecutor.cs index f18d8da3c..e21bd5ce4 100644 --- a/backend/src/CodeSpace.Core/Services/Agents/AgentRunExecutor.cs +++ b/backend/src/CodeSpace.Core/Services/Agents/AgentRunExecutor.cs @@ -19,6 +19,7 @@ using CodeSpace.Core.Services.Workflows.Llm; using CodeSpace.Core.Services.Workflows.Planning.Planners; using CodeSpace.Core.Services.Agents.Sandbox; +using CodeSpace.Core.Services.Agents.Sandbox.Exceptions; using CodeSpace.Core.Services.Agents.Sandbox.Isolation; using CodeSpace.Core.Services.Agents.Sandbox.Runners; using CodeSpace.Core.Services.Agents.Tools; @@ -622,17 +623,32 @@ public async Task ExecuteAsync(Guid agentRunId, CancellationToken cancellationTo } } - var reviseSpec = BuildSpec(reviseTask); + // A revise goal restates the contract, so it can cross a harness's own input cap the original goal fit under + // (Codex's 1,048,576 characters) — warm, or once rebuilt cold below. That refusal is about this round, not + // the run: the last round ran and was graded, and its result stands — the loop stops the way a spend + // refusal stops it, with nothing to settle. + SandboxSpec reviseSpec; + bool roundRanCold; - // The same verdict for the round's own session: warm only when the pipe can carry it. Only the - // conversation and the goal change — a model escalation already applied to this round stands. - var roundRanCold = ContinuationOverflowsTheFrame(reviseTask, reviseSpec); - - if (roundRanCold) + try { - var cold = BuildReviseTask(effectiveTask, result, reason, mayResume: false); - reviseTask = reviseTask with { Goal = cold.Goal, ResumeFromSessionId = null, RestoredTranscript = null }; reviseSpec = BuildSpec(reviseTask); + + // The same verdict for the round's own session: warm only when the pipe can carry it. Only the + // conversation and the goal change — a model escalation already applied to this round stands. + roundRanCold = ContinuationOverflowsTheFrame(reviseTask, reviseSpec); + + if (roundRanCold) + { + var cold = BuildReviseTask(effectiveTask, result, reason, mayResume: false); + reviseTask = reviseTask with { Goal = cold.Goal, ResumeFromSessionId = null, RestoredTranscript = null }; + reviseSpec = BuildSpec(reviseTask); + } + } + catch (SandboxArgumentTooLongException refusal) + { + await AppendReviseStopEventAsync(owner, ReviseSizeStoppedPrefix, refusal.Message, cancellationToken).ConfigureAwait(false); + break; } var priorUsage = result.TokenUsage; @@ -648,7 +664,7 @@ public async Task ExecuteAsync(Guid agentRunId, CancellationToken cancellationTo if (spendClaim is { RefusedDetail: { } roundRefusal }) { spendClaim = null; - await AppendReviseBudgetStopEventAsync(owner, roundRefusal, cancellationToken).ConfigureAwait(false); + await AppendReviseStopEventAsync(owner, ReviseBudgetStoppedPrefix, roundRefusal, cancellationToken).ConfigureAwait(false); break; } @@ -2794,17 +2810,20 @@ private async Task AppendReviseStalledEventAsync(AgentRunOwnerToken owner, strin /// The budget-stopped-revision announcement's pinned prefix — the operator-visible marker that a round was never bought, not that it ran and failed. internal const string ReviseBudgetStoppedPrefix = "Revision stopped — the run's cost cap had no headroom for another attempt"; - /// Announce that the revise loop stopped because the ledger refused the NEXT invocation's claim. The round that already succeeded keeps its result: throwing away finished work because the following attempt cannot be afforded would lose what the operator already paid for. Best-effort, exactly like the stalled announcement above. - private async Task AppendReviseBudgetStopEventAsync(AgentRunOwnerToken owner, string detail, CancellationToken cancellationToken) + /// The timeline's account of a revision the harness refused to build for its size: the last graded round's result stands. + internal const string ReviseSizeStoppedPrefix = "Revision stopped — the revised instruction is past what this agent can be handed, so the last round's result stands"; + + /// Announce that the revise loop stopped before the NEXT round was bought — the ledger refused its claim, or the harness refused to build it for its size. The round that already ran keeps its result: throwing away finished work because the following attempt cannot be run would lose what the operator already paid for. Best-effort, exactly like the stalled announcement above. + private async Task AppendReviseStopEventAsync(AgentRunOwnerToken owner, string reason, string detail, CancellationToken cancellationToken) { var runId = owner.RunId; try { - await _runs.AppendEventAsync(owner, new AgentEvent { Kind = AgentEventKind.Warning, Text = $"{ReviseBudgetStoppedPrefix}. {detail}" }, cancellationToken).ConfigureAwait(false); + await _runs.AppendEventAsync(owner, new AgentEvent { Kind = AgentEventKind.Warning, Text = $"{reason}. {detail}" }, cancellationToken).ConfigureAwait(false); } catch (Exception ex) when (ex is not OperationCanceledException and not AgentRunOwnershipLostException) { - _logger.LogWarning(ex, "Agent run {RunId}: could not record the revise-budget-stop event", runId); + _logger.LogWarning(ex, "Agent run {RunId}: could not record the revise-stop event", runId); } } diff --git a/backend/tests/CodeSpace.IntegrationTests/Workflows/AgentRunReviseLoopFlowTests.cs b/backend/tests/CodeSpace.IntegrationTests/Workflows/AgentRunReviseLoopFlowTests.cs index 0bc4dbc43..ae90c67b2 100644 --- a/backend/tests/CodeSpace.IntegrationTests/Workflows/AgentRunReviseLoopFlowTests.cs +++ b/backend/tests/CodeSpace.IntegrationTests/Workflows/AgentRunReviseLoopFlowTests.cs @@ -4,6 +4,7 @@ using CodeSpace.Core.Persistence.Entities; using CodeSpace.Core.Services.Agents; using CodeSpace.Core.Services.Agents.Harnesses.Claude; +using CodeSpace.Core.Services.Agents.Harnesses.Codex; using CodeSpace.Core.Services.Agents.ModelCredentials; using CodeSpace.Core.Services.Agents.Sandbox; using CodeSpace.Core.Services.Agents.Sandbox.Runners; @@ -130,6 +131,38 @@ public async Task A_cold_revision_refused_for_spend_leaves_no_note_that_it_conti events.ShouldNotContain(AgentRunExecutor.ReviseRanColdNote, "no round ran, so no round continued as a fresh conversation"); } + [Fact] + public async Task A_revision_whose_restated_goal_crosses_codexs_input_cap_stops_revising_and_keeps_the_verified_round() + { + // A cold revise goal restates the whole contract, so it is longer than the goal round 0 ran under. When the + // goal sits just under Codex's own input cap, the revision crosses it and the harness refuses it at build. + // That refusal is about the revision, not the run: round 0 ran, pushed and was graded, and its verdict stands. + // Thrown out of the loop instead, it replaced round 0's whole result — diff, summary, verdict — with a bare + // size refusal, and marked the run as if nothing had run. + if (OperatingSystem.IsWindows()) return; + + var (teamId, userId) = await SeedTeamAsync(); + using var remote = new BareRemote(); + await remote.SeedBaseAsync(CheckScript); + var repoId = await SeedBoundRepositoryAsync(teamId, remote.Url); + var goal = new string('x', CodexHarness.MaxInputCharacters - 500); + var runId = await CreateRunAsync(teamId, userId, TaskWith(repoId) with { Goal = goal, MaxReviseRounds = 1 }); + var harness = new CodexSpecHarness(); + + await ExecuteAsync(runId, harness); + + var (run, result) = await LoadAsync(runId); + var events = await LoadEventsAsync(runId); + + harness.Goals.Count.ShouldBe(2, "fixture check: round 0 was built, and so was the revision the cap refused"); + harness.Goals[1].Length.ShouldBeGreaterThan(CodexHarness.MaxInputCharacters, "fixture check: the restated goal really crosses the cap"); + result.ExitReason.ShouldBe(AgentAcceptanceContract.FailClosedExitReason, $"round 0's verdict stands — the run ended {result.ExitReason}: {run.Error}"); + result.ChangedFiles.ShouldContain("feature.txt", "round 0's work is still the run's result"); + result.ReviseRounds.ShouldBe(0); + events.ShouldContain(e => e.StartsWith(AgentRunExecutor.ReviseSizeStoppedPrefix, StringComparison.Ordinal), "the timeline says why the revision stopped"); + (await remote.BranchFileContentAsync(AgentRunExecutor.BuildBranchName(runId), "feature.txt")).ShouldNotBeNull("and round 0's branch is still there"); + } + [Fact] public async Task A_revision_receives_the_real_oracle_diagnosis_instead_of_only_its_exit_code() { @@ -800,6 +833,33 @@ public SandboxSpec BuildInvocation(AgentTask task) public string? SessionTranscriptRelativePath(string configHome, string? workspaceDirectory, string? sessionId) => _real.SessionTranscriptRelativePath(configHome, workspaceDirectory, sessionId); } + /// A "scripted" harness whose spec is the REAL Codex adapter's — its own input cap included — with only the executable swapped; round 0 writes draft work, a revision the fixed work. + private sealed class CodexSpecHarness : IAgentHarness + { + private readonly CodexHarness _real = new(); + + public string Kind => "scripted"; + public string Version => "test"; + public IReadOnlyList Models { get; } = new[] { "test-model" }; + public List Goals { get; } = new(); + + public SandboxSpec BuildInvocation(AgentTask task) + { + Goals.Add(task.Goal); + var real = _real.BuildInvocation(task); + var revising = task.Goal.StartsWith(AgentRunExecutor.ReviseInstructionPrefix, StringComparison.Ordinal); + return real with { Command = "/bin/sh", Args = new[] { "-c", "cat >/dev/null; " + (revising ? RevisedScript : DraftScript) } }; + } + + public IReadOnlyList ParseEvents(string rawLine) => + string.IsNullOrWhiteSpace(rawLine) ? Array.Empty() : new[] { new AgentEvent { Kind = AgentEventKind.AssistantMessage, Text = rawLine.Trim() } }; + + public IAgentEventFolder CreateFolder() => new TestEventFolder((fold, exitCode) => + exitCode == 0 + ? new AgentRunResult { Status = AgentRunStatus.Succeeded, ExitReason = "completed", Summary = fold.LastText } + : new AgentRunResult { Status = AgentRunStatus.Failed, ExitReason = "non-zero-exit", Error = $"exit {exitCode}" }); + } + private sealed class ReviseAwareHarness : IAgentHarness { private readonly string _first;