diff --git a/backend/src/CodeSpace.Core/Services/Workflows/Llm/Anthropic/AnthropicClient.cs b/backend/src/CodeSpace.Core/Services/Workflows/Llm/Anthropic/AnthropicClient.cs index 73d3f5eb7..06811e146 100644 --- a/backend/src/CodeSpace.Core/Services/Workflows/Llm/Anthropic/AnthropicClient.cs +++ b/backend/src/CodeSpace.Core/Services/Workflows/Llm/Anthropic/AnthropicClient.cs @@ -14,8 +14,8 @@ namespace CodeSpace.Core.Services.Workflows.Llm.Anthropic; /// /// Implements (free-text completion) AND the sibling /// (schema-constrained JSON) — the latter via Anthropic's -/// forced tool-use: a single tool whose input_schema IS the requested schema, with -/// tool_choice pinned to it, so the model's tool_use block carries schema-valid JSON. +/// forced tool-use: a single tool whose input_schema is the request's WireJsonSchema ?? JsonSchema (every reply +/// is validated against JsonSchema), with tool_choice pinned to it, so the model's tool_use block carries schema-valid JSON. /// /// Keep ONLY the wire-shape concerns here. Anything node-facing (prompt assembly, output /// trimming, retry policy) belongs in the llm.complete node or the LLM-side resilience @@ -195,7 +195,7 @@ private async Task CompleteStructuredOnceAsync(Structur } /// - /// Attempt 1: forced tool-use — a single tool whose input_schema IS the schema, tool_choice pinned to it. Returns + /// Attempt 1: forced tool-use — a single tool whose input_schema is WireJsonSchema ?? JsonSchema (validation still uses JsonSchema), tool_choice pinned to it. Returns /// the recovered JSON (or null to degrade to the prompt-only floor) PLUS the parsed response — so the caller can /// accumulate the BILLED usage even when the JSON is null (a 200 that produced no usable tool call still cost /// tokens). On a 400/422 reject the request was never generated, so both are null (nothing to bill). @@ -213,7 +213,7 @@ private async Task CompleteStructuredOnceAsync(Structur StopSequences = request.Sampling?.Stop, System = system, Messages = messages, - Tools = new[] { new AnthropicTool { Name = StructuredToolName, Description = "Return the result as structured JSON.", InputSchema = request.WireJsonSchema ?? request.JsonSchema } }, + Tools = new[] { new AnthropicTool { Name = StructuredToolName, Description = "Return the result as structured JSON.", InputSchema = request.ProviderSchema } }, ToolChoice = new AnthropicToolChoice { Type = "tool", Name = StructuredToolName } }; diff --git a/backend/src/CodeSpace.Core/Services/Workflows/Llm/IStructuredLLMClient.cs b/backend/src/CodeSpace.Core/Services/Workflows/Llm/IStructuredLLMClient.cs index 8a99e6613..77a7ce1ae 100644 --- a/backend/src/CodeSpace.Core/Services/Workflows/Llm/IStructuredLLMClient.cs +++ b/backend/src/CodeSpace.Core/Services/Workflows/Llm/IStructuredLLMClient.cs @@ -21,7 +21,9 @@ public interface IStructuredLLMClient /// /// One LLM call constrained to return JSON matching . /// The provider impl maps the schema to whatever its API offers (Anthropic forces a single tool whose - /// input_schema IS the schema; OpenAI would use response_format json_schema, …). + /// input_schema carries the schema; OpenAI would use response_format json_schema, …). The provider receives + /// when the request sets one, else ; + /// every reply is validated against either way. /// Task CompleteStructuredAsync(StructuredLLMCompletionRequest request, CancellationToken cancellationToken); } @@ -45,6 +47,9 @@ public sealed record StructuredLLMCompletionRequest /// public JsonElement? WireJsonSchema { get; init; } + /// The schema the provider is actually handed: when set, else . A default wire schema (kind Undefined) counts as unset — a nullable struct admits it as a non-null value, and it cannot be serialized. + internal JsonElement ProviderSchema => WireJsonSchema is { ValueKind: not JsonValueKind.Undefined } wire ? wire : JsonSchema; + /// Server-only validation of the consumer contract, alongside JSON schema. Violations enter the same bounded model re-ask; this callback never rewrites output. [JsonIgnore] public Func>? ResponseValidator { get; init; } diff --git a/backend/src/CodeSpace.Core/Services/Workflows/Llm/OpenAi/OpenAiClient.cs b/backend/src/CodeSpace.Core/Services/Workflows/Llm/OpenAi/OpenAiClient.cs index e0d8dd180..c9346a9c8 100644 --- a/backend/src/CodeSpace.Core/Services/Workflows/Llm/OpenAi/OpenAiClient.cs +++ b/backend/src/CodeSpace.Core/Services/Workflows/Llm/OpenAi/OpenAiClient.cs @@ -18,8 +18,8 @@ namespace CodeSpace.Core.Services.Workflows.Llm.OpenAi; /// post-S6b) — a call without a credential fails closed. /// /// Implements (free-text) AND (schema-constrained -/// JSON). Structured output uses FORCED FUNCTION-CALLING — a single function whose parameters IS the -/// requested schema, with tool_choice pinned to it — rather than response_format: json_schema, +/// JSON). Structured output uses FORCED FUNCTION-CALLING — a single function whose parameters is the request's +/// WireJsonSchema ?? JsonSchema (every reply is validated against JsonSchema), with tool_choice pinned to it — rather than response_format: json_schema, /// because function-calling is supported by far more OpenAI-compatible gateways than the newer structured-outputs /// feature, and it mirrors the Anthropic client's forced-tool design exactly (one coercion mechanism to reason /// about). The model's tool_calls[0].function.arguments is the schema-SHAPED JSON (classic function-calling @@ -229,7 +229,7 @@ private async Task CompleteStructuredOnceAsync(Structur } /// - /// Attempt 1: forced function-calling — a single function whose parameters IS the schema, tool_choice pinned to it. + /// Attempt 1: forced function-calling — a single function whose parameters is WireJsonSchema ?? JsonSchema (validation still uses JsonSchema), tool_choice pinned to it. /// Returns the recovered JSON (or null to degrade to the prompt-only floor) PLUS the parsed response — so the caller /// can accumulate the BILLED usage even when the JSON is null (a 200 that produced no usable function call still /// cost tokens). On a 400/422 reject the request was never generated, so both are null (nothing to bill). @@ -254,7 +254,7 @@ private async Task CompleteStructuredOnceAsync(Structur Stop = request.Sampling?.Stop, ReasoningEffort = LlmModelCapabilities.SupportsReasoningEffort(request.Model) ? request.ReasoningEffort : null, // sent ONLY to a reasoning model (a plain chat model 400s on it); the value rides verbatim (the API validates it per model) Messages = BuildMessages(system, request.UserPrompt), - Tools = new[] { new OpenAiTool { Function = new OpenAiFunction { Name = StructuredToolName, Description = "Return the result as structured JSON.", Parameters = request.WireJsonSchema ?? request.JsonSchema } } }, + Tools = new[] { new OpenAiTool { Function = new OpenAiFunction { Name = StructuredToolName, Description = "Return the result as structured JSON.", Parameters = request.ProviderSchema } } }, ToolChoice = new OpenAiToolChoice { Function = new OpenAiToolChoiceFunction { Name = StructuredToolName } }, }; diff --git a/backend/src/CodeSpace.Core/Services/Workflows/Nodes/Builtin/AgentSupervisorNode.cs b/backend/src/CodeSpace.Core/Services/Workflows/Nodes/Builtin/AgentSupervisorNode.cs index 79a0ad1b8..e6e488bf8 100644 --- a/backend/src/CodeSpace.Core/Services/Workflows/Nodes/Builtin/AgentSupervisorNode.cs +++ b/backend/src/CodeSpace.Core/Services/Workflows/Nodes/Builtin/AgentSupervisorNode.cs @@ -230,9 +230,10 @@ private async Task ParkForInfraOrStopAsync(NodeRunContext context, G } var delay = SupervisorInfraPark.DelayFor(state.Parks); - var marker = SupervisorInfraPark.Marker(state, fault.Message); + var said = InfraPark.FaultText(context.Scope, fault); + var marker = SupervisorInfraPark.Marker(state, said); - context.Logger.LogWarning("agent.supervisor run {RunId}: brain call hit a {Category} infra fault — parking {Delay} (park {Parks} since {First:o}) instead of failing the run", supervisorRunId, fault.Category, delay, state.Parks, state.FirstParkedAtUtc); + context.Logger.LogWarning("agent.supervisor run {RunId}: brain call hit a {Category} infra fault — parking {Delay} (park {Parks} since {First:o}) instead of failing the run: {Fault}", supervisorRunId, fault.Category, delay, state.Parks, state.FirstParkedAtUtc, said); return NodeResult.Suspend(new SuspensionToken { diff --git a/backend/src/CodeSpace.Core/Services/Workflows/Nodes/InfraPark.cs b/backend/src/CodeSpace.Core/Services/Workflows/Nodes/InfraPark.cs index 2c288f0c5..0d9a50f57 100644 --- a/backend/src/CodeSpace.Core/Services/Workflows/Nodes/InfraPark.cs +++ b/backend/src/CodeSpace.Core/Services/Workflows/Nodes/InfraPark.cs @@ -1,5 +1,6 @@ using CodeSpace.Core.Services.Supervisor; using CodeSpace.Core.Services.Workflows.Llm; +using CodeSpace.Core.Services.Workflows.Runtime; using CodeSpace.Messages.Constants; using Microsoft.Extensions.Logging; @@ -43,18 +44,19 @@ public static class InfraPark public static NodeResult Park(NodeRunContext context, LlmApiException fault, DateTimeOffset now) { var state = SupervisorInfraPark.Next(context.ResumePayload, now); + var said = FaultText(context.Scope, fault); if (state.WindowExhausted) { context.Logger.LogWarning("Node {NodeId}: the model plane stayed unavailable past the whole {Window} park window — failing the node honestly", context.NodeId, SupervisorInfraPark.MaxParkWindow); - return NodeResult.Fail($"The model plane stayed unavailable for {SupervisorInfraPark.MaxParkWindow.TotalHours:0}h: {fault.Message}", retryable: false); + return NodeResult.Fail($"The model plane stayed unavailable for {SupervisorInfraPark.MaxParkWindow.TotalHours:0}h: {said}", retryable: false); } var delay = SupervisorInfraPark.DelayFor(state.Parks); - var marker = SupervisorInfraPark.Marker(state, fault.Message); + var marker = SupervisorInfraPark.Marker(state, said); - context.Logger.LogWarning("Node {NodeId}: model call hit a {Category} infra fault — parking {Delay} (park {Parks} since {First:o}) instead of failing the run: {Fault}", context.NodeId, fault.Category, delay, state.Parks, state.FirstParkedAtUtc, fault.Message); + context.Logger.LogWarning("Node {NodeId}: model call hit a {Category} infra fault — parking {Delay} (park {Parks} since {First:o}) instead of failing the run: {Fault}", context.NodeId, fault.Category, delay, state.Parks, state.FirstParkedAtUtc, said); return NodeResult.Suspend(new SuspensionToken { @@ -68,4 +70,31 @@ public static NodeResult Park(NodeRunContext context, LlmApiException fault, Dat TimeoutPayload = marker, }); } + + /// The most characters of a fault's text a park writes down. The gateway's own words are what a reader needs; the provider's whole error body, which ends with, is not. + internal const int MaxFaultChars = 512; + + /// + /// The fault's own words in the form everything that OUTLIVES the call may hold — the park marker (durable, and what the + /// run detail shows while parked), the honest failure text and the park log all take this, never the raw + /// , which ends with the provider's error body verbatim and unbounded. Scope secrets are + /// redacted FIRST and the result is clamped to after: a clamp that ran first could cut a + /// secret in half and leave a fragment no redactor would recognise. + /// + internal static string FaultText(NodeRunScope scope, LlmApiException fault) + { + var redacted = PersistenceSecretRedactor.FromScope(scope).Redact(fault.Message).Value ?? PersistenceSecretRedactor.Marker; + + return Clamp(redacted, MaxFaultChars); + } + + /// The first characters plus an ellipsis, never ending inside a surrogate pair — a lone half is ill-formed text that Npgsql's strict UTF-8 encoder refuses to write. + private static string Clamp(string text, int max) + { + if (text.Length <= max) return text; + + var cut = char.IsHighSurrogate(text[max - 1]) ? max - 1 : max; + + return text[..cut] + "…"; + } } diff --git a/backend/src/CodeSpace.Core/Services/Workflows/Planning/PlannerSchema.cs b/backend/src/CodeSpace.Core/Services/Workflows/Planning/PlannerSchema.cs index 3ffa655d4..333a44e2d 100644 --- a/backend/src/CodeSpace.Core/Services/Workflows/Planning/PlannerSchema.cs +++ b/backend/src/CodeSpace.Core/Services/Workflows/Planning/PlannerSchema.cs @@ -5,8 +5,8 @@ namespace CodeSpace.Core.Services.Workflows.Planning; /// -/// The planner's COMMIT-CONTRACT: the JSON Schema the model is constrained to (via the structured-output -/// path) and the matching deserialization options. Co-located with the planner concern (Rule 18) and +/// The planner's COMMIT-CONTRACT: the JSON Schema every reply is VALIDATED against (and the prompt quotes) and the +/// matching deserialization options; the provider itself is handed the combinator-free instead. Co-located with the planner concern (Rule 18) and /// pinned by a unit test — a drift in either the schema or the property mapping is a contract change a /// reviewer must see, not an invisible refactor. /// @@ -18,7 +18,7 @@ namespace CodeSpace.Core.Services.Workflows.Planning; /// public static class PlannerSchema { - /// The JSON schema constraining fresh model output. PlannerAcceptanceDraft maps its typed acceptance payloads before the normalized PlannedWorkflow reaches persistence or execution. + /// The JSON schema every fresh model reply is VALIDATED against and the prompt quotes; the provider is handed , not this. PlannerAcceptanceDraft maps its typed acceptance payloads before the normalized PlannedWorkflow reaches persistence or execution. public static readonly JsonElement ResponseSchema = JsonDocument.Parse(""" { "type": "object", diff --git a/backend/src/CodeSpace.Core/Services/Workflows/Planning/Planners/LlmWorkflowPlanner.cs b/backend/src/CodeSpace.Core/Services/Workflows/Planning/Planners/LlmWorkflowPlanner.cs index 2f52a06e2..edd5b3ed0 100644 --- a/backend/src/CodeSpace.Core/Services/Workflows/Planning/Planners/LlmWorkflowPlanner.cs +++ b/backend/src/CodeSpace.Core/Services/Workflows/Planning/Planners/LlmWorkflowPlanner.cs @@ -13,8 +13,8 @@ namespace CodeSpace.Core.Services.Workflows.Planning.Planners; /// /// The structured-LLM (Rule 18.3 — an impl in the Planners/ variant /// folder). It resolves a structured-capable LLM client through the SAME -/// the llm.complete node uses, sends a system+user prompt constrained by -/// , and deserializes the schema-valid object into a +/// the llm.complete node uses, sends a system+user prompt whose reply is validated against +/// (the provider is handed the combinator-free ), and deserializes the schema-valid object into a /// . Fails cleanly when no registered provider offers structured output. /// /// The planner produces DATA only — it never wires nodes or runs anything. The grounding context diff --git a/backend/tests/CodeSpace.IntegrationTests/Workflows/Supervisor/InfraParkRideTests.cs b/backend/tests/CodeSpace.IntegrationTests/Workflows/Supervisor/InfraParkRideTests.cs index 5bfe3c280..50a225d19 100644 --- a/backend/tests/CodeSpace.IntegrationTests/Workflows/Supervisor/InfraParkRideTests.cs +++ b/backend/tests/CodeSpace.IntegrationTests/Workflows/Supervisor/InfraParkRideTests.cs @@ -1,6 +1,11 @@ +using System.Text.Json; using CodeSpace.Core.Services.Supervisor; +using CodeSpace.Core.Services.Workflows.Llm; +using CodeSpace.Core.Services.Workflows.Nodes; +using CodeSpace.Core.Services.Workflows.Runtime; using CodeSpace.Messages.Constants; using CodeSpace.Messages.Enums; +using Microsoft.Extensions.Logging.Abstractions; using Shouldly; namespace CodeSpace.IntegrationTests.Workflows.Supervisor; @@ -137,6 +142,21 @@ public async Task An_unresolved_park_names_the_fault_the_park_stored_on_its_mark ex.Message.ShouldContain("Anthropic API error (HTTP 500, Transient): Hosted_vllmException", Case.Sensitive, "the infra skip must carry the gateway's own words, not only the park's duration"); } + [Fact] + public async Task An_unresolved_park_quotes_the_clamped_text_the_production_park_stored() + { + // The park clamps the gateway's words before it stores them, so a provider that answers with a whole page no + // longer floods the job summary: the skip quotes the first 512 characters and stops. The marker is minted by the + // production park (InfraPark.Park) and read back by the ride's own reader, not built by hand. + var fault = new LlmApiException("Anthropic", 500, LlmErrorCategory.Transient, new string('x', 5_000)); + var park = InfraPark.Park(ParkingContext(), fault, DateTimeOffset.UtcNow); + var cell = Parked() with { WaitPayloadJson = park.SuspendUntil!.Payload.GetRawText() }; + + var ex = await Should.ThrowAsync(() => InfraParkRide.RideAsync(() => Task.FromResult(cell), _ => Task.CompletedTask, maxWakes: 1, wakePause: TimeSpan.Zero)); + + ex.Message.ShouldEndWith("The park's last fault: " + fault.Message[..512] + "…", Case.Sensitive); + } + [Fact] public void The_ride_pauses_for_real_between_wakes_so_a_recovery_can_actually_be_observed() { @@ -149,4 +169,16 @@ public void The_ride_pauses_for_real_between_wakes_so_a_recovery_can_actually_be private static ParkedCell Parked() => new() { CellStatus = NodeStatus.Suspended, PendingWaitKind = WorkflowWaitKinds.SupervisorInfraPark, NodeId = "planner" }; private static ParkedCell Settled() => new() { CellStatus = NodeStatus.Success, PendingWaitKind = null, NodeId = "planner" }; + + private static NodeRunContext ParkingContext() => new() + { + Inputs = new Dictionary(), + Config = new Dictionary(), + RawInputs = JsonDocument.Parse("{}").RootElement, + RawConfig = JsonDocument.Parse("{}").RootElement, + Scope = new NodeRunScope { Trigger = new Dictionary(), Sys = new Dictionary() }, + Logger = NullLogger.Instance, + Observability = NodeObservability.NoOp, + NodeId = "planner", + }; } diff --git a/backend/tests/CodeSpace.UnitTests/Workflows/AgentSupervisorNodeInfraParkTests.cs b/backend/tests/CodeSpace.UnitTests/Workflows/AgentSupervisorNodeInfraParkTests.cs new file mode 100644 index 000000000..7190e0912 --- /dev/null +++ b/backend/tests/CodeSpace.UnitTests/Workflows/AgentSupervisorNodeInfraParkTests.cs @@ -0,0 +1,97 @@ +using System.Text.Json; +using CodeSpace.Core.Services.Supervisor; +using CodeSpace.Core.Services.Workflows.Llm; +using CodeSpace.Core.Services.Workflows.Nodes; +using CodeSpace.Core.Services.Workflows.Nodes.Builtin; +using CodeSpace.Core.Services.Workflows.Runtime; +using CodeSpace.Messages.Agents; +using CodeSpace.Messages.Constants; +using CodeSpace.Messages.Dtos.Agents; +using CodeSpace.Messages.Enums; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Shouldly; + +namespace CodeSpace.UnitTests.Workflows; + +/// +/// 🟢 Unit (the real over a turn service whose turn faults): the supervisor's own +/// model-plane park writes the SAME redacted, clamped fault text the generic does — on its +/// marker AND in its park log line. The supervisor is the one lane that parks through its own code, and it stored the +/// raw fault.Message (the provider's whole error body, unredacted and unbounded) while logging no fault text. +/// +[Trait("Category", "Unit")] +public sealed class AgentSupervisorNodeInfraParkTests +{ + /// A value the run treats as secret — a team variable, listed on SecretPaths under the engine's own spelling (<bucket>.<variable>). + private const string GatewayToken = "gw-tok-7f3a9c2e41d8"; + + [Fact] + public async Task The_supervisors_park_writes_the_redacted_clamped_fault_text_on_its_marker_and_in_its_log() + { + var fault = new LlmApiException("Anthropic", 500, LlmErrorCategory.Transient, $"gateway rejected bearer {GatewayToken}: " + new string('x', 5_000)); + fault.Message.ShouldContain(GatewayToken, Case.Sensitive, "fixture check: the raw fault carries the secret, or the redaction asserted below proves nothing"); + var logger = new CapturingLogger(); + + var result = await new AgentSupervisorNode(ScopeFactoryOver(new FaultingTurns(fault))).RunAsync(ContextHoldingSecret(logger), CancellationToken.None); + + result.Status.ShouldBe(NodeStatus.Suspended, "an exhausted transient fault parks the supervisor — it must never terminalize the run"); + result.SuspendUntil!.Kind.ShouldBe(WorkflowWaitKinds.SupervisorInfraPark); + + var marker = result.SuspendUntil.Payload.GetProperty("error").GetString()!; + marker.ShouldNotContain(GatewayToken, Case.Sensitive, "the marker is durable and shown on the run detail while the supervisor is parked"); + marker.ShouldContain(PersistenceSecretRedactor.Marker, Case.Sensitive); + marker.Length.ShouldBe(513, "512 characters of the fault, then the ellipsis"); + marker.ShouldEndWith("…", Case.Sensitive); + + logger.Warnings.ShouldHaveSingleItem().ShouldEndWith(marker, Case.Sensitive, "the park line carries the fault text — the SAME text the marker stores, redacted and clamped alike"); + } + + private static IServiceScopeFactory ScopeFactoryOver(ISupervisorTurnService turns) => new ServiceCollection().AddSingleton(turns).BuildServiceProvider().GetRequiredService(); + + private static NodeRunContext ContextHoldingSecret(ILogger logger) => new() + { + Inputs = new Dictionary(), + Config = new Dictionary(), + RawInputs = JsonDocument.Parse("{}").RootElement, + RawConfig = JsonDocument.Parse("{}").RootElement, + Scope = new NodeRunScope + { + Trigger = new Dictionary(), + Sys = new Dictionary + { + [SystemScopeKeys.WorkflowRunId] = JsonSerializer.SerializeToElement(Guid.NewGuid()), + [SystemScopeKeys.TeamId] = JsonSerializer.SerializeToElement(Guid.NewGuid()), + }, + Team = new Dictionary { ["GATEWAY_TOKEN"] = JsonSerializer.SerializeToElement(GatewayToken) }, + SecretPaths = new HashSet { "team.GATEWAY_TOKEN" }, + }, + Logger = logger, + Observability = NodeObservability.NoOp, + NodeId = "supervisor", + }; + + /// A turn service with nothing pending whose turn faults — the state a supervisor re-enters into while the model plane is down. + private sealed class FaultingTurns(LlmApiException fault) : ISupervisorTurnService + { + public Task RehydrateFromDecisionLogAsync(Guid supervisorRunId, Guid teamId, string nodeId, string goal, SupervisorGoalConfig? goalConfig, CancellationToken cancellationToken) => throw new NotSupportedException(); + public Task RunTurnAsync(Guid supervisorRunId, Guid teamId, string nodeId, string goal, Guid? conversationId, SupervisorGoalConfig? goalConfig, CancellationToken cancellationToken) => Task.FromException(fault); + public Task CountPendingAgentWaitsAsync(Guid supervisorRunId, string nodeId, CancellationToken cancellationToken) => Task.FromResult(0); + public Task PendingHumanWaitTokenAsync(Guid supervisorRunId, string nodeId, CancellationToken cancellationToken) => Task.FromResult(null); + public Task ReopenDiscardedAskAsync(Guid supervisorRunId, string nodeId, CancellationToken cancellationToken) => Task.FromResult(false); + public Task ForceStopAsync(Guid supervisorRunId, Guid teamId, string nodeId, string goal, SupervisorGoalConfig? goalConfig, string reason, CancellationToken cancellationToken) => throw new NotSupportedException(); + public Task SupervisorDepthAsync(Guid supervisorRunId, Guid teamId, CancellationToken cancellationToken) => throw new NotSupportedException(); + } + + /// Keeps every warning the node writes, formatted the way a sink would render it. + private sealed class CapturingLogger : ILogger + { + public List Warnings { get; } = []; + + public IDisposable BeginScope(TState state) where TState : notnull => NullScope.Instance; + public bool IsEnabled(LogLevel logLevel) => true; + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) { if (logLevel == LogLevel.Warning) Warnings.Add(formatter(state, exception)); } + + private sealed class NullScope : IDisposable { public static readonly NullScope Instance = new(); public void Dispose() { } } + } +} diff --git a/backend/tests/CodeSpace.UnitTests/Workflows/InfraParkTests.cs b/backend/tests/CodeSpace.UnitTests/Workflows/InfraParkTests.cs index aeb472fa9..078623d71 100644 --- a/backend/tests/CodeSpace.UnitTests/Workflows/InfraParkTests.cs +++ b/backend/tests/CodeSpace.UnitTests/Workflows/InfraParkTests.cs @@ -17,8 +17,9 @@ namespace CodeSpace.UnitTests.Workflows; /// blip a Deep run sleeps through killed a Standard run in minutes. /// /// What these pin: the fault classes worth parking for (and the ones that must stay fail-fast), the ladder -/// continuing across wakes from its OWN marker only, the honest failure once the window is spent, and the -/// iteration-key choice that keeps a parked map-branch node inside its own branch cell. +/// continuing across wakes from its OWN marker only, the honest failure once the window is spent, the +/// iteration-key choice that keeps a parked map-branch node inside its own branch cell, and the text the park +/// writes down about the fault — redacted of scope secrets first, then clamped. /// [Trait("Category", "Unit")] public class InfraParkTests @@ -38,6 +39,39 @@ public class InfraParkTests private static LlmApiException Fault(LlmErrorCategory category) => new("Anthropic", 503, category, "upstream unavailable"); + private static LlmApiException FaultWith(string providerMessage) => new("Anthropic", 500, LlmErrorCategory.Transient, providerMessage); + + /// A value the run treats as secret — a team variable, listed on SecretPaths under the engine's own spelling (<bucket>.<variable>). + private const string GatewayToken = "gw-tok-7f3a9c2e41d8"; + + private static NodeRunContext ContextHoldingSecret() => Context() with + { + Scope = new NodeRunScope + { + Trigger = new Dictionary(), + Sys = new Dictionary(), + Team = new Dictionary { ["GATEWAY_TOKEN"] = JsonSerializer.SerializeToElement(GatewayToken) }, + SecretPaths = new HashSet { "team.GATEWAY_TOKEN" }, + }, + }; + + /// The fault's text as each place the park writes it shows it: the marker's error on a park, that park's log line, and the honest failure once the whole window is spent. + private static Dictionary TextsWritten(NodeRunContext context, LlmApiException fault) + { + var logger = new CapturingLogger(); + var now = DateTimeOffset.UtcNow; + + var parked = InfraPark.Park(context with { Logger = logger }, fault, now); + var failed = InfraPark.Park(context with { ResumePayload = parked.SuspendUntil!.TimeoutPayload }, fault, now + SupervisorInfraPark.MaxParkWindow); + + return new Dictionary + { + ["marker"] = parked.SuspendUntil.Payload.GetProperty("error").GetString()!, + ["log"] = logger.Messages.ShouldHaveSingleItem(), + ["failure"] = failed.Error!, + }; + } + // ── Which faults park, and which must never ────────────────────────────────────── [Theory] @@ -76,6 +110,68 @@ public void The_park_log_carries_the_faults_own_words() logger.Messages.ShouldHaveSingleItem().ShouldContain("upstream unavailable"); } + // ── What the park writes down about the fault ──────────────────────────────────── + + [Fact] + public void A_scope_secret_in_the_faults_text_is_redacted_from_the_marker_the_log_and_the_failure() + { + // LlmApiException.Message ends with the provider's error body verbatim, and a gateway that rejects a request can + // echo the credential it was sent. The marker is durable and shown on the run detail, the failure is the run's + // own error and the log line leaves the process — none of the three may carry a value the run treats as secret. + var fault = FaultWith($"gateway rejected bearer {GatewayToken} on /v1/messages"); + fault.Message.ShouldContain(GatewayToken, Case.Sensitive, "fixture check: the raw fault carries the secret, or the redaction asserted below proves nothing"); + + foreach (var (place, text) in TextsWritten(ContextHoldingSecret(), fault)) + { + text.ShouldNotContain(GatewayToken, Case.Sensitive, $"the {place} must never carry a scope secret"); + text.ShouldContain(PersistenceSecretRedactor.Marker, Case.Sensitive, $"the {place} says a value was withheld rather than silently losing words"); + } + } + + [Fact] + public void A_provider_body_of_thousands_of_characters_is_clamped_in_the_marker_the_log_and_the_failure() + { + // The same Message tail carries a gateway's WHOLE answer — an HTML page, a stack trace — onto the run row and + // into the log line. 512 characters of it and the ellipsis is what a reader needs; the rest is noise at rest. + var fault = FaultWith(new string('x', 5_000)); + var clamped = fault.Message[..512] + "…"; + + var written = TextsWritten(Context(), fault); + + written["marker"].ShouldBe(clamped); + written["log"].ShouldEndWith(clamped, Case.Sensitive); + written["failure"].ShouldEndWith(clamped, Case.Sensitive); + } + + [Fact] + public void A_secret_that_straddles_the_clamp_is_redacted_whole_and_never_left_as_a_fragment() + { + // Redaction runs BEFORE the clamp. Clamped first, the cut would land inside the token and leave its opening + // characters behind — a fragment no exact-value redactor would ever recognise. + var fault = FaultWith(new string('x', 512 - FaultWith("").Message.Length - 4) + GatewayToken + new string('y', 200)); + fault.Message.IndexOf(GatewayToken, StringComparison.Ordinal).ShouldBe(508, "fixture check: the token starts four characters before the 512-character cut"); + + foreach (var (place, text) in TextsWritten(ContextHoldingSecret(), fault)) + text.ShouldNotContain(GatewayToken[..4], Case.Sensitive, $"the {place} kept the opening of a secret that the clamp cut in half"); + } + + [Fact] + public void The_clamp_never_cuts_a_surrogate_pair_in_half() + { + // A lone surrogate is ill-formed UTF-16. System.Text.Json quietly swaps it for U+FFFD on the marker, but the + // failure text reaches the run row through Npgsql, whose strict UTF-8 encoder throws on it — so an emoji that + // straddles the cut goes whole or not at all. + var fault = FaultWith(new string('x', 511 - FaultWith("").Message.Length) + "\U0001F600tail"); + char.IsHighSurrogate(fault.Message[511]).ShouldBeTrue("fixture check: the 512-character cut would fall between the emoji's two halves"); + var clamped = fault.Message[..511] + "…"; + + var written = TextsWritten(Context(), fault); + + written["marker"].ShouldBe(clamped); + written["log"].ShouldEndWith(clamped, Case.Sensitive); + written["failure"].ShouldEndWith(clamped, Case.Sensitive); + } + [Fact] public void The_park_keeps_the_nodes_ambient_cell_so_a_map_branch_stays_in_its_branch() { diff --git a/backend/tests/CodeSpace.UnitTests/Workflows/JsonSchemaCombinatorsTests.cs b/backend/tests/CodeSpace.UnitTests/Workflows/JsonSchemaCombinatorsTests.cs index e5ed6eabf..2cc47646f 100644 --- a/backend/tests/CodeSpace.UnitTests/Workflows/JsonSchemaCombinatorsTests.cs +++ b/backend/tests/CodeSpace.UnitTests/Workflows/JsonSchemaCombinatorsTests.cs @@ -16,10 +16,11 @@ namespace CodeSpace.UnitTests.Workflows; /// /// Pins the combinator-free WIRE form of a structured-output schema: what -/// removes and what it must leave alone, and the portability rule that every schema the code hands a model reaches the -/// provider with no combinator at any depth. A hosted vLLM backend compiles that schema into a decoding grammar, and a -/// combinator shape it cannot compile costs the whole call an empty HTTP 500 — so the rule is pinned per schema rather -/// than rediscovered per outage. +/// removes and what it must leave alone, and the portability rules that every schema the code hands a model reaches the +/// provider with no combinator at any depth and with every array typed by its items. A hosted vLLM backend compiles +/// that schema into a decoding grammar, and a shape it cannot compile costs the whole call an empty HTTP 500 — so the +/// rules are pinned per schema rather than rediscovered per outage. The 500 named no keyword, and the schema that drew +/// it carried both a combinator and an items-less array, so both are guarded. /// [Trait("Category", "Unit")] public sealed class JsonSchemaCombinatorsTests @@ -60,10 +61,22 @@ public sealed class JsonSchemaCombinatorsTests public void Every_schema_the_code_sends_to_a_model_is_combinator_free_on_the_wire(string schema) => CombinatorPaths(WireForms[schema]).ShouldBeEmpty($"{schema} reaches the provider's constrained decoder; carry the combinator in a validation-only JsonSchema and send JsonSchemaCombinators.Strip of it as the WireJsonSchema"); + [Theory] + [MemberData(nameof(WireFormNames))] + public void Every_array_in_a_schema_the_code_sends_to_a_model_declares_its_items(string schema) + { + // The schema that drew the empty 500 carried an array with no `items` (rubric.criteria, inside a per-kind branch) + // beside its combinators, and the error names no keyword — so this second candidate is guarded too. A decoding + // grammar has no element type to build for an array that does not say what it holds. + var untyped = ArraysWithoutItems(WireForms[schema]); + + untyped.ShouldBeEmpty($"{schema} declares an array with no `items` at {string.Join(", ", untyped)}; declare the element schema, because the provider's grammar compiler needs one"); + } + [Fact] public void Every_schema_constant_in_core_is_either_sent_on_the_wire_or_only_validates() { - // The guard above is only as good as its list. Every schema a model is handed today is a static *Schema field, + // The guards above are only as good as their list. Every schema a model is handed today is a static *Schema field, // so a new one lands here and must be sorted: sent to a provider (combinator-free) or validation-only. var declared = typeof(PlannerSchema).Assembly.GetTypes() .SelectMany(type => type.GetFields(BindingFlags.Static | BindingFlags.Public | BindingFlags.NonPublic).Where(field => field.FieldType == typeof(JsonElement) && field.Name.EndsWith("Schema", StringComparison.Ordinal)).Select(field => $"{type.Name}.{field.Name}")); @@ -113,6 +126,44 @@ public void Strip_keeps_property_names_and_literal_values_that_happen_to_spell_a _ => [], }; + /// Every path at which an array-typed schema node declares no items, at ANY depth. Blind to what a key means, like , so it is stricter than it needs to be: any object whose type names array must say what its elements are. + private static IReadOnlyList ArraysWithoutItems(JsonElement node, string path = "$") => node.ValueKind switch + { + JsonValueKind.Object => (DeclaresArray(node) && !node.TryGetProperty("items", out _) ? [path] : Array.Empty()).Concat(node.EnumerateObject().SelectMany(property => ArraysWithoutItems(property.Value, $"{path}.{property.Name}"))).ToArray(), + JsonValueKind.Array => node.EnumerateArray().SelectMany((item, index) => ArraysWithoutItems(item, $"{path}[{index}]")).ToArray(), + _ => [], + }; + + /// Whether the node's type is, or lists, array. + private static bool DeclaresArray(JsonElement node) => node.TryGetProperty("type", out var type) && type.ValueKind switch + { + JsonValueKind.String => type.GetString() == "array", + JsonValueKind.Array => type.EnumerateArray().Any(entry => entry.ValueKind == JsonValueKind.String && entry.GetString() == "array"), + _ => false, + }; + + [Fact] + public void ArraysWithoutItems_names_every_array_typed_node_that_declares_no_items_and_only_those() + { + // The guard is only as good as its walker: it must find a bare array at any depth and in either spelling of + // `type`, and must not flag a declared one — nor a property that merely happens to be NAMED "type". + const string schema = """ + { + "type": "object", + "properties": { + "declared": { "type": "array", "items": { "type": "string" } }, + "bare": { "type": "array", "minItems": 1 }, + "nullable": { "type": ["null", "array"] }, + "nested": { "type": "object", "properties": { "inner": { "type": "array" } } }, + "listOfLists": { "type": "array", "items": { "type": "array" } }, + "type": { "type": "string" } + } + } + """; + + ArraysWithoutItems(JsonDocument.Parse(schema).RootElement).ShouldBe(new[] { "$.properties.bare", "$.properties.nullable", "$.properties.nested.properties.inner", "$.properties.listOfLists.items" }, ignoreOrder: true); + } + /// One schema with beside a sibling at every position a subschema can sit — or, for a null keyword, the same schema written without it. private static string SchemaAtEveryPosition(string? keyword) { diff --git a/backend/tests/CodeSpace.UnitTests/Workflows/StructuredResponseContractTests.cs b/backend/tests/CodeSpace.UnitTests/Workflows/StructuredResponseContractTests.cs index 0354e3cf1..a5e464ec8 100644 --- a/backend/tests/CodeSpace.UnitTests/Workflows/StructuredResponseContractTests.cs +++ b/backend/tests/CodeSpace.UnitTests/Workflows/StructuredResponseContractTests.cs @@ -1,11 +1,13 @@ using System.Net; using System.Text; using System.Text.Json; +using System.Text.Json.Nodes; using CodeSpace.Core.Services.Workflows.Llm; using CodeSpace.Core.Services.Workflows.Llm.Anthropic; using CodeSpace.Core.Services.Workflows.Llm.OpenAi; using CodeSpace.Messages.Agents; using CodeSpace.Core.Services.Agents.ModelCredentials; +using CodeSpace.Core.Services.Workflows.Planning; using CodeSpace.Core.Services.Workflows.Planning.Planners; using CodeSpace.Messages.Dtos.Workflows.Planning; using Shouldly; @@ -409,7 +411,7 @@ public async Task The_reask_preamble_calls_a_fatal_miss_invalid_and_never_says_t [Theory] [InlineData("Anthropic")] [InlineData("OpenAI")] - public async Task The_planner_request_sends_the_provider_a_combinator_free_schema_that_still_declares_every_acceptance_field(string provider) + public async Task The_planner_request_sends_the_provider_a_combinator_free_schema_and_its_prompt_still_quotes_the_full_one(string provider) { // A hosted vLLM backend compiles the forced tool's schema into its decoding grammar. The per-kind oneOf // branches (#1854) broke that compile: every planner call came back an EMPTY HTTP 500, the planner parked, @@ -422,6 +424,13 @@ public async Task The_planner_request_sends_the_provider_a_combinator_free_schem var sent = SentToolSchema(provider, handler.Bodies[0]); JsonSchemaCombinatorsTests.CombinatorPaths(sent).ShouldBeEmpty("the provider's constrained decoder must never be handed a combinator"); + // The tool schema lost its per-kind branches, so the SYSTEM prompt — which quotes the FULL contract schema — is + // now the only place the model reads which payload each oracle kind requires. Handing the prompt the wire form + // too would leave the model no statement of those requirements at all, and nothing else would notice. + var system = SentSystemText(provider, handler.Bodies[0]); + system.ShouldContain("\"oneOf\"", Case.Sensitive, "the prompt is where the model learns the per-kind acceptance requirements now that the tool schema has no branches"); + system.ShouldContain(PlannerSchema.ResponseSchema.GetRawText(), Case.Sensitive, "the prompt quotes the full contract schema verbatim, not the wire form"); + var acceptance = sent.GetProperty("properties").GetProperty("subtasks").GetProperty("items").GetProperty("properties").GetProperty("acceptance").GetProperty("properties"); foreach (var field in new[] { "formatVersion", "kind", "argv", "artifactPaths" }) acceptance.TryGetProperty(field, out _).ShouldBeTrue($"the decoder can only emit acceptance.{field} if the wire schema declares it"); @@ -450,6 +459,23 @@ public async Task A_reply_the_wire_schema_admits_but_the_contract_rejects_still_ handler.Bodies.Count.ShouldBe(2, "the combinator the wire dropped still earned its one re-ask"); handler.Bodies[1].ShouldContain("oneOf requires exactly one matching schema"); JsonSchemaCombinatorsTests.CombinatorPaths(SentToolSchema(provider, handler.Bodies[0])).ShouldBeEmpty("the provider was handed the wire schema, not the contract"); + SameJson(SentToolSchema(provider, handler.Bodies[1]), request.WireJsonSchema!.Value).ShouldBeTrue("the re-ask is a second request through the same wire — its tool schema is the wire schema too, never the contract"); + SentSystemText(provider, handler.Bodies[1]).ShouldContain("\"oneOf\"", Case.Sensitive, "the re-ask's prompt still quotes the full contract beside the violations it names"); + } + + [Theory] + [InlineData("Anthropic")] + [InlineData("OpenAI")] + public async Task An_undefined_wire_schema_counts_as_unset_and_the_provider_gets_the_contract_schema(string provider) + { + // WireJsonSchema is a nullable struct, so default(JsonElement) is a NON-null value of kind Undefined. A bare `??` + // would pick it over the contract and the request body could not even be serialized. + var request = Request(provider) with { WireJsonSchema = default(JsonElement) }; + var handler = new WireHandler(provider, ["""{"argv":["sh"]}"""]); + + await Client(provider, handler).CompleteStructuredAsync(request, CancellationToken.None); + + SameJson(SentToolSchema(provider, handler.Bodies[0]), request.JsonSchema).ShouldBeTrue("an Undefined wire schema is no schema at all — the provider is handed the contract's own"); } /// The live regression shape: one subtask that names an oracle kind and authors NO payload for it — a consumer-contract defect the model-visible schema ALSO faults (no per-kind oneOf branch matches), which is why the two must be read as one severity. appends raw acceptance keys, so an EMPTY payload can be authored too. @@ -488,6 +514,16 @@ private static JsonElement SentToolSchema(string provider, string body) return provider == "Anthropic" ? tool.GetProperty("input_schema").Clone() : tool.GetProperty("function").GetProperty("parameters").Clone(); } + /// The system text the request carried, which is where the prompt quotes the schema: Anthropic's top-level system, the OpenAI wire's first (system-role) message. + private static string SentSystemText(string provider, string body) + { + var root = JsonDocument.Parse(body).RootElement; + + return (provider == "Anthropic" ? root.GetProperty("system") : root.GetProperty("messages")[0].GetProperty("content")).GetString()!; + } + + private static bool SameJson(JsonElement actual, JsonElement expected) => JsonNode.DeepEquals(JsonNode.Parse(actual.GetRawText()), JsonNode.Parse(expected.GetRawText())); + private static IStructuredLLMClient Client(string provider, WireHandler handler) => provider == "Anthropic" ? new AnthropicClient(new Factory(handler)) : new OpenAiClient(new Factory(handler)); private sealed class Factory(HttpMessageHandler handler) : IHttpClientFactory { diff --git a/backend/tests/CodeSpace.UnitTests/Workflows/TypedModelSchemaBranchTests.cs b/backend/tests/CodeSpace.UnitTests/Workflows/TypedModelSchemaBranchTests.cs index 94945feda..bceb9359c 100644 --- a/backend/tests/CodeSpace.UnitTests/Workflows/TypedModelSchemaBranchTests.cs +++ b/backend/tests/CodeSpace.UnitTests/Workflows/TypedModelSchemaBranchTests.cs @@ -10,12 +10,14 @@ namespace CodeSpace.UnitTests.Workflows; public sealed class TypedModelSchemaBranchTests { [Fact] - public void The_generator_sees_every_field_an_oracle_branch_requires_on_one_flat_acceptance() + public void The_generator_sees_every_field_an_oracle_branch_declares_or_requires_on_one_flat_acceptance() { // The per-kind branches no longer reach a structured-output generator: a hosted vLLM backend answered every // planner call that carried them with an empty HTTP 500, so the provider is handed PlannerSchema.WireSchema. // The branches still VALIDATE each reply — one per oracle kind — and the generator can only emit a field the - // wire declares, so every field any branch requires must be declared on the wire's flat acceptance. + // wire declares, so every field any branch declares OR requires must be declared on the wire's flat acceptance. + // Declares, not only requires: an optional property that only a branch mentions is just as unreachable, because + // the flat acceptance is additionalProperties:false and nothing else on the wire would let the generator write it. var branches = AcceptanceSchema().GetProperty("oneOf").EnumerateArray().ToArray(); var kinds = AcceptanceSchema().GetProperty("properties").GetProperty("kind").GetProperty("enum").EnumerateArray().Select(kind => kind.GetString()).ToArray(); @@ -23,9 +25,13 @@ public void The_generator_sees_every_field_an_oracle_branch_requires_on_one_flat var wire = WireAcceptanceSchema(); wire.TryGetProperty("oneOf", out _).ShouldBeFalse(); + wire.GetProperty("additionalProperties").ValueKind.ShouldBe(JsonValueKind.False, "premise: a field the wire acceptance does not declare is one the generator cannot write"); - foreach (var field in branches.SelectMany(branch => branch.GetProperty("required").EnumerateArray()).Select(name => name.GetString()!).Distinct()) - wire.GetProperty("properties").TryGetProperty(field, out _).ShouldBeTrue($"a branch requires '{field}', so the wire acceptance must declare it or the generator can never emit it"); + var fields = branches.SelectMany(BranchFields).Distinct().ToArray(); + fields.ShouldContain("kind", "fixture check: the walk reads the branches' own property names"); + + foreach (var field in fields) + wire.GetProperty("properties").TryGetProperty(field, out _).ShouldBeTrue($"a branch declares or requires '{field}', so the wire acceptance must declare it or the generator can never emit it"); wire.GetProperty("required").EnumerateArray().Select(name => name.GetString()).ShouldBe(new[] { "formatVersion", "kind" }, "the requirement every branch shares stays on the wire"); } @@ -51,17 +57,18 @@ public void Every_valid_oracle_reply_is_also_valid_on_the_wire(string acceptance JsonSchemaValidator.Validate(reply, PlannerSchema.WireSchema).ShouldBeEmpty("the wire schema may never forbid a reply the validation schema accepts"); } + /// The VALIDATION schema — the one every reply is checked against and the prompt quotes — requires the payload of the oracle kind a reply selects. The decoder is handed the wire form instead, which no longer says so (see ). [Theory] [InlineData("TestsPass")] [InlineData("ArtifactPresent")] [InlineData("LlmJudge")] [InlineData("CitationsResolve")] [InlineData("ArtifactSchema")] - public void The_model_visible_schema_itself_requires_the_selected_oracle_payload(string kind) + public void The_validation_schema_itself_requires_the_selected_oracle_payload(string kind) { var schema = AcceptanceSchema(); var missing = JsonSerializer.SerializeToElement(new { formatVersion = 2, kind }); - JsonSchemaValidator.Validate(missing, schema).ShouldNotBeEmpty("model-visible structural constraints must express the same required payload as the runtime converter"); + JsonSchemaValidator.Validate(missing, schema).ShouldNotBeEmpty("the validation schema's structural constraints must express the same required payload as the runtime converter"); } [Theory] @@ -70,7 +77,7 @@ public void The_model_visible_schema_itself_requires_the_selected_oracle_payload [InlineData("""{"formatVersion":2,"kind":"TestsPass","argv":["sh"],"artifactPaths":["out.txt"]}""")] [InlineData("""{"formatVersion":2,"kind":"LlmJudge","artifactPaths":["out.txt"]}""")] [InlineData("""{"formatVersion":2,"kind":"ArtifactSchema","artifactPaths":["out.txt"]}""")] - public void A_present_but_empty_conflicting_or_incomplete_payload_does_not_satisfy_the_model_schema(string response) => + public void A_present_but_empty_conflicting_or_incomplete_payload_does_not_satisfy_the_validation_schema(string response) => JsonSchemaValidator.Validate(JsonDocument.Parse(response).RootElement, AcceptanceSchema()).ShouldNotBeEmpty(); public static TheoryData ValidOracles => new( @@ -128,10 +135,16 @@ public void A_non_executable_task_can_explicitly_decline_a_literal_source_refere public void Alternative_schemas_enforce_their_actual_matching_semantics(string schema, string response, bool valid) => (JsonSchemaValidator.Validate(JsonDocument.Parse(response).RootElement, JsonDocument.Parse(schema).RootElement).Count == 0).ShouldBe(valid); + /// The acceptance inside the VALIDATION schema, per-kind branches and all: what every reply is checked against and the prompt quotes. private static JsonElement AcceptanceSchema() => PlannerSchema.ResponseSchema.GetProperty("properties").GetProperty("subtasks").GetProperty("items").GetProperty("properties").GetProperty("acceptance"); + /// The acceptance inside the WIRE schema: the flat, branch-free object the provider's decoder is handed. private static JsonElement WireAcceptanceSchema() => PlannerSchema.WireSchema.GetProperty("properties").GetProperty("subtasks").GetProperty("items").GetProperty("properties").GetProperty("acceptance"); + /// Every field name one branch mentions: the ones it declares under properties and the ones it requires. + private static IEnumerable BranchFields(JsonElement branch) => + branch.GetProperty("properties").EnumerateObject().Select(property => property.Name).Concat(branch.GetProperty("required").EnumerateArray().Select(name => name.GetString()!)); + [Fact] public void Deeply_branching_schema_validation_is_bounded_and_cannot_turn_exhaustion_into_success() {