From 02159af8bcd6ee5e56724b314cc978c55c739d7e Mon Sep 17 00:00:00 2001 From: "Mars.P" Date: Wed, 30 Sep 2026 08:58:51 +0800 Subject: [PATCH] Redact and clamp park fault text; pin the planner wire schema A model-plane park stored LlmApiException.Message on its marker, in the node's failure text and in the park log. That message ends with the provider's whole error body, verbatim and unbounded, so a gateway that echoed a credential or answered with a page wrote it to the durable run row and the log. InfraPark.FaultText now redacts scope secrets, then clamps to 512 characters (never inside a surrogate pair, which Npgsql refuses to write), and the generic park and the supervisor's own park use that one text for the marker, the failure and the log line. The planner's provider schema stopped being its validation schema in the previous change, but tests and docs still said otherwise: - the prompt still quoting the full contract schema, and the re-ask request still sending the wire schema, are now asserted - the validation-schema tests are named for what they check, and the flat-acceptance test walks every field a branch declares, not only the ones it requires - the portability guard also requires every array to declare its items, the second candidate for the empty HTTP 500 - the structured client docs say the provider receives WireJsonSchema ?? JsonSchema and that validation uses JsonSchema - an Undefined WireJsonSchema counts as unset instead of failing at serialization --- .../Llm/Anthropic/AnthropicClient.cs | 8 +- .../Workflows/Llm/IStructuredLLMClient.cs | 7 +- .../Workflows/Llm/OpenAi/OpenAiClient.cs | 8 +- .../Nodes/Builtin/AgentSupervisorNode.cs | 5 +- .../Services/Workflows/Nodes/InfraPark.cs | 35 +++++- .../Workflows/Planning/PlannerSchema.cs | 6 +- .../Planning/Planners/LlmWorkflowPlanner.cs | 4 +- .../Supervisor/InfraParkRideTests.cs | 32 ++++++ .../AgentSupervisorNodeInfraParkTests.cs | 97 +++++++++++++++++ .../Workflows/InfraParkTests.cs | 100 +++++++++++++++++- .../Workflows/JsonSchemaCombinatorsTests.cs | 61 ++++++++++- .../StructuredResponseContractTests.cs | 38 ++++++- .../Workflows/TypedModelSchemaBranchTests.cs | 27 +++-- 13 files changed, 394 insertions(+), 34 deletions(-) create mode 100644 backend/tests/CodeSpace.UnitTests/Workflows/AgentSupervisorNodeInfraParkTests.cs diff --git a/backend/src/CodeSpace.Core/Services/Workflows/Llm/Anthropic/AnthropicClient.cs b/backend/src/CodeSpace.Core/Services/Workflows/Llm/Anthropic/AnthropicClient.cs index 73d3f5eb7..06811e146 100644 --- a/backend/src/CodeSpace.Core/Services/Workflows/Llm/Anthropic/AnthropicClient.cs +++ b/backend/src/CodeSpace.Core/Services/Workflows/Llm/Anthropic/AnthropicClient.cs @@ -14,8 +14,8 @@ namespace CodeSpace.Core.Services.Workflows.Llm.Anthropic; /// /// Implements (free-text completion) AND the sibling /// (schema-constrained JSON) — the latter via Anthropic's -/// forced tool-use: a single tool whose input_schema IS the requested schema, with -/// tool_choice pinned to it, so the model's tool_use block carries schema-valid JSON. +/// forced tool-use: a single tool whose input_schema is the request's WireJsonSchema ?? JsonSchema (every reply +/// is validated against JsonSchema), with tool_choice pinned to it, so the model's tool_use block carries schema-valid JSON. /// /// Keep ONLY the wire-shape concerns here. Anything node-facing (prompt assembly, output /// trimming, retry policy) belongs in the llm.complete node or the LLM-side resilience @@ -195,7 +195,7 @@ private async Task CompleteStructuredOnceAsync(Structur } /// - /// Attempt 1: forced tool-use — a single tool whose input_schema IS the schema, tool_choice pinned to it. Returns + /// Attempt 1: forced tool-use — a single tool whose input_schema is WireJsonSchema ?? JsonSchema (validation still uses JsonSchema), tool_choice pinned to it. Returns /// the recovered JSON (or null to degrade to the prompt-only floor) PLUS the parsed response — so the caller can /// accumulate the BILLED usage even when the JSON is null (a 200 that produced no usable tool call still cost /// tokens). On a 400/422 reject the request was never generated, so both are null (nothing to bill). @@ -213,7 +213,7 @@ private async Task CompleteStructuredOnceAsync(Structur StopSequences = request.Sampling?.Stop, System = system, Messages = messages, - Tools = new[] { new AnthropicTool { Name = StructuredToolName, Description = "Return the result as structured JSON.", InputSchema = request.WireJsonSchema ?? request.JsonSchema } }, + Tools = new[] { new AnthropicTool { Name = StructuredToolName, Description = "Return the result as structured JSON.", InputSchema = request.ProviderSchema } }, ToolChoice = new AnthropicToolChoice { Type = "tool", Name = StructuredToolName } }; diff --git a/backend/src/CodeSpace.Core/Services/Workflows/Llm/IStructuredLLMClient.cs b/backend/src/CodeSpace.Core/Services/Workflows/Llm/IStructuredLLMClient.cs index 8a99e6613..77a7ce1ae 100644 --- a/backend/src/CodeSpace.Core/Services/Workflows/Llm/IStructuredLLMClient.cs +++ b/backend/src/CodeSpace.Core/Services/Workflows/Llm/IStructuredLLMClient.cs @@ -21,7 +21,9 @@ public interface IStructuredLLMClient /// /// One LLM call constrained to return JSON matching . /// The provider impl maps the schema to whatever its API offers (Anthropic forces a single tool whose - /// input_schema IS the schema; OpenAI would use response_format json_schema, …). + /// input_schema carries the schema; OpenAI would use response_format json_schema, …). The provider receives + /// when the request sets one, else ; + /// every reply is validated against either way. /// Task CompleteStructuredAsync(StructuredLLMCompletionRequest request, CancellationToken cancellationToken); } @@ -45,6 +47,9 @@ public sealed record StructuredLLMCompletionRequest /// public JsonElement? WireJsonSchema { get; init; } + /// The schema the provider is actually handed: when set, else . A default wire schema (kind Undefined) counts as unset — a nullable struct admits it as a non-null value, and it cannot be serialized. + internal JsonElement ProviderSchema => WireJsonSchema is { ValueKind: not JsonValueKind.Undefined } wire ? wire : JsonSchema; + /// Server-only validation of the consumer contract, alongside JSON schema. Violations enter the same bounded model re-ask; this callback never rewrites output. [JsonIgnore] public Func>? ResponseValidator { get; init; } diff --git a/backend/src/CodeSpace.Core/Services/Workflows/Llm/OpenAi/OpenAiClient.cs b/backend/src/CodeSpace.Core/Services/Workflows/Llm/OpenAi/OpenAiClient.cs index e0d8dd180..c9346a9c8 100644 --- a/backend/src/CodeSpace.Core/Services/Workflows/Llm/OpenAi/OpenAiClient.cs +++ b/backend/src/CodeSpace.Core/Services/Workflows/Llm/OpenAi/OpenAiClient.cs @@ -18,8 +18,8 @@ namespace CodeSpace.Core.Services.Workflows.Llm.OpenAi; /// post-S6b) — a call without a credential fails closed. /// /// Implements (free-text) AND (schema-constrained -/// JSON). Structured output uses FORCED FUNCTION-CALLING — a single function whose parameters IS the -/// requested schema, with tool_choice pinned to it — rather than response_format: json_schema, +/// JSON). Structured output uses FORCED FUNCTION-CALLING — a single function whose parameters is the request's +/// WireJsonSchema ?? JsonSchema (every reply is validated against JsonSchema), with tool_choice pinned to it — rather than response_format: json_schema, /// because function-calling is supported by far more OpenAI-compatible gateways than the newer structured-outputs /// feature, and it mirrors the Anthropic client's forced-tool design exactly (one coercion mechanism to reason /// about). The model's tool_calls[0].function.arguments is the schema-SHAPED JSON (classic function-calling @@ -229,7 +229,7 @@ private async Task CompleteStructuredOnceAsync(Structur } /// - /// Attempt 1: forced function-calling — a single function whose parameters IS the schema, tool_choice pinned to it. + /// Attempt 1: forced function-calling — a single function whose parameters is WireJsonSchema ?? JsonSchema (validation still uses JsonSchema), tool_choice pinned to it. /// Returns the recovered JSON (or null to degrade to the prompt-only floor) PLUS the parsed response — so the caller /// can accumulate the BILLED usage even when the JSON is null (a 200 that produced no usable function call still /// cost tokens). On a 400/422 reject the request was never generated, so both are null (nothing to bill). @@ -254,7 +254,7 @@ private async Task CompleteStructuredOnceAsync(Structur Stop = request.Sampling?.Stop, ReasoningEffort = LlmModelCapabilities.SupportsReasoningEffort(request.Model) ? request.ReasoningEffort : null, // sent ONLY to a reasoning model (a plain chat model 400s on it); the value rides verbatim (the API validates it per model) Messages = BuildMessages(system, request.UserPrompt), - Tools = new[] { new OpenAiTool { Function = new OpenAiFunction { Name = StructuredToolName, Description = "Return the result as structured JSON.", Parameters = request.WireJsonSchema ?? request.JsonSchema } } }, + Tools = new[] { new OpenAiTool { Function = new OpenAiFunction { Name = StructuredToolName, Description = "Return the result as structured JSON.", Parameters = request.ProviderSchema } } }, ToolChoice = new OpenAiToolChoice { Function = new OpenAiToolChoiceFunction { Name = StructuredToolName } }, }; diff --git a/backend/src/CodeSpace.Core/Services/Workflows/Nodes/Builtin/AgentSupervisorNode.cs b/backend/src/CodeSpace.Core/Services/Workflows/Nodes/Builtin/AgentSupervisorNode.cs index 79a0ad1b8..e6e488bf8 100644 --- a/backend/src/CodeSpace.Core/Services/Workflows/Nodes/Builtin/AgentSupervisorNode.cs +++ b/backend/src/CodeSpace.Core/Services/Workflows/Nodes/Builtin/AgentSupervisorNode.cs @@ -230,9 +230,10 @@ private async Task ParkForInfraOrStopAsync(NodeRunContext context, G } var delay = SupervisorInfraPark.DelayFor(state.Parks); - var marker = SupervisorInfraPark.Marker(state, fault.Message); + var said = InfraPark.FaultText(context.Scope, fault); + var marker = SupervisorInfraPark.Marker(state, said); - context.Logger.LogWarning("agent.supervisor run {RunId}: brain call hit a {Category} infra fault — parking {Delay} (park {Parks} since {First:o}) instead of failing the run", supervisorRunId, fault.Category, delay, state.Parks, state.FirstParkedAtUtc); + context.Logger.LogWarning("agent.supervisor run {RunId}: brain call hit a {Category} infra fault — parking {Delay} (park {Parks} since {First:o}) instead of failing the run: {Fault}", supervisorRunId, fault.Category, delay, state.Parks, state.FirstParkedAtUtc, said); return NodeResult.Suspend(new SuspensionToken { diff --git a/backend/src/CodeSpace.Core/Services/Workflows/Nodes/InfraPark.cs b/backend/src/CodeSpace.Core/Services/Workflows/Nodes/InfraPark.cs index 2c288f0c5..0d9a50f57 100644 --- a/backend/src/CodeSpace.Core/Services/Workflows/Nodes/InfraPark.cs +++ b/backend/src/CodeSpace.Core/Services/Workflows/Nodes/InfraPark.cs @@ -1,5 +1,6 @@ using CodeSpace.Core.Services.Supervisor; using CodeSpace.Core.Services.Workflows.Llm; +using CodeSpace.Core.Services.Workflows.Runtime; using CodeSpace.Messages.Constants; using Microsoft.Extensions.Logging; @@ -43,18 +44,19 @@ public static class InfraPark public static NodeResult Park(NodeRunContext context, LlmApiException fault, DateTimeOffset now) { var state = SupervisorInfraPark.Next(context.ResumePayload, now); + var said = FaultText(context.Scope, fault); if (state.WindowExhausted) { context.Logger.LogWarning("Node {NodeId}: the model plane stayed unavailable past the whole {Window} park window — failing the node honestly", context.NodeId, SupervisorInfraPark.MaxParkWindow); - return NodeResult.Fail($"The model plane stayed unavailable for {SupervisorInfraPark.MaxParkWindow.TotalHours:0}h: {fault.Message}", retryable: false); + return NodeResult.Fail($"The model plane stayed unavailable for {SupervisorInfraPark.MaxParkWindow.TotalHours:0}h: {said}", retryable: false); } var delay = SupervisorInfraPark.DelayFor(state.Parks); - var marker = SupervisorInfraPark.Marker(state, fault.Message); + var marker = SupervisorInfraPark.Marker(state, said); - context.Logger.LogWarning("Node {NodeId}: model call hit a {Category} infra fault — parking {Delay} (park {Parks} since {First:o}) instead of failing the run: {Fault}", context.NodeId, fault.Category, delay, state.Parks, state.FirstParkedAtUtc, fault.Message); + context.Logger.LogWarning("Node {NodeId}: model call hit a {Category} infra fault — parking {Delay} (park {Parks} since {First:o}) instead of failing the run: {Fault}", context.NodeId, fault.Category, delay, state.Parks, state.FirstParkedAtUtc, said); return NodeResult.Suspend(new SuspensionToken { @@ -68,4 +70,31 @@ public static NodeResult Park(NodeRunContext context, LlmApiException fault, Dat TimeoutPayload = marker, }); } + + /// The most characters of a fault's text a park writes down. The gateway's own words are what a reader needs; the provider's whole error body, which ends with, is not. + internal const int MaxFaultChars = 512; + + /// + /// The fault's own words in the form everything that OUTLIVES the call may hold — the park marker (durable, and what the + /// run detail shows while parked), the honest failure text and the park log all take this, never the raw + /// , which ends with the provider's error body verbatim and unbounded. Scope secrets are + /// redacted FIRST and the result is clamped to after: a clamp that ran first could cut a + /// secret in half and leave a fragment no redactor would recognise. + /// + internal static string FaultText(NodeRunScope scope, LlmApiException fault) + { + var redacted = PersistenceSecretRedactor.FromScope(scope).Redact(fault.Message).Value ?? PersistenceSecretRedactor.Marker; + + return Clamp(redacted, MaxFaultChars); + } + + /// The first characters plus an ellipsis, never ending inside a surrogate pair — a lone half is ill-formed text that Npgsql's strict UTF-8 encoder refuses to write. + private static string Clamp(string text, int max) + { + if (text.Length <= max) return text; + + var cut = char.IsHighSurrogate(text[max - 1]) ? max - 1 : max; + + return text[..cut] + "…"; + } } diff --git a/backend/src/CodeSpace.Core/Services/Workflows/Planning/PlannerSchema.cs b/backend/src/CodeSpace.Core/Services/Workflows/Planning/PlannerSchema.cs index 3ffa655d4..333a44e2d 100644 --- a/backend/src/CodeSpace.Core/Services/Workflows/Planning/PlannerSchema.cs +++ b/backend/src/CodeSpace.Core/Services/Workflows/Planning/PlannerSchema.cs @@ -5,8 +5,8 @@ namespace CodeSpace.Core.Services.Workflows.Planning; /// -/// The planner's COMMIT-CONTRACT: the JSON Schema the model is constrained to (via the structured-output -/// path) and the matching deserialization options. Co-located with the planner concern (Rule 18) and +/// The planner's COMMIT-CONTRACT: the JSON Schema every reply is VALIDATED against (and the prompt quotes) and the +/// matching deserialization options; the provider itself is handed the combinator-free instead. Co-located with the planner concern (Rule 18) and /// pinned by a unit test — a drift in either the schema or the property mapping is a contract change a /// reviewer must see, not an invisible refactor. /// @@ -18,7 +18,7 @@ namespace CodeSpace.Core.Services.Workflows.Planning; /// public static class PlannerSchema { - /// The JSON schema constraining fresh model output. PlannerAcceptanceDraft maps its typed acceptance payloads before the normalized PlannedWorkflow reaches persistence or execution. + /// The JSON schema every fresh model reply is VALIDATED against and the prompt quotes; the provider is handed , not this. PlannerAcceptanceDraft maps its typed acceptance payloads before the normalized PlannedWorkflow reaches persistence or execution. public static readonly JsonElement ResponseSchema = JsonDocument.Parse(""" { "type": "object", diff --git a/backend/src/CodeSpace.Core/Services/Workflows/Planning/Planners/LlmWorkflowPlanner.cs b/backend/src/CodeSpace.Core/Services/Workflows/Planning/Planners/LlmWorkflowPlanner.cs index 2f52a06e2..edd5b3ed0 100644 --- a/backend/src/CodeSpace.Core/Services/Workflows/Planning/Planners/LlmWorkflowPlanner.cs +++ b/backend/src/CodeSpace.Core/Services/Workflows/Planning/Planners/LlmWorkflowPlanner.cs @@ -13,8 +13,8 @@ namespace CodeSpace.Core.Services.Workflows.Planning.Planners; /// /// The structured-LLM (Rule 18.3 — an impl in the Planners/ variant /// folder). It resolves a structured-capable LLM client through the SAME -/// the llm.complete node uses, sends a system+user prompt constrained by -/// , and deserializes the schema-valid object into a +/// the llm.complete node uses, sends a system+user prompt whose reply is validated against +/// (the provider is handed the combinator-free ), and deserializes the schema-valid object into a /// . Fails cleanly when no registered provider offers structured output. /// /// The planner produces DATA only — it never wires nodes or runs anything. The grounding context diff --git a/backend/tests/CodeSpace.IntegrationTests/Workflows/Supervisor/InfraParkRideTests.cs b/backend/tests/CodeSpace.IntegrationTests/Workflows/Supervisor/InfraParkRideTests.cs index 5bfe3c280..50a225d19 100644 --- a/backend/tests/CodeSpace.IntegrationTests/Workflows/Supervisor/InfraParkRideTests.cs +++ b/backend/tests/CodeSpace.IntegrationTests/Workflows/Supervisor/InfraParkRideTests.cs @@ -1,6 +1,11 @@ +using System.Text.Json; using CodeSpace.Core.Services.Supervisor; +using CodeSpace.Core.Services.Workflows.Llm; +using CodeSpace.Core.Services.Workflows.Nodes; +using CodeSpace.Core.Services.Workflows.Runtime; using CodeSpace.Messages.Constants; using CodeSpace.Messages.Enums; +using Microsoft.Extensions.Logging.Abstractions; using Shouldly; namespace CodeSpace.IntegrationTests.Workflows.Supervisor; @@ -137,6 +142,21 @@ public async Task An_unresolved_park_names_the_fault_the_park_stored_on_its_mark ex.Message.ShouldContain("Anthropic API error (HTTP 500, Transient): Hosted_vllmException", Case.Sensitive, "the infra skip must carry the gateway's own words, not only the park's duration"); } + [Fact] + public async Task An_unresolved_park_quotes_the_clamped_text_the_production_park_stored() + { + // The park clamps the gateway's words before it stores them, so a provider that answers with a whole page no + // longer floods the job summary: the skip quotes the first 512 characters and stops. The marker is minted by the + // production park (InfraPark.Park) and read back by the ride's own reader, not built by hand. + var fault = new LlmApiException("Anthropic", 500, LlmErrorCategory.Transient, new string('x', 5_000)); + var park = InfraPark.Park(ParkingContext(), fault, DateTimeOffset.UtcNow); + var cell = Parked() with { WaitPayloadJson = park.SuspendUntil!.Payload.GetRawText() }; + + var ex = await Should.ThrowAsync(() => InfraParkRide.RideAsync(() => Task.FromResult(cell), _ => Task.CompletedTask, maxWakes: 1, wakePause: TimeSpan.Zero)); + + ex.Message.ShouldEndWith("The park's last fault: " + fault.Message[..512] + "…", Case.Sensitive); + } + [Fact] public void The_ride_pauses_for_real_between_wakes_so_a_recovery_can_actually_be_observed() { @@ -149,4 +169,16 @@ public void The_ride_pauses_for_real_between_wakes_so_a_recovery_can_actually_be private static ParkedCell Parked() => new() { CellStatus = NodeStatus.Suspended, PendingWaitKind = WorkflowWaitKinds.SupervisorInfraPark, NodeId = "planner" }; private static ParkedCell Settled() => new() { CellStatus = NodeStatus.Success, PendingWaitKind = null, NodeId = "planner" }; + + private static NodeRunContext ParkingContext() => new() + { + Inputs = new Dictionary(), + Config = new Dictionary(), + RawInputs = JsonDocument.Parse("{}").RootElement, + RawConfig = JsonDocument.Parse("{}").RootElement, + Scope = new NodeRunScope { Trigger = new Dictionary(), Sys = new Dictionary() }, + Logger = NullLogger.Instance, + Observability = NodeObservability.NoOp, + NodeId = "planner", + }; } diff --git a/backend/tests/CodeSpace.UnitTests/Workflows/AgentSupervisorNodeInfraParkTests.cs b/backend/tests/CodeSpace.UnitTests/Workflows/AgentSupervisorNodeInfraParkTests.cs new file mode 100644 index 000000000..7190e0912 --- /dev/null +++ b/backend/tests/CodeSpace.UnitTests/Workflows/AgentSupervisorNodeInfraParkTests.cs @@ -0,0 +1,97 @@ +using System.Text.Json; +using CodeSpace.Core.Services.Supervisor; +using CodeSpace.Core.Services.Workflows.Llm; +using CodeSpace.Core.Services.Workflows.Nodes; +using CodeSpace.Core.Services.Workflows.Nodes.Builtin; +using CodeSpace.Core.Services.Workflows.Runtime; +using CodeSpace.Messages.Agents; +using CodeSpace.Messages.Constants; +using CodeSpace.Messages.Dtos.Agents; +using CodeSpace.Messages.Enums; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Shouldly; + +namespace CodeSpace.UnitTests.Workflows; + +/// +/// 🟢 Unit (the real over a turn service whose turn faults): the supervisor's own +/// model-plane park writes the SAME redacted, clamped fault text the generic does — on its +/// marker AND in its park log line. The supervisor is the one lane that parks through its own code, and it stored the +/// raw fault.Message (the provider's whole error body, unredacted and unbounded) while logging no fault text. +/// +[Trait("Category", "Unit")] +public sealed class AgentSupervisorNodeInfraParkTests +{ + /// A value the run treats as secret — a team variable, listed on SecretPaths under the engine's own spelling (<bucket>.<variable>). + private const string GatewayToken = "gw-tok-7f3a9c2e41d8"; + + [Fact] + public async Task The_supervisors_park_writes_the_redacted_clamped_fault_text_on_its_marker_and_in_its_log() + { + var fault = new LlmApiException("Anthropic", 500, LlmErrorCategory.Transient, $"gateway rejected bearer {GatewayToken}: " + new string('x', 5_000)); + fault.Message.ShouldContain(GatewayToken, Case.Sensitive, "fixture check: the raw fault carries the secret, or the redaction asserted below proves nothing"); + var logger = new CapturingLogger(); + + var result = await new AgentSupervisorNode(ScopeFactoryOver(new FaultingTurns(fault))).RunAsync(ContextHoldingSecret(logger), CancellationToken.None); + + result.Status.ShouldBe(NodeStatus.Suspended, "an exhausted transient fault parks the supervisor — it must never terminalize the run"); + result.SuspendUntil!.Kind.ShouldBe(WorkflowWaitKinds.SupervisorInfraPark); + + var marker = result.SuspendUntil.Payload.GetProperty("error").GetString()!; + marker.ShouldNotContain(GatewayToken, Case.Sensitive, "the marker is durable and shown on the run detail while the supervisor is parked"); + marker.ShouldContain(PersistenceSecretRedactor.Marker, Case.Sensitive); + marker.Length.ShouldBe(513, "512 characters of the fault, then the ellipsis"); + marker.ShouldEndWith("…", Case.Sensitive); + + logger.Warnings.ShouldHaveSingleItem().ShouldEndWith(marker, Case.Sensitive, "the park line carries the fault text — the SAME text the marker stores, redacted and clamped alike"); + } + + private static IServiceScopeFactory ScopeFactoryOver(ISupervisorTurnService turns) => new ServiceCollection().AddSingleton(turns).BuildServiceProvider().GetRequiredService(); + + private static NodeRunContext ContextHoldingSecret(ILogger logger) => new() + { + Inputs = new Dictionary(), + Config = new Dictionary(), + RawInputs = JsonDocument.Parse("{}").RootElement, + RawConfig = JsonDocument.Parse("{}").RootElement, + Scope = new NodeRunScope + { + Trigger = new Dictionary(), + Sys = new Dictionary + { + [SystemScopeKeys.WorkflowRunId] = JsonSerializer.SerializeToElement(Guid.NewGuid()), + [SystemScopeKeys.TeamId] = JsonSerializer.SerializeToElement(Guid.NewGuid()), + }, + Team = new Dictionary { ["GATEWAY_TOKEN"] = JsonSerializer.SerializeToElement(GatewayToken) }, + SecretPaths = new HashSet { "team.GATEWAY_TOKEN" }, + }, + Logger = logger, + Observability = NodeObservability.NoOp, + NodeId = "supervisor", + }; + + /// A turn service with nothing pending whose turn faults — the state a supervisor re-enters into while the model plane is down. + private sealed class FaultingTurns(LlmApiException fault) : ISupervisorTurnService + { + public Task RehydrateFromDecisionLogAsync(Guid supervisorRunId, Guid teamId, string nodeId, string goal, SupervisorGoalConfig? goalConfig, CancellationToken cancellationToken) => throw new NotSupportedException(); + public Task RunTurnAsync(Guid supervisorRunId, Guid teamId, string nodeId, string goal, Guid? conversationId, SupervisorGoalConfig? goalConfig, CancellationToken cancellationToken) => Task.FromException(fault); + public Task CountPendingAgentWaitsAsync(Guid supervisorRunId, string nodeId, CancellationToken cancellationToken) => Task.FromResult(0); + public Task PendingHumanWaitTokenAsync(Guid supervisorRunId, string nodeId, CancellationToken cancellationToken) => Task.FromResult(null); + public Task ReopenDiscardedAskAsync(Guid supervisorRunId, string nodeId, CancellationToken cancellationToken) => Task.FromResult(false); + public Task ForceStopAsync(Guid supervisorRunId, Guid teamId, string nodeId, string goal, SupervisorGoalConfig? goalConfig, string reason, CancellationToken cancellationToken) => throw new NotSupportedException(); + public Task SupervisorDepthAsync(Guid supervisorRunId, Guid teamId, CancellationToken cancellationToken) => throw new NotSupportedException(); + } + + /// Keeps every warning the node writes, formatted the way a sink would render it. + private sealed class CapturingLogger : ILogger + { + public List Warnings { get; } = []; + + public IDisposable BeginScope(TState state) where TState : notnull => NullScope.Instance; + public bool IsEnabled(LogLevel logLevel) => true; + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) { if (logLevel == LogLevel.Warning) Warnings.Add(formatter(state, exception)); } + + private sealed class NullScope : IDisposable { public static readonly NullScope Instance = new(); public void Dispose() { } } + } +} diff --git a/backend/tests/CodeSpace.UnitTests/Workflows/InfraParkTests.cs b/backend/tests/CodeSpace.UnitTests/Workflows/InfraParkTests.cs index aeb472fa9..078623d71 100644 --- a/backend/tests/CodeSpace.UnitTests/Workflows/InfraParkTests.cs +++ b/backend/tests/CodeSpace.UnitTests/Workflows/InfraParkTests.cs @@ -17,8 +17,9 @@ namespace CodeSpace.UnitTests.Workflows; /// blip a Deep run sleeps through killed a Standard run in minutes. /// /// What these pin: the fault classes worth parking for (and the ones that must stay fail-fast), the ladder -/// continuing across wakes from its OWN marker only, the honest failure once the window is spent, and the -/// iteration-key choice that keeps a parked map-branch node inside its own branch cell. +/// continuing across wakes from its OWN marker only, the honest failure once the window is spent, the +/// iteration-key choice that keeps a parked map-branch node inside its own branch cell, and the text the park +/// writes down about the fault — redacted of scope secrets first, then clamped. /// [Trait("Category", "Unit")] public class InfraParkTests @@ -38,6 +39,39 @@ public class InfraParkTests private static LlmApiException Fault(LlmErrorCategory category) => new("Anthropic", 503, category, "upstream unavailable"); + private static LlmApiException FaultWith(string providerMessage) => new("Anthropic", 500, LlmErrorCategory.Transient, providerMessage); + + /// A value the run treats as secret — a team variable, listed on SecretPaths under the engine's own spelling (<bucket>.<variable>). + private const string GatewayToken = "gw-tok-7f3a9c2e41d8"; + + private static NodeRunContext ContextHoldingSecret() => Context() with + { + Scope = new NodeRunScope + { + Trigger = new Dictionary(), + Sys = new Dictionary(), + Team = new Dictionary { ["GATEWAY_TOKEN"] = JsonSerializer.SerializeToElement(GatewayToken) }, + SecretPaths = new HashSet { "team.GATEWAY_TOKEN" }, + }, + }; + + /// The fault's text as each place the park writes it shows it: the marker's error on a park, that park's log line, and the honest failure once the whole window is spent. + private static Dictionary TextsWritten(NodeRunContext context, LlmApiException fault) + { + var logger = new CapturingLogger(); + var now = DateTimeOffset.UtcNow; + + var parked = InfraPark.Park(context with { Logger = logger }, fault, now); + var failed = InfraPark.Park(context with { ResumePayload = parked.SuspendUntil!.TimeoutPayload }, fault, now + SupervisorInfraPark.MaxParkWindow); + + return new Dictionary + { + ["marker"] = parked.SuspendUntil.Payload.GetProperty("error").GetString()!, + ["log"] = logger.Messages.ShouldHaveSingleItem(), + ["failure"] = failed.Error!, + }; + } + // ── Which faults park, and which must never ────────────────────────────────────── [Theory] @@ -76,6 +110,68 @@ public void The_park_log_carries_the_faults_own_words() logger.Messages.ShouldHaveSingleItem().ShouldContain("upstream unavailable"); } + // ── What the park writes down about the fault ──────────────────────────────────── + + [Fact] + public void A_scope_secret_in_the_faults_text_is_redacted_from_the_marker_the_log_and_the_failure() + { + // LlmApiException.Message ends with the provider's error body verbatim, and a gateway that rejects a request can + // echo the credential it was sent. The marker is durable and shown on the run detail, the failure is the run's + // own error and the log line leaves the process — none of the three may carry a value the run treats as secret. + var fault = FaultWith($"gateway rejected bearer {GatewayToken} on /v1/messages"); + fault.Message.ShouldContain(GatewayToken, Case.Sensitive, "fixture check: the raw fault carries the secret, or the redaction asserted below proves nothing"); + + foreach (var (place, text) in TextsWritten(ContextHoldingSecret(), fault)) + { + text.ShouldNotContain(GatewayToken, Case.Sensitive, $"the {place} must never carry a scope secret"); + text.ShouldContain(PersistenceSecretRedactor.Marker, Case.Sensitive, $"the {place} says a value was withheld rather than silently losing words"); + } + } + + [Fact] + public void A_provider_body_of_thousands_of_characters_is_clamped_in_the_marker_the_log_and_the_failure() + { + // The same Message tail carries a gateway's WHOLE answer — an HTML page, a stack trace — onto the run row and + // into the log line. 512 characters of it and the ellipsis is what a reader needs; the rest is noise at rest. + var fault = FaultWith(new string('x', 5_000)); + var clamped = fault.Message[..512] + "…"; + + var written = TextsWritten(Context(), fault); + + written["marker"].ShouldBe(clamped); + written["log"].ShouldEndWith(clamped, Case.Sensitive); + written["failure"].ShouldEndWith(clamped, Case.Sensitive); + } + + [Fact] + public void A_secret_that_straddles_the_clamp_is_redacted_whole_and_never_left_as_a_fragment() + { + // Redaction runs BEFORE the clamp. Clamped first, the cut would land inside the token and leave its opening + // characters behind — a fragment no exact-value redactor would ever recognise. + var fault = FaultWith(new string('x', 512 - FaultWith("").Message.Length - 4) + GatewayToken + new string('y', 200)); + fault.Message.IndexOf(GatewayToken, StringComparison.Ordinal).ShouldBe(508, "fixture check: the token starts four characters before the 512-character cut"); + + foreach (var (place, text) in TextsWritten(ContextHoldingSecret(), fault)) + text.ShouldNotContain(GatewayToken[..4], Case.Sensitive, $"the {place} kept the opening of a secret that the clamp cut in half"); + } + + [Fact] + public void The_clamp_never_cuts_a_surrogate_pair_in_half() + { + // A lone surrogate is ill-formed UTF-16. System.Text.Json quietly swaps it for U+FFFD on the marker, but the + // failure text reaches the run row through Npgsql, whose strict UTF-8 encoder throws on it — so an emoji that + // straddles the cut goes whole or not at all. + var fault = FaultWith(new string('x', 511 - FaultWith("").Message.Length) + "\U0001F600tail"); + char.IsHighSurrogate(fault.Message[511]).ShouldBeTrue("fixture check: the 512-character cut would fall between the emoji's two halves"); + var clamped = fault.Message[..511] + "…"; + + var written = TextsWritten(Context(), fault); + + written["marker"].ShouldBe(clamped); + written["log"].ShouldEndWith(clamped, Case.Sensitive); + written["failure"].ShouldEndWith(clamped, Case.Sensitive); + } + [Fact] public void The_park_keeps_the_nodes_ambient_cell_so_a_map_branch_stays_in_its_branch() { diff --git a/backend/tests/CodeSpace.UnitTests/Workflows/JsonSchemaCombinatorsTests.cs b/backend/tests/CodeSpace.UnitTests/Workflows/JsonSchemaCombinatorsTests.cs index e5ed6eabf..2cc47646f 100644 --- a/backend/tests/CodeSpace.UnitTests/Workflows/JsonSchemaCombinatorsTests.cs +++ b/backend/tests/CodeSpace.UnitTests/Workflows/JsonSchemaCombinatorsTests.cs @@ -16,10 +16,11 @@ namespace CodeSpace.UnitTests.Workflows; /// /// Pins the combinator-free WIRE form of a structured-output schema: what -/// removes and what it must leave alone, and the portability rule that every schema the code hands a model reaches the -/// provider with no combinator at any depth. A hosted vLLM backend compiles that schema into a decoding grammar, and a -/// combinator shape it cannot compile costs the whole call an empty HTTP 500 — so the rule is pinned per schema rather -/// than rediscovered per outage. +/// removes and what it must leave alone, and the portability rules that every schema the code hands a model reaches the +/// provider with no combinator at any depth and with every array typed by its items. A hosted vLLM backend compiles +/// that schema into a decoding grammar, and a shape it cannot compile costs the whole call an empty HTTP 500 — so the +/// rules are pinned per schema rather than rediscovered per outage. The 500 named no keyword, and the schema that drew +/// it carried both a combinator and an items-less array, so both are guarded. /// [Trait("Category", "Unit")] public sealed class JsonSchemaCombinatorsTests @@ -60,10 +61,22 @@ public sealed class JsonSchemaCombinatorsTests public void Every_schema_the_code_sends_to_a_model_is_combinator_free_on_the_wire(string schema) => CombinatorPaths(WireForms[schema]).ShouldBeEmpty($"{schema} reaches the provider's constrained decoder; carry the combinator in a validation-only JsonSchema and send JsonSchemaCombinators.Strip of it as the WireJsonSchema"); + [Theory] + [MemberData(nameof(WireFormNames))] + public void Every_array_in_a_schema_the_code_sends_to_a_model_declares_its_items(string schema) + { + // The schema that drew the empty 500 carried an array with no `items` (rubric.criteria, inside a per-kind branch) + // beside its combinators, and the error names no keyword — so this second candidate is guarded too. A decoding + // grammar has no element type to build for an array that does not say what it holds. + var untyped = ArraysWithoutItems(WireForms[schema]); + + untyped.ShouldBeEmpty($"{schema} declares an array with no `items` at {string.Join(", ", untyped)}; declare the element schema, because the provider's grammar compiler needs one"); + } + [Fact] public void Every_schema_constant_in_core_is_either_sent_on_the_wire_or_only_validates() { - // The guard above is only as good as its list. Every schema a model is handed today is a static *Schema field, + // The guards above are only as good as their list. Every schema a model is handed today is a static *Schema field, // so a new one lands here and must be sorted: sent to a provider (combinator-free) or validation-only. var declared = typeof(PlannerSchema).Assembly.GetTypes() .SelectMany(type => type.GetFields(BindingFlags.Static | BindingFlags.Public | BindingFlags.NonPublic).Where(field => field.FieldType == typeof(JsonElement) && field.Name.EndsWith("Schema", StringComparison.Ordinal)).Select(field => $"{type.Name}.{field.Name}")); @@ -113,6 +126,44 @@ public void Strip_keeps_property_names_and_literal_values_that_happen_to_spell_a _ => [], }; + /// Every path at which an array-typed schema node declares no items, at ANY depth. Blind to what a key means, like , so it is stricter than it needs to be: any object whose type names array must say what its elements are. + private static IReadOnlyList ArraysWithoutItems(JsonElement node, string path = "$") => node.ValueKind switch + { + JsonValueKind.Object => (DeclaresArray(node) && !node.TryGetProperty("items", out _) ? [path] : Array.Empty()).Concat(node.EnumerateObject().SelectMany(property => ArraysWithoutItems(property.Value, $"{path}.{property.Name}"))).ToArray(), + JsonValueKind.Array => node.EnumerateArray().SelectMany((item, index) => ArraysWithoutItems(item, $"{path}[{index}]")).ToArray(), + _ => [], + }; + + /// Whether the node's type is, or lists, array. + private static bool DeclaresArray(JsonElement node) => node.TryGetProperty("type", out var type) && type.ValueKind switch + { + JsonValueKind.String => type.GetString() == "array", + JsonValueKind.Array => type.EnumerateArray().Any(entry => entry.ValueKind == JsonValueKind.String && entry.GetString() == "array"), + _ => false, + }; + + [Fact] + public void ArraysWithoutItems_names_every_array_typed_node_that_declares_no_items_and_only_those() + { + // The guard is only as good as its walker: it must find a bare array at any depth and in either spelling of + // `type`, and must not flag a declared one — nor a property that merely happens to be NAMED "type". + const string schema = """ + { + "type": "object", + "properties": { + "declared": { "type": "array", "items": { "type": "string" } }, + "bare": { "type": "array", "minItems": 1 }, + "nullable": { "type": ["null", "array"] }, + "nested": { "type": "object", "properties": { "inner": { "type": "array" } } }, + "listOfLists": { "type": "array", "items": { "type": "array" } }, + "type": { "type": "string" } + } + } + """; + + ArraysWithoutItems(JsonDocument.Parse(schema).RootElement).ShouldBe(new[] { "$.properties.bare", "$.properties.nullable", "$.properties.nested.properties.inner", "$.properties.listOfLists.items" }, ignoreOrder: true); + } + /// One schema with beside a sibling at every position a subschema can sit — or, for a null keyword, the same schema written without it. private static string SchemaAtEveryPosition(string? keyword) { diff --git a/backend/tests/CodeSpace.UnitTests/Workflows/StructuredResponseContractTests.cs b/backend/tests/CodeSpace.UnitTests/Workflows/StructuredResponseContractTests.cs index 0354e3cf1..a5e464ec8 100644 --- a/backend/tests/CodeSpace.UnitTests/Workflows/StructuredResponseContractTests.cs +++ b/backend/tests/CodeSpace.UnitTests/Workflows/StructuredResponseContractTests.cs @@ -1,11 +1,13 @@ using System.Net; using System.Text; using System.Text.Json; +using System.Text.Json.Nodes; using CodeSpace.Core.Services.Workflows.Llm; using CodeSpace.Core.Services.Workflows.Llm.Anthropic; using CodeSpace.Core.Services.Workflows.Llm.OpenAi; using CodeSpace.Messages.Agents; using CodeSpace.Core.Services.Agents.ModelCredentials; +using CodeSpace.Core.Services.Workflows.Planning; using CodeSpace.Core.Services.Workflows.Planning.Planners; using CodeSpace.Messages.Dtos.Workflows.Planning; using Shouldly; @@ -409,7 +411,7 @@ public async Task The_reask_preamble_calls_a_fatal_miss_invalid_and_never_says_t [Theory] [InlineData("Anthropic")] [InlineData("OpenAI")] - public async Task The_planner_request_sends_the_provider_a_combinator_free_schema_that_still_declares_every_acceptance_field(string provider) + public async Task The_planner_request_sends_the_provider_a_combinator_free_schema_and_its_prompt_still_quotes_the_full_one(string provider) { // A hosted vLLM backend compiles the forced tool's schema into its decoding grammar. The per-kind oneOf // branches (#1854) broke that compile: every planner call came back an EMPTY HTTP 500, the planner parked, @@ -422,6 +424,13 @@ public async Task The_planner_request_sends_the_provider_a_combinator_free_schem var sent = SentToolSchema(provider, handler.Bodies[0]); JsonSchemaCombinatorsTests.CombinatorPaths(sent).ShouldBeEmpty("the provider's constrained decoder must never be handed a combinator"); + // The tool schema lost its per-kind branches, so the SYSTEM prompt — which quotes the FULL contract schema — is + // now the only place the model reads which payload each oracle kind requires. Handing the prompt the wire form + // too would leave the model no statement of those requirements at all, and nothing else would notice. + var system = SentSystemText(provider, handler.Bodies[0]); + system.ShouldContain("\"oneOf\"", Case.Sensitive, "the prompt is where the model learns the per-kind acceptance requirements now that the tool schema has no branches"); + system.ShouldContain(PlannerSchema.ResponseSchema.GetRawText(), Case.Sensitive, "the prompt quotes the full contract schema verbatim, not the wire form"); + var acceptance = sent.GetProperty("properties").GetProperty("subtasks").GetProperty("items").GetProperty("properties").GetProperty("acceptance").GetProperty("properties"); foreach (var field in new[] { "formatVersion", "kind", "argv", "artifactPaths" }) acceptance.TryGetProperty(field, out _).ShouldBeTrue($"the decoder can only emit acceptance.{field} if the wire schema declares it"); @@ -450,6 +459,23 @@ public async Task A_reply_the_wire_schema_admits_but_the_contract_rejects_still_ handler.Bodies.Count.ShouldBe(2, "the combinator the wire dropped still earned its one re-ask"); handler.Bodies[1].ShouldContain("oneOf requires exactly one matching schema"); JsonSchemaCombinatorsTests.CombinatorPaths(SentToolSchema(provider, handler.Bodies[0])).ShouldBeEmpty("the provider was handed the wire schema, not the contract"); + SameJson(SentToolSchema(provider, handler.Bodies[1]), request.WireJsonSchema!.Value).ShouldBeTrue("the re-ask is a second request through the same wire — its tool schema is the wire schema too, never the contract"); + SentSystemText(provider, handler.Bodies[1]).ShouldContain("\"oneOf\"", Case.Sensitive, "the re-ask's prompt still quotes the full contract beside the violations it names"); + } + + [Theory] + [InlineData("Anthropic")] + [InlineData("OpenAI")] + public async Task An_undefined_wire_schema_counts_as_unset_and_the_provider_gets_the_contract_schema(string provider) + { + // WireJsonSchema is a nullable struct, so default(JsonElement) is a NON-null value of kind Undefined. A bare `??` + // would pick it over the contract and the request body could not even be serialized. + var request = Request(provider) with { WireJsonSchema = default(JsonElement) }; + var handler = new WireHandler(provider, ["""{"argv":["sh"]}"""]); + + await Client(provider, handler).CompleteStructuredAsync(request, CancellationToken.None); + + SameJson(SentToolSchema(provider, handler.Bodies[0]), request.JsonSchema).ShouldBeTrue("an Undefined wire schema is no schema at all — the provider is handed the contract's own"); } /// The live regression shape: one subtask that names an oracle kind and authors NO payload for it — a consumer-contract defect the model-visible schema ALSO faults (no per-kind oneOf branch matches), which is why the two must be read as one severity. appends raw acceptance keys, so an EMPTY payload can be authored too. @@ -488,6 +514,16 @@ private static JsonElement SentToolSchema(string provider, string body) return provider == "Anthropic" ? tool.GetProperty("input_schema").Clone() : tool.GetProperty("function").GetProperty("parameters").Clone(); } + /// The system text the request carried, which is where the prompt quotes the schema: Anthropic's top-level system, the OpenAI wire's first (system-role) message. + private static string SentSystemText(string provider, string body) + { + var root = JsonDocument.Parse(body).RootElement; + + return (provider == "Anthropic" ? root.GetProperty("system") : root.GetProperty("messages")[0].GetProperty("content")).GetString()!; + } + + private static bool SameJson(JsonElement actual, JsonElement expected) => JsonNode.DeepEquals(JsonNode.Parse(actual.GetRawText()), JsonNode.Parse(expected.GetRawText())); + private static IStructuredLLMClient Client(string provider, WireHandler handler) => provider == "Anthropic" ? new AnthropicClient(new Factory(handler)) : new OpenAiClient(new Factory(handler)); private sealed class Factory(HttpMessageHandler handler) : IHttpClientFactory { diff --git a/backend/tests/CodeSpace.UnitTests/Workflows/TypedModelSchemaBranchTests.cs b/backend/tests/CodeSpace.UnitTests/Workflows/TypedModelSchemaBranchTests.cs index 94945feda..bceb9359c 100644 --- a/backend/tests/CodeSpace.UnitTests/Workflows/TypedModelSchemaBranchTests.cs +++ b/backend/tests/CodeSpace.UnitTests/Workflows/TypedModelSchemaBranchTests.cs @@ -10,12 +10,14 @@ namespace CodeSpace.UnitTests.Workflows; public sealed class TypedModelSchemaBranchTests { [Fact] - public void The_generator_sees_every_field_an_oracle_branch_requires_on_one_flat_acceptance() + public void The_generator_sees_every_field_an_oracle_branch_declares_or_requires_on_one_flat_acceptance() { // The per-kind branches no longer reach a structured-output generator: a hosted vLLM backend answered every // planner call that carried them with an empty HTTP 500, so the provider is handed PlannerSchema.WireSchema. // The branches still VALIDATE each reply — one per oracle kind — and the generator can only emit a field the - // wire declares, so every field any branch requires must be declared on the wire's flat acceptance. + // wire declares, so every field any branch declares OR requires must be declared on the wire's flat acceptance. + // Declares, not only requires: an optional property that only a branch mentions is just as unreachable, because + // the flat acceptance is additionalProperties:false and nothing else on the wire would let the generator write it. var branches = AcceptanceSchema().GetProperty("oneOf").EnumerateArray().ToArray(); var kinds = AcceptanceSchema().GetProperty("properties").GetProperty("kind").GetProperty("enum").EnumerateArray().Select(kind => kind.GetString()).ToArray(); @@ -23,9 +25,13 @@ public void The_generator_sees_every_field_an_oracle_branch_requires_on_one_flat var wire = WireAcceptanceSchema(); wire.TryGetProperty("oneOf", out _).ShouldBeFalse(); + wire.GetProperty("additionalProperties").ValueKind.ShouldBe(JsonValueKind.False, "premise: a field the wire acceptance does not declare is one the generator cannot write"); - foreach (var field in branches.SelectMany(branch => branch.GetProperty("required").EnumerateArray()).Select(name => name.GetString()!).Distinct()) - wire.GetProperty("properties").TryGetProperty(field, out _).ShouldBeTrue($"a branch requires '{field}', so the wire acceptance must declare it or the generator can never emit it"); + var fields = branches.SelectMany(BranchFields).Distinct().ToArray(); + fields.ShouldContain("kind", "fixture check: the walk reads the branches' own property names"); + + foreach (var field in fields) + wire.GetProperty("properties").TryGetProperty(field, out _).ShouldBeTrue($"a branch declares or requires '{field}', so the wire acceptance must declare it or the generator can never emit it"); wire.GetProperty("required").EnumerateArray().Select(name => name.GetString()).ShouldBe(new[] { "formatVersion", "kind" }, "the requirement every branch shares stays on the wire"); } @@ -51,17 +57,18 @@ public void Every_valid_oracle_reply_is_also_valid_on_the_wire(string acceptance JsonSchemaValidator.Validate(reply, PlannerSchema.WireSchema).ShouldBeEmpty("the wire schema may never forbid a reply the validation schema accepts"); } + /// The VALIDATION schema — the one every reply is checked against and the prompt quotes — requires the payload of the oracle kind a reply selects. The decoder is handed the wire form instead, which no longer says so (see ). [Theory] [InlineData("TestsPass")] [InlineData("ArtifactPresent")] [InlineData("LlmJudge")] [InlineData("CitationsResolve")] [InlineData("ArtifactSchema")] - public void The_model_visible_schema_itself_requires_the_selected_oracle_payload(string kind) + public void The_validation_schema_itself_requires_the_selected_oracle_payload(string kind) { var schema = AcceptanceSchema(); var missing = JsonSerializer.SerializeToElement(new { formatVersion = 2, kind }); - JsonSchemaValidator.Validate(missing, schema).ShouldNotBeEmpty("model-visible structural constraints must express the same required payload as the runtime converter"); + JsonSchemaValidator.Validate(missing, schema).ShouldNotBeEmpty("the validation schema's structural constraints must express the same required payload as the runtime converter"); } [Theory] @@ -70,7 +77,7 @@ public void The_model_visible_schema_itself_requires_the_selected_oracle_payload [InlineData("""{"formatVersion":2,"kind":"TestsPass","argv":["sh"],"artifactPaths":["out.txt"]}""")] [InlineData("""{"formatVersion":2,"kind":"LlmJudge","artifactPaths":["out.txt"]}""")] [InlineData("""{"formatVersion":2,"kind":"ArtifactSchema","artifactPaths":["out.txt"]}""")] - public void A_present_but_empty_conflicting_or_incomplete_payload_does_not_satisfy_the_model_schema(string response) => + public void A_present_but_empty_conflicting_or_incomplete_payload_does_not_satisfy_the_validation_schema(string response) => JsonSchemaValidator.Validate(JsonDocument.Parse(response).RootElement, AcceptanceSchema()).ShouldNotBeEmpty(); public static TheoryData ValidOracles => new( @@ -128,10 +135,16 @@ public void A_non_executable_task_can_explicitly_decline_a_literal_source_refere public void Alternative_schemas_enforce_their_actual_matching_semantics(string schema, string response, bool valid) => (JsonSchemaValidator.Validate(JsonDocument.Parse(response).RootElement, JsonDocument.Parse(schema).RootElement).Count == 0).ShouldBe(valid); + /// The acceptance inside the VALIDATION schema, per-kind branches and all: what every reply is checked against and the prompt quotes. private static JsonElement AcceptanceSchema() => PlannerSchema.ResponseSchema.GetProperty("properties").GetProperty("subtasks").GetProperty("items").GetProperty("properties").GetProperty("acceptance"); + /// The acceptance inside the WIRE schema: the flat, branch-free object the provider's decoder is handed. private static JsonElement WireAcceptanceSchema() => PlannerSchema.WireSchema.GetProperty("properties").GetProperty("subtasks").GetProperty("items").GetProperty("properties").GetProperty("acceptance"); + /// Every field name one branch mentions: the ones it declares under properties and the ones it requires. + private static IEnumerable BranchFields(JsonElement branch) => + branch.GetProperty("properties").EnumerateObject().Select(property => property.Name).Concat(branch.GetProperty("required").EnumerateArray().Select(name => name.GetString()!)); + [Fact] public void Deeply_branching_schema_validation_is_bounded_and_cannot_turn_exhaustion_into_success() {