Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,12 @@ public static bool IsInfraFailure(string? detail, bool workPresent)
|| effective.StartsWith("clone-failed:", StringComparison.Ordinal)
|| effective.StartsWith("setup-failed:", StringComparison.Ordinal)
|| effective.StartsWith("oracle-restore-failed:", StringComparison.Ordinal)
|| effective.StartsWith(SetupSeveredDetailPrefix, StringComparison.Ordinal)
|| effective is "no-rubric" or "no-schema" or "tests-timed-out" or "setup-timed-out"
// The check was killed at the grade's OWN memory ceiling (the producing run's posture): an
// environment fact like the grader's own wall clock, and what the agent's own run already reports
// for the same kill. Never a verdict on the code, so no revise round is spent on it.
or Eval.Benchmark.Graders.TestsPassGrader.ResourceExhaustedDetail
// POSIX "cannot run the command" codes (B6): 127 = command not found, 126 = found but not
// executable — the CHECK ITSELF could not run, exactly like a grader start-throw. The two must
// classify identically across runners or the same broken oracle reads infra locally (direct exec
Expand All @@ -169,6 +174,24 @@ public static bool IsInfraFailure(string? detail, bool workPresent)
|| (effective == "no-branch-or-repo" && workPresent));
}

/// <summary>
/// The detail prefix of a contract setup step that failed with the network the producing run's posture took from
/// it, on a host that enforced that (<c>SupervisorAcceptanceGrader</c> writes it only when the runner reports the
/// step severed). Infra-classed like any setup failure — the check never ran — but, unlike <c>setup-failed:</c>,
/// not transient: the posture comes from the same stored task on every attempt. Pinned by a unit test (Rule 8):
/// the grader writes it and <see cref="IsDecidedByGradePosture"/> reads it across a durable resume payload.
/// </summary>
public const string SetupSeveredDetailPrefix = "setup-failed-network-severed:";

/// <summary>
/// Whether an acceptance-failure DETAIL was decided by the grade's own posture rather than by the work or a
/// transient fault: a respawn of the same task grades under the same posture and reaches the same failure, so it
/// is infra (<see cref="IsInfraFailure(string?, bool)"/>) and still not worth a retry. Sees through the same display
/// tags the infra classification does.
/// </summary>
public static bool IsDecidedByGradePosture(string? detail) =>
StripGateLabel(StripRepoTag(detail))?.StartsWith(SetupSeveredDetailPrefix, StringComparison.Ordinal) == true;

/// <summary>
/// The multi-repo grade paths wrap a classifiable detail in a uniform machine-authored display tag
/// (<c>repo 'alias': </c>) — classification must see through it, or a grader crash on one repo of a
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3042,14 +3042,15 @@
var grader = scope.ServiceProvider.GetRequiredService<ISupervisorAcceptanceGrader>();
var fullSpec = spec with { Command = command };
var timeoutSeconds = spec.TimeoutSeconds ?? SupervisorLane.AcceptanceGradeTimeoutSeconds;
var posture = AcceptanceGradingPosturePolicy.For(task);

// C3: the run's own recorded base is the oracle anchor — the grader restores the acceptance command's
// program file from it, so an agent cannot buy its own pass by rewriting the check script it is graded
// with. The patch lane always had this anchor; the BRANCH lane discarded it and graded the candidate's
// bytes as the judge.
grade = hasBranch
? await grader.GradeAsync(repositoryId, run.TeamId, result.ProducedBranch!, fullSpec, timeoutSeconds, new OracleAnchor(result.BaseSha, OracleFloorPrograms(fullSpec)), cancellationToken).ConfigureAwait(false)
: await grader.GradePatchAsync(repositoryId, run.TeamId, result.BaseSha!, result.Patch, result.PatchArtifactId, fullSpec, timeoutSeconds, OracleFloorPrograms(fullSpec), cancellationToken).ConfigureAwait(false);
? await grader.GradeAsync(new RepositoryAcceptanceGradeRequest { RepositoryId = repositoryId, TeamId = run.TeamId, Branch = result.ProducedBranch!, Spec = fullSpec, TimeoutSeconds = timeoutSeconds, Anchor = new OracleAnchor(result.BaseSha, OracleFloorPrograms(fullSpec)), Posture = posture }, cancellationToken).ConfigureAwait(false)
: await grader.GradePatchAsync(new PatchAcceptanceGradeRequest { RepositoryId = repositoryId, TeamId = run.TeamId, BaseSha = result.BaseSha!, InlinePatch = result.Patch, PatchArtifactId = result.PatchArtifactId, Spec = fullSpec, TimeoutSeconds = timeoutSeconds, OracleFloorPrograms = OracleFloorPrograms(fullSpec), Posture = posture }, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex) when (ex is not OperationCanceledException and not AgentRunOwnershipLostException)
{
Expand Down Expand Up @@ -3118,6 +3119,7 @@
var command = spec.Command.ToArray();
var fullSpec = spec with { Command = command };
var timeoutSeconds = spec.TimeoutSeconds ?? SupervisorLane.AcceptanceGradeTimeoutSeconds;
var posture = AcceptanceGradingPosturePolicy.For(task);

var targets = result.RepositoryResults.Where(r => !string.IsNullOrEmpty(r.ProducedBranch) && r.RepositoryId is not null).ToList();

Expand All @@ -3142,7 +3144,7 @@
try
{
// C3: each repo's own recorded base anchors ITS oracle restore — same protection as the single-repo lane.
grade = await grader.GradeAsync(target.RepositoryId!.Value, run.TeamId, target.ProducedBranch!, fullSpec, timeoutSeconds, new OracleAnchor(target.BaseSha, OracleFloorPrograms(fullSpec)), cancellationToken).ConfigureAwait(false);
grade = await grader.GradeAsync(new RepositoryAcceptanceGradeRequest { RepositoryId = target.RepositoryId!.Value, TeamId = run.TeamId, Branch = target.ProducedBranch!, Spec = fullSpec, TimeoutSeconds = timeoutSeconds, Anchor = new OracleAnchor(target.BaseSha, OracleFloorPrograms(fullSpec)), Posture = posture }, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex) when (ex is not OperationCanceledException and not AgentRunOwnershipLostException)
{
Expand Down Expand Up @@ -5059,10 +5061,10 @@
/// <summary>The same reconstruction from a payload that came from somewhere other than the row — an offloaded one fetched back out of the artifact store.</summary>
private static AgentEvent ReplayedEvent(AgentEventKind kind, string? text, string? dataJson)
{
if (dataJson is not { Length: > 0 } json) return new AgentEvent { Kind = kind, Text = text };

Check warning on line 5064 in backend/src/CodeSpace.Core/Services/Agents/AgentRunExecutor.cs

View workflow job for this annotation

GitHub Actions / recurring jobs fire (worker host · Postgres)

Possible null reference assignment.

Check warning on line 5064 in backend/src/CodeSpace.Core/Services/Agents/AgentRunExecutor.cs

View workflow job for this annotation

GitHub Actions / dotnet test (E2ETests · HTTP · Postgres)

Possible null reference assignment.

Check warning on line 5064 in backend/src/CodeSpace.Core/Services/Agents/AgentRunExecutor.cs

View workflow job for this annotation

GitHub Actions / dotnet test (UnitTests)

Possible null reference assignment.

Check warning on line 5064 in backend/src/CodeSpace.Core/Services/Agents/AgentRunExecutor.cs

View workflow job for this annotation

GitHub Actions / dotnet test (UnitTests)

Possible null reference assignment.

Check warning on line 5064 in backend/src/CodeSpace.Core/Services/Agents/AgentRunExecutor.cs

View workflow job for this annotation

GitHub Actions / dotnet test (IntegrationTests · Postgres)

Possible null reference assignment.

try { using var doc = JsonDocument.Parse(json); return new AgentEvent { Kind = kind, Text = text, Data = doc.RootElement.Clone() }; }

Check warning on line 5066 in backend/src/CodeSpace.Core/Services/Agents/AgentRunExecutor.cs

View workflow job for this annotation

GitHub Actions / recurring jobs fire (worker host · Postgres)

Possible null reference assignment.

Check warning on line 5066 in backend/src/CodeSpace.Core/Services/Agents/AgentRunExecutor.cs

View workflow job for this annotation

GitHub Actions / dotnet test (E2ETests · HTTP · Postgres)

Possible null reference assignment.

Check warning on line 5066 in backend/src/CodeSpace.Core/Services/Agents/AgentRunExecutor.cs

View workflow job for this annotation

GitHub Actions / dotnet test (UnitTests)

Possible null reference assignment.

Check warning on line 5066 in backend/src/CodeSpace.Core/Services/Agents/AgentRunExecutor.cs

View workflow job for this annotation

GitHub Actions / dotnet test (UnitTests)

Possible null reference assignment.

Check warning on line 5066 in backend/src/CodeSpace.Core/Services/Agents/AgentRunExecutor.cs

View workflow job for this annotation

GitHub Actions / dotnet test (IntegrationTests · Postgres)

Possible null reference assignment.
catch (JsonException) { return new AgentEvent { Kind = kind, Text = text }; }

Check warning on line 5067 in backend/src/CodeSpace.Core/Services/Agents/AgentRunExecutor.cs

View workflow job for this annotation

GitHub Actions / recurring jobs fire (worker host · Postgres)

Possible null reference assignment.

Check warning on line 5067 in backend/src/CodeSpace.Core/Services/Agents/AgentRunExecutor.cs

View workflow job for this annotation

GitHub Actions / dotnet test (E2ETests · HTTP · Postgres)

Possible null reference assignment.

Check warning on line 5067 in backend/src/CodeSpace.Core/Services/Agents/AgentRunExecutor.cs

View workflow job for this annotation

GitHub Actions / dotnet test (UnitTests)

Possible null reference assignment.

Check warning on line 5067 in backend/src/CodeSpace.Core/Services/Agents/AgentRunExecutor.cs

View workflow job for this annotation

GitHub Actions / dotnet test (UnitTests)

Possible null reference assignment.

Check warning on line 5067 in backend/src/CodeSpace.Core/Services/Agents/AgentRunExecutor.cs

View workflow job for this annotation

GitHub Actions / dotnet test (IntegrationTests · Postgres)

Possible null reference assignment.
}

/// <summary>Ask the row, on a token of its own, whether the run actually reached a terminal state — the only honest answer to "did the landing take?" once an exception has been raised somewhere after the fenced write.</summary>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ public async Task<BenchmarkResult> RunAsync(BenchmarkTask task, BenchmarkMode mo

var attempts = await RunWithFormatFaultRespawnAsync(task, agentTask, context, cancellationToken).ConfigureAwait(false);

var grade = await BenchmarkTaskGrading.GradeAsync(_graders, _runners, new BenchmarkTaskGradingRequest { Task = task, WorkspaceDirectory = workspaceDirectory, TeamId = teamId, ProducerModel = ProducerModelOf(selection, attempts) }, cancellationToken).ConfigureAwait(false);
var grade = await BenchmarkTaskGrading.GradeAsync(_graders, _runners, new BenchmarkTaskGradingRequest { Task = task, WorkspaceDirectory = workspaceDirectory, TeamId = teamId, ProducerModel = ProducerModelOf(selection, attempts), Posture = Supervisor.AcceptanceGradingPosturePolicy.For(agentTask) }, cancellationToken).ConfigureAwait(false);

grade = ApplyMcpFabricRule(grade, mode, attempts[^1]);

Expand Down
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
using CodeSpace.Core.Services.Supervisor;
using CodeSpace.Messages.Agents;
using CodeSpace.Messages.Agents.Benchmark;
using CodeSpace.Messages.Review;

Expand All @@ -8,15 +10,19 @@ namespace CodeSpace.Core.Services.Agents.Eval.Benchmark;
/// <see cref="BenchmarkTask"/> against a post-run workspace directory (<c>BenchmarkRunner</c> for a direct-harness
/// cell, <c>TaskLaunch.TaskLaunchBenchmarkCellRunner</c> for a Launch-mode cell) — grading is independent of HOW
/// the workspace got there, so both resolve the SAME grader + sandbox runner through this one seam instead of each
/// duplicating the two resolves.
/// duplicating the two resolves. The runner is bound to the producing agent's posture
/// (<see cref="AcceptanceGradingPosturePolicy.Bind"/>), exactly as every acceptance grade's is: the check runs bytes
/// that agent wrote, so it never gets more memory, CPU or network than the agent had.
/// </summary>
internal static class BenchmarkTaskGrading
{
public static async Task<BenchmarkGrade> GradeAsync(IBenchmarkGraderRegistry graders, Sandbox.ISandboxRunnerRegistry runners, BenchmarkTaskGradingRequest request, CancellationToken cancellationToken)
{
var grader = graders.Resolve(request.Task.Grading);

var context = new BenchmarkGradingContext { Task = request.Task, WorkspaceDirectory = request.WorkspaceDirectory, Runner = runners.Resolve(Sandbox.SandboxKinds.Local), TeamId = request.TeamId, ProducerModel = request.ProducerModel };
var runner = AcceptanceGradingPosturePolicy.Bind(runners.Resolve(Sandbox.SandboxKinds.Local), request.Posture ?? AcceptanceGradingPosturePolicy.FailClosed);

var context = new BenchmarkGradingContext { Task = request.Task, WorkspaceDirectory = request.WorkspaceDirectory, Runner = runner, TeamId = request.TeamId, ProducerModel = request.ProducerModel };

return await grader.GradeAsync(context, cancellationToken).ConfigureAwait(false);
}
Expand All @@ -28,4 +34,7 @@ internal sealed record BenchmarkTaskGradingRequest
public required string WorkspaceDirectory { get; init; }
public Guid? TeamId { get; init; }
public ReviewModelIdentity? ProducerModel { get; init; }

/// <summary>The posture of the agent run whose workspace this grades. The fixture's test command imports that agent's code, so it runs under the same ceilings and network cut. Required so no instrument can forget it; null grades under <see cref="AcceptanceGradingPosturePolicy.FailClosed"/>.</summary>
public required AcceptanceGradingPosture? Posture { get; init; }
}
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,15 @@ namespace CodeSpace.Core.Services.Agents.Eval.Benchmark.Graders;
/// </summary>
public sealed class TestsPassGrader : IBenchmarkGrader, ISingletonDependency
{
/// <summary>
/// The detail of a check the runner killed at its own memory ceiling (<see cref="SandboxStatus.ResourceExhausted"/>,
/// read from the cgroup's oom_kill counter, never guessed from the exit code). A grade now runs under the producing
/// run's ceilings, so a ceiling can end a check, and that says nothing about whether the code is right: it is an
/// <see cref="GradeFailureClass.Environment"/> fact like <c>tests-timed-out</c>. Pinned by a unit test (Rule 8): the
/// infra classifier and the agent.code retry verdict read this literal across a durable resume payload.
/// </summary>
public const string ResourceExhaustedDetail = "tests-resource-exhausted";

public BenchmarkGradingKind Kind => BenchmarkGradingKind.TestsPass;

public async Task<BenchmarkGrade> GradeAsync(BenchmarkGradingContext context, CancellationToken cancellationToken)
Expand All @@ -31,7 +40,7 @@ public async Task<BenchmarkGrade> GradeAsync(BenchmarkGradingContext context, Ca

return result.Status == SandboxStatus.Success
? new BenchmarkGrade { Passed = true, Detail = "tests-passed", EvidenceText = evidence }
: Fail(DetailFor(result), result.Status == SandboxStatus.TimedOut ? GradeFailureClass.Environment : GradeFailureClass.Genuine) with { EvidenceText = evidence };
: Fail(DetailFor(result), result.Status is SandboxStatus.TimedOut or SandboxStatus.ResourceExhausted ? GradeFailureClass.Environment : GradeFailureClass.Genuine) with { EvidenceText = evidence };
}

/// <summary>The grading command runs in the post-run workspace with a fresh, short wall-clock cap — the tests are tiny, and a hung test is a fail, not a hang. The env is the runner's scrubbed default (no agent secret injected — the grader is independent of the agent's credential).</summary>
Expand All @@ -49,8 +58,12 @@ private static string EvidenceTextFor(BenchmarkGradingContext context, SandboxRe

private static string Tail(string text) => text.Length <= 8_192 ? text : text[^8_192..];

private static string DetailFor(SandboxResult result) =>
result.Status == SandboxStatus.TimedOut ? "tests-timed-out" : $"tests-failed-exit-{result.ExitCode}";
private static string DetailFor(SandboxResult result) => result.Status switch
{
SandboxStatus.TimedOut => "tests-timed-out",
SandboxStatus.ResourceExhausted => ResourceExhaustedDetail,
_ => $"tests-failed-exit-{result.ExitCode}",
};

private static BenchmarkGrade Fail(string detail, GradeFailureClass? failureClass = null) => new() { Passed = false, Detail = detail, Class = failureClass };
}
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,21 @@ private Task<List<AgentRun>> LoadAgentRunsAsync(Guid runId, CancellationToken ca
internal static string? ObservedModelOf(IReadOnlyList<AgentRun> attempts) =>
ParseResult(attempts[^1])?.Model ?? attempts.Select(ParseResult).Select(r => r?.Model).FirstOrDefault(model => model is not null);

/// <summary>
/// The posture the reconstructed workspace's check runs under: the one every attempt that wrote it ran with, read off
/// each attempt's stored task. Like <see cref="ProducerModelOf"/>, it claims one only when every attempt agrees; a
/// union of work from different postures, or an attempt whose task cannot be read, has none — null, so the grade
/// fails closed (<c>AcceptanceGradingPosturePolicy.FailClosed</c>). Internal so the rule is pinned directly.
/// </summary>
internal static AcceptanceGradingPosture? GradedPosture(IReadOnlyList<AgentRun> attempts)
{
var postures = attempts.Select(attempt => Supervisor.AcceptanceGradingPosturePolicy.ForStoredTask(attempt.TaskJson)).ToList();

if (postures.Count == 0 || postures.Any(posture => posture is null)) return null;

return postures.Select(posture => JsonSerializer.Serialize(posture, AgentJson.Options)).Distinct(StringComparer.Ordinal).Count() == 1 ? postures[0] : null;
}

internal static ReviewModelIdentity ProducerModelOf(BenchmarkAgentSelection? selection, IReadOnlyList<AgentRun> attempts)
{
var observed = attempts.Select(ParseResult).Select(result => result?.Model).Where(model => !string.IsNullOrWhiteSpace(model)).Distinct(StringComparer.OrdinalIgnoreCase).ToList();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ public async Task<BenchmarkResult> RunAsync(BenchmarkTask task, BenchmarkMode mo

await ReconstructWorkspaceAsync(context.WorkspaceDirectory, attempts, cancellationToken).ConfigureAwait(false);

var grade = await BenchmarkTaskGrading.GradeAsync(_graders, _runners, new BenchmarkTaskGradingRequest { Task = task, WorkspaceDirectory = context.WorkspaceDirectory, TeamId = context.TeamId, ProducerModel = ProducerModelOf(context.Selection, attempts) }, cancellationToken).ConfigureAwait(false);
var grade = await BenchmarkTaskGrading.GradeAsync(_graders, _runners, new BenchmarkTaskGradingRequest { Task = task, WorkspaceDirectory = context.WorkspaceDirectory, TeamId = context.TeamId, ProducerModel = ProducerModelOf(context.Selection, attempts), Posture = GradedPosture(attempts) }, cancellationToken).ConfigureAwait(false);

var completionMode = await LoadCompletionEnforcementModeAsync(launched.RunId, cancellationToken).ConfigureAwait(false);

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
using CodeSpace.Core.Services.Agents.Sandbox.Isolation;
using CodeSpace.Messages.Agents;

namespace CodeSpace.Core.Services.Agents.Sandbox;

/// <summary>
/// Optional capability a sandbox runner MAY implement alongside <see cref="ISandboxRunner"/> (Rule 7 / ISP — a sibling
/// interface, never a widening of the base contract): say, before it runs <c>spec</c>, what egress it would actually
/// ENFORCE for it — which is not always what the spec asks. A spec with <see cref="SandboxSpec.AllowNetwork"/> false is
/// severed only where the runner confines; one with an allowlist is filtered only where the host can filter, and
/// otherwise severed or, where nothing confines, not narrowed at all. The runner is the one place that knows what it
/// can build on this host, so a caller that must report what a narrowing actually did asks it here rather than
/// reading the spec back as if it were the outcome. A runner without this capability makes no claim.
/// </summary>
public interface ISandboxEgressEnforcement
{
/// <summary>The egress <paramref name="spec"/> would actually get on this host: <see cref="SandboxEgressMode.None"/> severed, <see cref="SandboxEgressMode.Filtered"/> held to its allowlist, <see cref="SandboxEgressMode.Full"/> the host network.</summary>
SandboxEgressMode EnforcedEgress(SandboxSpec spec);
}
Original file line number Diff line number Diff line change
Expand Up @@ -383,6 +383,27 @@ internal static void EnsureEgressAdmissible(SandboxSpec spec, bool confines, boo
throw new SealedEgressUnavailableException(cause);
}

/// <inheritdoc />
public SandboxEgressMode EnforcedEgress(SandboxSpec spec)
{
var confines = BubblewrapSandbox.Available is not null;

return EnforcedEgress(spec, confines, HostFiltersAllowlist(confines));
}

/// <summary>
/// <see cref="EnforcedEgress(SandboxSpec)"/> over whether this host <paramref name="confines"/> and whether it plans
/// an allowlist into a filtered namespace (<paramref name="filtersAllowlist"/>), so the table is pinned on any host.
/// The same <see cref="EgressPolicyFor"/> derivation the launch reads, with the one thing it leaves implicit made
/// explicit: a severed policy is enforced only by bubblewrap's fresh namespace, so where nothing confines the child
/// keeps the worker's network. A filtered namespace is built with or without bubblewrap.
/// </summary>
internal static SandboxEgressMode EnforcedEgress(SandboxSpec spec, bool confines, bool filtersAllowlist) => EgressPolicyFor(spec, filtersAllowlist).Mode switch
{
SandboxEgressMode.None when !confines => SandboxEgressMode.Full,
var mode => mode,
};

/// <summary>
/// Why a brokered child with a network of its own could not reach its broker from this host, or null when it can
/// or needs nothing: no broker port, a network it shares with the worker, or both halves of the relay in place.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ namespace CodeSpace.Core.Services.Agents.Sandbox.Runners;
/// Caller cancellation is honoured distinctly from the spec timeout: it terminates the process and rethrows
/// (the durable path differs — see its remarks: cancellation stops observing without killing).
/// </summary>
public sealed partial class LocalProcessRunner : ISandboxRunner, ISandboxStreamRunner, ISandboxDurableRunner, ISandboxLaunchIdentityRunner, ISandboxDurableLogSource, ISandboxDurableDiagnosticSource, ISandboxEgressAdmission, ISingletonDependency
public sealed partial class LocalProcessRunner : ISandboxRunner, ISandboxStreamRunner, ISandboxDurableRunner, ISandboxLaunchIdentityRunner, ISandboxDurableLogSource, ISandboxDurableDiagnosticSource, ISandboxEgressAdmission, ISandboxEgressEnforcement, ISingletonDependency
{
/// <summary>This runner's registry key. The runner-local spelling of the shared <see cref="SandboxKinds.Local"/> — same constant, so there is one literal.</summary>
public const string LocalKind = SandboxKinds.Local;
Expand Down
Loading
Loading