Refuse a network-off run that cannot be sealed before it spends

.github/workflows/backend-e2e.yml

@@ -173,13 +173,15 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - name: Install bubblewrap + util-linux (prlimit)
+      - name: Install bubblewrap + util-linux (prlimit) + iproute2/nftables
         # The headline E2E spawns a REAL agent process through the production runner, which wraps it in
         # bubblewrap + prlimit (the production confinement posture). Running as root in the SDK container, so
-        # no sudo. ca-certificates so the container can fetch packages; util-linux for prlimit.
+        # no sudo. ca-certificates so the container can fetch packages; util-linux for prlimit; iproute2 and
+        # nftables so a network-off brokered agent is SEALED to its broker, as a confining host must, rather than
+        # refused as sandbox_sealed_egress_unavailable.
         run: |
           apt-get update
-          apt-get install -y --no-install-recommends bubblewrap util-linux ca-certificates
+          apt-get install -y --no-install-recommends bubblewrap util-linux ca-certificates iproute2 nftables
           echo "bwrap: $(bwrap --version)"
           echo "prlimit: $(prlimit --version)"
 

.github/workflows/sandbox-isolation.yml

@@ -224,8 +224,8 @@ jobs:
           executed=$(grep -oE 'executed="[0-9]+"' "$trx" | head -1 | grep -oE '[0-9]+')
           passed=$(grep -oE 'passed="[0-9]+"' "$trx" | head -1 | grep -oE '[0-9]+')
           echo "executed=${executed:-0} passed=${passed:-0}"
-          if [ "${executed:-0}" -lt 65 ]; then
-            echo "::error::Expected >=65 sandbox isolation tests to run (bwrap/prlimit confinement + cap-drop + cgroup-namespace re-root + egress-allowlist filter + cgroup resource cap + durable-launch cgroup wiring + argv/envp per-string kernel ceiling + a prompt past it riding stdin + a read-only workspace mount + a network-off run sealed to its broker + a read-only reviewer reading its diff with the real CLIs), but only ${executed:-0} did — the Category=Sandbox filter matched too few (trait regression?). If a case was deliberately removed, lower this number in the same PR."
+          if [ "${executed:-0}" -lt 66 ]; then
+            echo "::error::Expected >=66 sandbox isolation tests to run (bwrap/prlimit confinement + cap-drop + cgroup-namespace re-root + egress-allowlist filter + cgroup resource cap + durable-launch cgroup wiring + argv/envp per-string kernel ceiling + a prompt past it riding stdin + a read-only workspace mount + a network-off run sealed to its broker + a read-only reviewer reading its diff with the real CLIs), but only ${executed:-0} did — the Category=Sandbox filter matched too few (trait regression?). If a case was deliberately removed, lower this number in the same PR."
             exit 1
           fi
 
@@ -258,12 +258,12 @@ jobs:
           print(f'All {len(arms)} reviewer E2E arms ran and passed.')
 
           # The sealed-egress E2E returns early on a host that cannot seal, which reads as Passed; require each arm's marker.
-          for arm in ('durable', 'non-durable', 'ipv6', 'restart-reissue', 'policy-route-discard'):
+          for arm in ('durable', 'non-durable', 'ipv6', 'restart-reissue', 'policy-route-discard', 'setup-failure'):
               assert f'[sealed-egress-e2e] ran {arm}' in text, f'sealed-egress E2E arm "{arm}" did not run — this lane is root with bwrap, ip and nft, so it must seal'
-          for method, rows in (('A_network_off_brokered_run_reaches_its_broker_and_nothing_else', 2), ('A_sealed_namespace_drops_the_gateway_over_ipv6_link_local_too', 1), ('A_30_still_held_by_a_run_that_outlived_its_worker_is_not_handed_to_the_next_run', 1), ('A_host_whose_policy_rule_discards_the_run_s_replies_fails_the_setup_and_leaks_nothing', 1)):
+          for method, rows in (('A_network_off_brokered_run_reaches_its_broker_and_nothing_else', 2), ('A_sealed_namespace_drops_the_gateway_over_ipv6_link_local_too', 1), ('A_30_still_held_by_a_run_that_outlived_its_worker_is_not_handed_to_the_next_run', 1), ('A_host_whose_policy_rule_discards_the_run_s_replies_fails_the_setup_and_leaks_nothing', 1), ('A_sealed_setup_that_fails_on_this_host_refuses_the_launch_typed_and_leaks_nothing', 1)):
               cases = [r for r in results if 'SealedEgressE2ETests.' + method in r.get('testName', '')]
               assert len(cases) == rows and all(r.get('outcome') == 'Passed' for r in cases), f'{method}: all {rows} case(s) must pass'
-          print('All 5 sealed-egress arms ran and passed.')
+          print('All 6 sealed-egress arms ran and passed.')
 
           print('All 10 batch/stream kernel cases passed.')
           PY

backend/Dockerfile.worker

@@ -34,7 +34,8 @@
 # The same namespace machinery SEALS a network-off run whose model is brokered to that broker (no route, no NAT, no
 # DNS, one gateway port). It is taken only where bubblewrap confines AND FilteredEgressNetns.CanSeal has proved, by
 # building a throwaway namespace, that this process may: root + CAP_NET_ADMIN + CAP_SYS_ADMIN, no sysctl needed. A
-# pod without them keeps severing such a run — which also severs it from its broker, so it reaches no model.
+# pod without them that DOES confine refuses such a run before it spends anything (sandbox_sealed_egress_unavailable),
+# because severing it instead would cut it off from its broker and leave it reaching no model.
 #
 # CONFINEMENT ARMING: bubblewrap is installed here so the capability is PRESENT, but the fail-closed guard
 # Sandbox:RequireConfinement is left to the DEPLOYMENT to arm (k8s pod / compose) on a host that grants user

backend/src/CodeSpace.Core/Services/Agents/AgentRunExecutor.cs

@@ -484,6 +484,11 @@
                 spec = BuildSpec(RunCold(effectiveTask));
             }
 
+            // A spec this runner could only launch with a network that leaves the agent unable to work — a network-off
+            // brokered run on a host that confines but cannot seal — is refused HERE, the last moment a refusal costs
+            // nothing: before the local acceptance is prepared, the spend admitted or a process started.
+            (runner as ISandboxEgressAdmission)?.EnsureEgressAdmissible(spec, brokeredCredential?.ReachableFromNamespace ?? false);
+
             // Verification is judged against the contract the envelope persisted — never a goal amended for the
             // dispatch alone (this cold hint, or an unreadable checkpoint's) — or the contract hash cannot match.
             var contract = effectiveTask with { Goal = task.Goal };
@@ -4957,10 +4962,10 @@
     /// <summary>The same reconstruction from a payload that came from somewhere other than the row — an offloaded one fetched back out of the artifact store.</summary>
     private static AgentEvent ReplayedEvent(AgentEventKind kind, string? text, string? dataJson)
     {
         if (dataJson is not { Length: > 0 } json) return new AgentEvent { Kind = kind, Text = text };
 
         try { using var doc = JsonDocument.Parse(json); return new AgentEvent { Kind = kind, Text = text, Data = doc.RootElement.Clone() }; }
         catch (JsonException) { return new AgentEvent { Kind = kind, Text = text }; }
     }
 
     /// <summary>Ask the row, on a token of its own, whether the run actually reached a terminal state — the only honest answer to "did the landing take?" once an exception has been raised somewhere after the fenced write.</summary>

backend/src/CodeSpace.Core/Services/Agents/Credentials/Broker/LoopbackModelCredentialBroker.cs

@@ -173,7 +173,7 @@ private static HttpMessageHandler DefaultUpstreamHandler() =>
         _logger.LogDebug("Model credential brokered for agent run {RunId} on port {Port} (team {TeamId}, epoch {Epoch}) until {ExpiresAt:O}", lease.RunId, lease.Port, lease.TeamId, lease.Epoch, lease.ExpiresAt);
         WarnIfUnreachableFromNetns(lease, bound.Host);
 
-        return Task.FromResult<BrokeredModelCredential?>(new(BaseUrlFor(lease), lease.Token, lease.ExpiresAt) { RebindPort = lease.Port, RebindRoute = lease.PathId });
+        return Task.FromResult<BrokeredModelCredential?>(new(BaseUrlFor(lease), lease.Token, lease.ExpiresAt) { RebindPort = lease.Port, RebindRoute = lease.PathId, ReachableFromNamespace = bound.Host == AnyHost });
     }
 
     /// <summary>

backend/src/CodeSpace.Core/Services/Agents/Sandbox/Exceptions/SealedEgressUnavailableException.cs

@@ -0,0 +1,50 @@
+using CodeSpace.Messages.Failures;
+
+namespace CodeSpace.Core.Services.Agents.Sandbox.Exceptions;
+
+/// <summary>
+/// A network-off run whose model is brokered, REFUSED because this host would confine it but cannot seal its network
+/// to that broker. Severing it instead — what a host that cannot seal would otherwise do — cuts the broker off along
+/// with everything else, so the agent reaches no model and the run burns its whole timeout and the CLI's retries on
+/// failures that read like a provider outage. Refusing names the wall instead, and does it before anything is spent.
+///
+/// <para>Unavailable, like <see cref="EgressSubnetReservationUnavailableException"/>: nothing about the launch can be
+/// changed to make it work, and the identical launch succeeds untouched once an operator grants the worker what a
+/// sealed namespace needs — or, for a setup step that failed on a host that can seal (<see cref="SetupFailed"/>),
+/// fixes what that step names. <see cref="Cause"/> says which it was, in the same words the message uses.</para>
+/// </summary>
+public sealed class SealedEgressUnavailableException : Exception, IFailure
+{
+    /// <summary>The worker lacks the <c>ip</c> or <c>nft</c> binary a sealed namespace is built with.</summary>
+    public const string CauseMissingTools = "ip or nft is not installed on this worker";
+
+    /// <summary>The binaries are there, but this process could not build a throwaway namespace.</summary>
+    public const string CauseNoPrivilege = "this worker may not create a network namespace (it needs root with CAP_NET_ADMIN and CAP_SYS_ADMIN)";
+
+    /// <summary>The run's model broker could only listen on loopback, which a sealed namespace cannot reach.</summary>
+    public const string CauseBrokerLoopbackOnly = "the run's model broker could only listen on loopback, which a sealed namespace cannot reach";
+
+    /// <summary>What an operator does about a worker that cannot build a sealed namespace at all.</summary>
+    private const string GrantRemedy = "Grant the worker what a sealed namespace needs (see backend/Dockerfile.worker, EGRESS FILTERING); a retry on this host helps only once it can build one, which it re-checks at most once a minute.";
+
+    /// <summary>What an operator does about one setup step that failed on a worker that can build a sealed namespace.</summary>
+    private const string SetupRemedy = "This worker can build a sealed namespace, but a step of this one failed on its host: fix what that step names (a route or policy rule that discards the run's /30, or a namespace or veth left behind under the run's name). Every launch runs the setup afresh.";
+
+    public SealedEgressUnavailableException(string cause) : this(cause, GrantRemedy) { }
+
+    private SealedEgressUnavailableException(string cause, string remedy)
+        : base($"This run's network is off and its model is reached through its broker; on this worker that is enforced with a network namespace sealed to that broker, but one cannot be built here: {cause}. Refusing to launch an agent that could not reach its model. {remedy}")
+    {
+        Cause = cause;
+    }
+
+    /// <summary>A sealed setup that failed at launch on a host that proved it can seal — a name collision, a route the kernel will not send the run's replies down — carrying the failed step's own error.</summary>
+    public static SealedEgressUnavailableException SetupFailed(string? setupError) => new($"the sealed namespace's setup failed: {setupError}", SetupRemedy);
+
+    /// <summary>Which wall the host hit — one of the <c>Cause*</c> constants, or a failed setup step's own error.</summary>
+    public string Cause { get; }
+
+    FailureKind IFailure.Kind => FailureKind.Unavailable;
+    string IFailure.Code => FailureCodes.SandboxSealedEgressUnavailable;
+    string? IFailure.ClientMessage => "This host cannot give a network-off run a sealed route to its model.";
+}

backend/src/CodeSpace.Core/Services/Agents/Sandbox/ISandboxEgressAdmission.cs

@@ -0,0 +1,24 @@
+using CodeSpace.Messages.Agents;
+
+namespace CodeSpace.Core.Services.Agents.Sandbox;
+
+/// <summary>
+/// Optional capability a sandbox runner MAY implement alongside <see cref="ISandboxRunner"/> (Rule 7 / ISP — a sibling
+/// interface, never a widening of the base contract): refuse, BEFORE anything is spent, a spec this runner could only
+/// launch with a network that leaves the agent unable to do its work. The runner is the one place that knows what it
+/// can actually build on this host, and the executor asks it at the one moment a refusal still costs nothing — after
+/// the spec is built, before the spend is admitted and the process started.
+///
+/// <para>The first such spec is a network-off run whose model is brokered (<see cref="SandboxSpec.ModelBrokerPort"/>)
+/// on a host that confines but cannot seal: severing it would cut its broker off too. A runner without this capability
+/// simply launches, exactly as it always has.</para>
+/// </summary>
+public interface ISandboxEgressAdmission
+{
+    /// <summary>
+    /// Throws an <see cref="CodeSpace.Messages.Failures.IFailure"/> naming the wall when this runner cannot give
+    /// <paramref name="spec"/> the egress it needs; returns otherwise. <paramref name="modelBrokerReachableFromNamespace"/>
+    /// is whether the run's broker listens where a per-run namespace can reach it (<c>BrokeredModelCredential.ReachableFromNamespace</c>).
+    /// </summary>
+    void EnsureEgressAdmissible(SandboxSpec spec, bool modelBrokerReachableFromNamespace);
+}

backend/src/CodeSpace.Core/Services/Agents/Sandbox/Runners/LocalProcessRunner.Durable.cs

@@ -4,6 +4,7 @@
 using System.Text;
 using CodeSpace.Core.Services.Agents.AgentRunLogging;
 using CodeSpace.Core.Services.Agents.Mcp;
+using CodeSpace.Core.Services.Agents.Sandbox.Exceptions;
 using CodeSpace.Core.Services.Agents.Sandbox.Isolation;
 using CodeSpace.Messages.Agents;
 using CodeSpace.NativeLaunch;
@@ -354,12 +355,36 @@ private static bool TryFileLength(string path, out long length)
     {
         var setup = await FilteredEgressNetns.SetupSealedAsync(spoolKey, brokerPort, EgressSetupTimeoutSeconds, ct).ConfigureAwait(false);
 
+        // The same refusal EnsureEgressAdmissible raises before any spend, for the rarer case the probe could not
+        // foresee — a setup step that fails on a host that proved it can seal (a name collision, a kernel refusal).
         if (!setup.SetupOk)
-            throw new InvalidOperationException($"Sealed-egress netns setup failed (fail-closed — run aborted rather than launched with a network it was not given): {setup.SetupError}");
+            throw SealedEgressUnavailableException.SetupFailed(setup.SetupError);
 
         return (setup.ExecPrefix, spoolKey, setup.HostIp);
     }
 
+    /// <summary>
+    /// Refuse, before anything is spent, a network-off brokered run this host would confine but cannot seal — the
+    /// mirror of <see cref="SealableBrokerPort"/>, which would otherwise quietly sever it from its broker. A spec with
+    /// no broker port, or a host that does not confine, is admitted untouched: nothing about its launch changes.
+    /// </summary>
+    public void EnsureEgressAdmissible(SandboxSpec spec, bool modelBrokerReachableFromNamespace)
+    {
+        if (spec.ModelBrokerPort is null || BubblewrapSandbox.Available is null) return;
+
+        if (SealRefusal(FilteredEgressNetns.IsSupported, FilteredEgressNetns.CanSeal, modelBrokerReachableFromNamespace) is not { } cause) return;
+
+        // The probe keeps the step that failed and its output; an operator needs that more than the category.
+        throw new SealedEgressUnavailableException(cause == SealedEgressUnavailableException.CauseNoPrivilege && FilteredEgressNetns.SealUnavailableReason is { } probe ? $"{cause}; the probe: {probe}" : cause);
+    }
+
+    /// <summary>Why a confining host cannot seal a brokered network-off run, or null when it can. Pure over the host's three facts, so every cause is testable on a host that has none of them.</summary>
+    internal static string? SealRefusal(bool haveTools, bool canSeal, bool brokerReachableFromNamespace) =>
+        !haveTools ? SealedEgressUnavailableException.CauseMissingTools
+        : !canSeal ? SealedEgressUnavailableException.CauseNoPrivilege
+        : !brokerReachableFromNamespace ? SealedEgressUnavailableException.CauseBrokerLoopbackOnly
+        : null;
+
     /// <summary>
     /// Create this run's cgroup-v2 resource-cap leaf (B4) when a memory/cpu cap is requested AND the operator delegated
     /// a root (<see cref="CgroupResourceLimit.CgroupRoot"/>) on a cgroup-v2 host — returning the self-add prefix the

backend/src/CodeSpace.Core/Services/Agents/Sandbox/Runners/LocalProcessRunner.cs

@@ -23,7 +23,7 @@ namespace CodeSpace.Core.Services.Agents.Sandbox.Runners;
 /// Caller cancellation is honoured distinctly from the spec timeout: it terminates the process and rethrows
 /// (the durable path differs — see its remarks: cancellation stops observing without killing).
 /// </summary>
-public sealed partial class LocalProcessRunner : ISandboxRunner, ISandboxStreamRunner, ISandboxDurableRunner, ISandboxLaunchIdentityRunner, ISandboxDurableLogSource, ISandboxDurableDiagnosticSource, ISingletonDependency
+public sealed partial class LocalProcessRunner : ISandboxRunner, ISandboxStreamRunner, ISandboxDurableRunner, ISandboxLaunchIdentityRunner, ISandboxDurableLogSource, ISandboxDurableDiagnosticSource, ISandboxEgressAdmission, ISingletonDependency
 {
     /// <summary>This runner's registry key. The runner-local spelling of the shared <see cref="SandboxKinds.Local"/> — same constant, so there is one literal.</summary>
     public const string LocalKind = SandboxKinds.Local;

backend/src/CodeSpace.Core/Services/Supervisor/Deciders/LlmSupervisorDecider.cs

@@ -1774,6 +1774,8 @@ private static void AppendFailedVerdict(StringBuilder builder, SupervisorAgentRe
             "RETRY this exact subtask so it runs on a live worker; do NOT re-plan it and do NOT amend its check — there is nothing wrong with either.",
         Messages.Failures.FailureCodes.ModelCredentialBrokerUnavailable =>
             "RETRY this exact subtask once, in case another worker can broker its model credential; if it ends the same way again, 'ask_human' — that is a deployment setting only an operator can change. Either way do NOT re-plan it and do NOT amend its check — there is nothing wrong with either.",
+        Messages.Failures.FailureCodes.SandboxSealedEgressUnavailable =>
+            "RETRY this exact subtask once, in case another worker can seal its network to its model broker; if it ends the same way again, 'ask_human' — that is a deployment setting only an operator can change. Either way do NOT re-plan it and do NOT amend its check — there is nothing wrong with either.",
         _ => "This is an infrastructure fault with no recorded remedy: 'ask_human' to rule. Do NOT re-plan it and do NOT amend its check — neither is where the fault is.",
     };
 

backend/src/CodeSpace.Messages/Agents/BrokeredModelCredential.cs

@@ -33,4 +33,12 @@ public sealed record BrokeredModelCredential(string BaseUrl, string RunToken, Da
 
     /// <summary>The unguessable route segment of <see cref="BaseUrl"/>, for the same reason as <see cref="RebindPort"/>: a re-bind has to install the run's OWN route, never mint a fresh one, or the address the agent holds resolves to nothing. Null exactly when <see cref="RebindPort"/> is.</summary>
     public string? RebindRoute { get; init; }
+
+    /// <summary>
+    /// Whether the lease listens on every address, so a child inside a per-run network namespace — which reaches the
+    /// worker at its namespace gateway, never on loopback — can reach it. False (the default) is the fail-closed
+    /// answer: a broker that could only bind loopback, or one that does not say, cannot serve a sealed network-off run,
+    /// and such a run is refused before launch rather than left calling an address nothing answers.
+    /// </summary>
+    public bool ReachableFromNamespace { get; init; }
 }