Seal a network-off run to its model broker

.github/workflows/sandbox-isolation.yml

@@ -224,8 +224,8 @@ jobs:
           executed=$(grep -oE 'executed="[0-9]+"' "$trx" | head -1 | grep -oE '[0-9]+')
           passed=$(grep -oE 'passed="[0-9]+"' "$trx" | head -1 | grep -oE '[0-9]+')
           echo "executed=${executed:-0} passed=${passed:-0}"
-          if [ "${executed:-0}" -lt 59 ]; then
-            echo "::error::Expected >=59 sandbox isolation tests to run (bwrap/prlimit confinement + cap-drop + cgroup-namespace re-root + egress-allowlist filter + cgroup resource cap + durable-launch cgroup wiring + argv/envp per-string kernel ceiling + a prompt past it riding stdin + a read-only workspace mount + a read-only reviewer reading its diff with the real CLIs), but only ${executed:-0} did — the Category=Sandbox filter matched too few (trait regression?). If a case was deliberately removed, lower this number in the same PR."
+          if [ "${executed:-0}" -lt 65 ]; then
+            echo "::error::Expected >=65 sandbox isolation tests to run (bwrap/prlimit confinement + cap-drop + cgroup-namespace re-root + egress-allowlist filter + cgroup resource cap + durable-launch cgroup wiring + argv/envp per-string kernel ceiling + a prompt past it riding stdin + a read-only workspace mount + a network-off run sealed to its broker + a read-only reviewer reading its diff with the real CLIs), but only ${executed:-0} did — the Category=Sandbox filter matched too few (trait regression?). If a case was deliberately removed, lower this number in the same PR."
             exit 1
           fi
 
@@ -249,14 +249,22 @@ jobs:
           # The reviewer E2E returns early when it is not armed, and an early return reads as Passed — so a passing
           # outcome is not evidence it ran. Each arm prints a marker when it really did; require every one.
           text = open('backend/TestResults/sandbox.trx', encoding='utf-8').read()
-          arms = ('read-diff claude-code Confined', 'read-diff codex-cli Confined', 'read-diff codex-cli Standard', 'codex-resume-stand-down', 'write-refused claude-code', 'write-refused codex-cli', 'standard-codex-writes', 'network-off claude-code', 'network-off codex-cli')
+          arms = ('read-diff claude-code Confined', 'read-diff codex-cli Confined', 'read-diff codex-cli Standard', 'codex-resume-stand-down', 'write-refused claude-code', 'write-refused codex-cli', 'standard-codex-writes', 'network-off-sealed claude-code', 'network-off-sealed codex-cli')
           for arm in arms:
               assert f'[review-diff-e2e] ran {arm}' in text, f'reviewer E2E arm "{arm}" did not run — check CODESPACE_REQUIRE_REVIEW_CLIS and the CLI install step'
-          for method, rows in (('A_read_only_claude_reviewer_reads_the_diff_between_two_commits_with_git', 1), ('A_codex_reviewer_reads_the_diff_with_git_wherever_it_runs', 2), ('The_pinned_codex_accepts_the_resume_spelling_of_its_stand_down', 1), ('A_confined_reviewer_cannot_write_its_workspace', 2), ('A_standard_codex_writes_its_workspace_under_our_confinement_but_not_the_system_root', 1), ('A_network_off_reviewer_under_confinement_cannot_reach_its_model', 2)):
+          for method, rows in (('A_read_only_claude_reviewer_reads_the_diff_between_two_commits_with_git', 1), ('A_codex_reviewer_reads_the_diff_with_git_wherever_it_runs', 2), ('The_pinned_codex_accepts_the_resume_spelling_of_its_stand_down', 1), ('A_confined_reviewer_cannot_write_its_workspace', 2), ('A_standard_codex_writes_its_workspace_under_our_confinement_but_not_the_system_root', 1), ('A_network_off_reviewer_reaches_its_model_through_the_sealed_namespace', 2)):
               cases = [r for r in results if 'ReviewerReadsItsDiffE2ETests.' + method in r.get('testName', '')]
               assert len(cases) == rows and all(r.get('outcome') == 'Passed' for r in cases), f'{method}: all {rows} case(s) must pass'
           print(f'All {len(arms)} reviewer E2E arms ran and passed.')
 
+          # The sealed-egress E2E returns early on a host that cannot seal, which reads as Passed; require each arm's marker.
+          for arm in ('durable', 'non-durable', 'ipv6', 'restart-reissue', 'policy-route-discard'):
+              assert f'[sealed-egress-e2e] ran {arm}' in text, f'sealed-egress E2E arm "{arm}" did not run — this lane is root with bwrap, ip and nft, so it must seal'
+          for method, rows in (('A_network_off_brokered_run_reaches_its_broker_and_nothing_else', 2), ('A_sealed_namespace_drops_the_gateway_over_ipv6_link_local_too', 1), ('A_30_still_held_by_a_run_that_outlived_its_worker_is_not_handed_to_the_next_run', 1), ('A_host_whose_policy_rule_discards_the_run_s_replies_fails_the_setup_and_leaks_nothing', 1)):
+              cases = [r for r in results if 'SealedEgressE2ETests.' + method in r.get('testName', '')]
+              assert len(cases) == rows and all(r.get('outcome') == 'Passed' for r in cases), f'{method}: all {rows} case(s) must pass'
+          print('All 5 sealed-egress arms ran and passed.')
+
           print('All 10 batch/stream kernel cases passed.')
           PY
 

backend/Dockerfile.worker

@@ -31,6 +31,11 @@
 # CAP_NET_ADMIN (and run the egress path privileged enough to write the sysctl) on any deployment that relies on
 # allowlisted egress; a pod that does not is still correct, it just gets Denied instead of Filtered.
 #
+# The same namespace machinery SEALS a network-off run whose model is brokered to that broker (no route, no NAT, no
+# DNS, one gateway port). It is taken only where bubblewrap confines AND FilteredEgressNetns.CanSeal has proved, by
+# building a throwaway namespace, that this process may: root + CAP_NET_ADMIN + CAP_SYS_ADMIN, no sysctl needed. A
+# pod without them keeps severing such a run — which also severs it from its broker, so it reaches no model.
+#
 # CONFINEMENT ARMING: bubblewrap is installed here so the capability is PRESENT, but the fail-closed guard
 # Sandbox:RequireConfinement is left to the DEPLOYMENT to arm (k8s pod / compose) on a host that grants user
 # namespaces — hardcoding it in the image would fail-close every run on an environment where in-container userns

backend/src/CodeSpace.Core/Services/Agents/AgentAutonomyPolicy.cs

@@ -179,6 +179,9 @@ public static string DescribeNetwork(AgentAutonomyLevel effective, AgentAutonomy
         return WithModelCredentialPosture($"Network: off ({effective}){qualifier}", confinement);
     }
 
+    /// <summary>What an "off" run sealed to its model broker adds to the sentence: the one route it kept.</summary>
+    public const string SealedToBrokerQualifier = " — confined: egress sealed to the run's model broker";
+
     /// <summary>The write posture derived from the same permission row the process runner receives. Before launch there is no confinement record, so read-only is explicitly qualified rather than presented as an OS guarantee.</summary>
     public static string DescribeWrite(AgentAutonomyLevel effective) => Derive(effective).WriteScope switch
     {
@@ -230,11 +233,13 @@ public static string DescribeApproval(AgentAutonomyLevel effective)
     /// materially different fact from a severed namespace and must not read like a milder version of it.
     /// <see cref="SandboxConfinementOutcome.NotApplicable"/> is such a run too — no confinement was even attempted.
     /// Both unconfined verdicts also carry <see cref="UnconfinedIsolationCaveat"/>: egress is the loudest thing an
-    /// unconfined run loses, but not the only one.
+    /// unconfined run loses, but not the only one. A run sealed to its broker holds a per-run /30 exactly as an
+    /// allowlisted one does, so it carries the same host subnet caveat where this host has proved it.
     /// </summary>
     private static string OffQualifier(SandboxConfinement? confinement) => confinement switch
     {
         null => ConfinementCaveat,
+        { Outcome: SandboxConfinementOutcome.Confined, EgressSealedToBroker: true } => WithHostSubnetPosture(SealedToBrokerQualifier, EgressSubnetAllocator.ObservedHostDegradation),
         { Outcome: SandboxConfinementOutcome.Confined, NetworkSevered: true } => " — confined: egress severed",
         { Outcome: SandboxConfinementOutcome.Confined } => " — confined, but egress was NOT severed",
         { Outcome: SandboxConfinementOutcome.Unconfined } c => $" — OFF REQUESTED BUT UNCONFINED: this host cannot sever egress ({c.Reason ?? "unknown"}){UnconfinedIsolationCaveat}",

backend/src/CodeSpace.Core/Services/Agents/AgentRunExecutor.cs

@@ -460,7 +460,9 @@
             // receives the governed tools the endpoint serves (today the harness projects ONLY task.Tools, so a restricted
             // run couldn't call them). Additive + tier-filtered; a no-op when the author named no tools (the CLI default
             // already reaches a declared MCP server's tools). Drives BuildInvocation off the augmented task.
-            SandboxSpec BuildSpec(AgentTask built) => HardenSpec(harness.BuildInvocation(AugmentToolsForMcp(built, mcp, mcpWiring)) with { Mcp = mcpWiring }, built, modelBaseUrl, modelProvider, workspaceProvision);
+            var hardening = new SpecHardening(modelBaseUrl, modelProvider, workspaceProvision, brokeredCredential?.RebindPort);
+
+            SandboxSpec BuildSpec(AgentTask built) => HardenSpec(harness.BuildInvocation(AugmentToolsForMcp(built, mcp, mcpWiring)) with { Mcp = mcpWiring }, built, hardening);
 
             var spec = BuildSpec(effectiveTask);
 
@@ -3764,9 +3766,9 @@
 
         // The non-secret base URL + provider tag flow out so a restricted (Allowlist) run can pin its model-API host
         // in the egress allowlist (B3.3b) — the UPSTREAM ones even under brokerage, deliberately: the allowlist is
-        // enforced inside the run's netns, the broker reaches the provider from the host outside it, and narrowing
-        // the allowlist to just the broker is a separate change (a brokered run keeping the provider host reachable
-        // loses nothing — the token it holds is refused there). DefaultModel flows out so a model-less ("auto") run
+        // enforced inside the run's netns, the broker reaches the provider from the host outside it, and a brokered
+        // run keeping the provider host reachable loses nothing — the token it holds is refused there. (A network-OFF
+        // brokered run is the one sealed to just its broker; see ApplySealedEgress.) DefaultModel flows out so a model-less ("auto") run
         // falls back to one of the credential's own models instead of the CLI default. All null when no credential
         // resolved. CredentialId names the ROW whose key this run authenticates with (null for the operator-global
         // key, which has no row) — D3 bounds an escalation's candidate models to exactly that row.
@@ -4046,9 +4048,25 @@
     internal static SandboxSpec ApplyWriteScope(SandboxSpec spec, AgentPermissions permissions) =>
         spec with { ReadOnlyWorkingDirectory = permissions.WriteScope != AgentWriteScope.Workspace };
 
-    /// <summary>The harness invocation with every one of the executor's own spec post-processings applied — the egress posture, the write scope and the tier's resource ceilings. One name so the launch and each revise round cannot drift apart on which hardening they got.</summary>
-    private static SandboxSpec HardenSpec(SandboxSpec spec, AgentTask task, string? modelBaseUrl, string? modelProvider, WorkspaceProvisionRequest? workspace) =>
-        ApplyResourceCeilings(ApplyWriteScope(ApplyEgressPolicy(spec, task.Permissions, modelBaseUrl, modelProvider, workspace), task.Permissions), task.Autonomy, RuntimeSettings.Current.AgentMemoryCeilingMb);
+    /// <summary>
+    /// Stamp a network-off run's broker port onto its spec, so a confining runner able to build one runs it in a
+    /// namespace SEALED to that broker instead of severing it from everything — the broker included, which left such a
+    /// run unable to reach any model. Only network-off runs: a run with network reaches its broker already, and its
+    /// egress is <see cref="ApplyEgressPolicy"/>'s business. Returns the spec itself when there is nothing to stamp.
+    /// </summary>
+    internal static SandboxSpec ApplySealedEgress(SandboxSpec spec, AgentPermissions permissions, int? brokerPort) =>
+        permissions.Network == AgentNetworkAccess.Off && brokerPort is { } port ? spec with { ModelBrokerPort = port } : spec;
+
+    /// <summary>What the executor's hardening reads beyond the task itself: the model endpoint and the workspace the egress allowlist is built from, and the port of the run's brokered model lease, if it has one.</summary>
+    private readonly record struct SpecHardening(string? ModelBaseUrl, string? ModelProvider, WorkspaceProvisionRequest? Workspace, int? ModelBrokerPort);
+
+    /// <summary>The harness invocation with every one of the executor's own spec post-processings applied — the egress posture, the broker seal, the write scope and the tier's resource ceilings. One name so the launch and each revise round cannot drift apart on which hardening they got.</summary>
+    private static SandboxSpec HardenSpec(SandboxSpec spec, AgentTask task, SpecHardening hardening)
+    {
+        var egress = ApplySealedEgress(ApplyEgressPolicy(spec, task.Permissions, hardening.ModelBaseUrl, hardening.ModelProvider, hardening.Workspace), task.Permissions, hardening.ModelBrokerPort);
+
+        return ApplyResourceCeilings(ApplyWriteScope(egress, task.Permissions), task.Autonomy, RuntimeSettings.Current.AgentMemoryCeilingMb);
+    }
 
     /// <summary>The git clone URLs of every repo in the run's workspace provision (empty for a no-repo run) — the source of the allowlist's git hosts.</summary>
     private static IReadOnlyList<string> CloneUrlsOf(WorkspaceProvisionRequest? workspace) =>
@@ -4939,10 +4957,10 @@
     /// <summary>The same reconstruction from a payload that came from somewhere other than the row — an offloaded one fetched back out of the artifact store.</summary>
     private static AgentEvent ReplayedEvent(AgentEventKind kind, string? text, string? dataJson)
     {
         if (dataJson is not { Length: > 0 } json) return new AgentEvent { Kind = kind, Text = text };
 
         try { using var doc = JsonDocument.Parse(json); return new AgentEvent { Kind = kind, Text = text, Data = doc.RootElement.Clone() }; }
         catch (JsonException) { return new AgentEvent { Kind = kind, Text = text }; }
     }
 
     /// <summary>Ask the row, on a token of its own, whether the run actually reached a terminal state — the only honest answer to "did the landing take?" once an exception has been raised somewhere after the fenced write.</summary>

backend/src/CodeSpace.Core/Services/Agents/Sandbox/Isolation/BubblewrapSandbox.cs

@@ -78,13 +78,17 @@ public static void EnsureSatisfiable(string? available, bool required)
     /// <see cref="EgressFor"/> derivation <see cref="BuildArgs"/> turns into <c>--unshare-net</c> — so the record can
     /// neither claim a severance the argv did not request NOR miss one it did (an unenforceable allowlist fails
     /// closed to severed even while the launch asked to share the network).</para>
+    ///
+    /// <para><paramref name="sealedToBroker"/> says the shared network IS a sealed namespace whose only destination is
+    /// the run's model broker: severed from everything else, so recorded as severed — and as sealed, so a reader knows
+    /// the one route it kept.</para>
     /// </summary>
-    public static SandboxConfinement DeriveConfinement(string? available, string? unavailableReason, bool shareNetwork, IReadOnlyList<string>? egressAllowlist)
+    public static SandboxConfinement DeriveConfinement(string? available, string? unavailableReason, bool shareNetwork, IReadOnlyList<string>? egressAllowlist, bool sealedToBroker = false)
     {
         if (available is null)
             return new SandboxConfinement { Outcome = SandboxConfinementOutcome.Unconfined, Reason = unavailableReason ?? SandboxConfinement.ReasonNoBubblewrap };
 
-        return new SandboxConfinement { Outcome = SandboxConfinementOutcome.Confined, NetworkSevered = EgressFor(shareNetwork, egressAllowlist).Mode != SandboxEgressMode.Full };
+        return new SandboxConfinement { Outcome = SandboxConfinementOutcome.Confined, NetworkSevered = sealedToBroker || EgressFor(shareNetwork, egressAllowlist).Mode != SandboxEgressMode.Full, EgressSealedToBroker = sealedToBroker };
     }
 
     /// <summary>

backend/src/CodeSpace.Core/Services/Agents/Sandbox/Isolation/CapabilityProbe.cs

@@ -0,0 +1,60 @@
+namespace CodeSpace.Core.Services.Agents.Sandbox.Isolation;
+
+/// <summary>
+/// Keeps the answer to a question this process can only settle by trying — are <c>ip</c> and <c>nft</c> runnable
+/// (<see cref="FilteredEgressNetns.IsSupported"/>), can it build a sealed namespace (<see cref="FilteredEgressNetns.CanSeal"/>).
+/// A proof holds for the process. A failure may be transient — a fork that failed once, a slow first mount of
+/// <c>/run/netns</c>, rtnl held by a burst of teardowns — so it stands for one retry interval and is then probed
+/// again, with its reason kept for whoever reports what could not be done.
+///
+/// <para>The FIRST probe is waited for by every caller: a fresh worker that may well seal must not refuse its first
+/// launches while it finds out. A RE-probe is made by one caller outside the lock while the others take the standing
+/// failure — which they would have got anyway — instead of queueing behind a probe that can take tens of seconds on a
+/// host that keeps failing slowly. Timed on a monotonic clock, so a wall-clock step cannot hold a failure forever or
+/// re-probe on every call.</para>
+/// </summary>
+internal sealed class CapabilityProbe(Func<string?> probe, Func<TimeSpan> monotonicNow, TimeSpan retryInterval)
+{
+    private readonly object _lock = new();
+    private bool _probed;
+    private bool _proven;
+    private bool _probing;
+    private TimeSpan _retryAt;
+    private string? _reason;
+
+    public bool Holds
+    {
+        get
+        {
+            lock (_lock)
+            {
+                if (_proven) return true;
+                if (!_probed) return Record(probe());
+                if (_probing || monotonicNow() < _retryAt) return false;
+
+                _probing = true;
+            }
+
+            var reason = probe();
+
+            lock (_lock)
+            {
+                _probing = false;
+                return Record(reason);
+            }
+        }
+    }
+
+    /// <summary>Why the last probe failed, or null when none has failed since the last proof.</summary>
+    public string? UnavailableReason { get { lock (_lock) return _reason; } }
+
+    private bool Record(string? reason)
+    {
+        _probed = true;
+        _reason = reason;
+        _proven = reason is null;
+        _retryAt = monotonicNow() + retryInterval;
+
+        return _proven;
+    }
+}