diff --git a/.github/workflows/backend-e2e.yml b/.github/workflows/backend-e2e.yml index 74f1a88e0..5a741b492 100644 --- a/.github/workflows/backend-e2e.yml +++ b/.github/workflows/backend-e2e.yml @@ -177,8 +177,9 @@ jobs: # The headline E2E spawns a REAL agent process through the production runner, which wraps it in # bubblewrap + prlimit (the production confinement posture). Running as root in the SDK container, so # no sudo. ca-certificates so the container can fetch packages; util-linux for prlimit; iproute2 and - # nftables so a network-off brokered agent is SEALED to its broker, as a confining host must, rather than - # refused as sandbox_sealed_egress_unavailable. + # nftables because the worker image ships them for allowlist runs. A network-off brokered agent needs + # neither: it reaches its broker through the codespace-mcp relay, which the test project's build places + # beside its assembly, and without which it is refused as sandbox_sealed_egress_unavailable. run: | apt-get update apt-get install -y --no-install-recommends bubblewrap util-linux ca-certificates iproute2 nftables diff --git a/.github/workflows/sandbox-isolation.yml b/.github/workflows/sandbox-isolation.yml index 6134616dc..d8a3186c0 100644 --- a/.github/workflows/sandbox-isolation.yml +++ b/.github/workflows/sandbox-isolation.yml @@ -224,8 +224,8 @@ jobs: executed=$(grep -oE 'executed="[0-9]+"' "$trx" | head -1 | grep -oE '[0-9]+') passed=$(grep -oE 'passed="[0-9]+"' "$trx" | head -1 | grep -oE '[0-9]+') echo "executed=${executed:-0} passed=${passed:-0}" - if [ "${executed:-0}" -lt 74 ]; then - echo "::error::Expected >=74 sandbox isolation tests to run (bwrap/prlimit confinement + cap-drop + cgroup-namespace re-root + egress-allowlist filter + cgroup resource cap + durable-launch cgroup wiring + argv/envp per-string kernel ceiling + a prompt past it riding stdin + a read-only workspace mount + a network-off run sealed to its broker + a read-only reviewer reading its diff with the real CLIs + a bwrap probe that runs the launch argv + the MCP helper bound file by file behind a read-only socket dir + a CLI reaching its broker socket through the relay + a severed child reaching its broker over the lease socket across a worker restart), but only ${executed:-0} did — the Category=Sandbox filter matched too few (trait regression?). If a case was deliberately removed, lower this number in the same PR." + if [ "${executed:-0}" -lt 76 ]; then + echo "::error::Expected >=76 sandbox isolation tests to run (bwrap/prlimit confinement + cap-drop + cgroup-namespace re-root + egress-allowlist filter + cgroup resource cap + durable-launch cgroup wiring + argv/envp per-string kernel ceiling + a prompt past it riding stdin + a read-only workspace mount + a network-off run reaching its broker through the relay and nothing else + an allowlist run relayed to its broker + a read-only reviewer reading its diff with the real CLIs + a bwrap probe that runs the launch argv + the MCP helper bound file by file behind a read-only socket dir + a CLI reaching its broker socket through the relay + a severed child reaching its broker over the lease socket across a worker restart + a pre-relay namespaced run re-bound at its gateway and torn down with its seal), but only ${executed:-0} did — the Category=Sandbox filter matched too few (trait regression?). If a case was deliberately removed, lower this number in the same PR." exit 1 fi @@ -249,21 +249,35 @@ jobs: # The reviewer E2E returns early when it is not armed, and an early return reads as Passed — so a passing # outcome is not evidence it ran. Each arm prints a marker when it really did; require every one. text = open('backend/TestResults/sandbox.trx', encoding='utf-8').read() - arms = ('read-diff claude-code Confined', 'read-diff codex-cli Confined', 'read-diff codex-cli Standard', 'codex-resume-stand-down', 'write-refused claude-code', 'write-refused codex-cli', 'standard-codex-writes', 'network-off-sealed claude-code', 'network-off-sealed codex-cli') + arms = ('read-diff claude-code Confined', 'read-diff codex-cli Confined', 'read-diff codex-cli Standard', 'codex-resume-stand-down', 'write-refused claude-code', 'write-refused codex-cli', 'standard-codex-writes', 'network-off-relayed claude-code', 'network-off-relayed codex-cli') for arm in arms: assert f'[review-diff-e2e] ran {arm}' in text, f'reviewer E2E arm "{arm}" did not run — check CODESPACE_REQUIRE_REVIEW_CLIS and the CLI install step' - for method, rows in (('A_read_only_claude_reviewer_reads_the_diff_between_two_commits_with_git', 1), ('A_codex_reviewer_reads_the_diff_with_git_wherever_it_runs', 2), ('The_pinned_codex_accepts_the_resume_spelling_of_its_stand_down', 1), ('A_confined_reviewer_cannot_write_its_workspace', 2), ('A_standard_codex_writes_its_workspace_under_our_confinement_but_not_the_system_root', 1), ('A_network_off_reviewer_reaches_its_model_through_the_sealed_namespace', 2)): + for method, rows in (('A_read_only_claude_reviewer_reads_the_diff_between_two_commits_with_git', 1), ('A_codex_reviewer_reads_the_diff_with_git_wherever_it_runs', 2), ('The_pinned_codex_accepts_the_resume_spelling_of_its_stand_down', 1), ('A_confined_reviewer_cannot_write_its_workspace', 2), ('A_standard_codex_writes_its_workspace_under_our_confinement_but_not_the_system_root', 1), ('A_network_off_reviewer_reaches_its_model_through_the_relay', 2)): cases = [r for r in results if 'ReviewerReadsItsDiffE2ETests.' + method in r.get('testName', '')] assert len(cases) == rows and all(r.get('outcome') == 'Passed' for r in cases), f'{method}: all {rows} case(s) must pass' print(f'All {len(arms)} reviewer E2E arms ran and passed.') - # The sealed-egress E2E returns early on a host that cannot seal, which reads as Passed; require each arm's marker. - for arm in ('durable', 'non-durable', 'ipv6', 'restart-reissue', 'policy-route-discard-dst', 'policy-route-discard-l4', 'setup-failure'): - assert f'[sealed-egress-e2e] ran {arm}' in text, f'sealed-egress E2E arm "{arm}" did not run — this lane is root with bwrap, ip and nft, so it must seal' - for method, rows in (('A_network_off_brokered_run_reaches_its_broker_and_nothing_else', 2), ('A_sealed_namespace_drops_the_gateway_over_ipv6_link_local_too', 1), ('A_30_still_held_by_a_run_that_outlived_its_worker_is_not_handed_to_the_next_run', 1), ('A_host_whose_policy_rule_discards_the_run_s_replies_fails_the_setup_and_leaks_nothing', 2), ('A_sealed_setup_that_fails_on_this_host_refuses_the_launch_typed_and_leaks_nothing', 1)): + # The sealed-egress E2E returns early on a host that cannot confine, which reads as Passed; require each arm's marker. + for arm in ('durable', 'non-durable', 'relay-ipv6', 'restart', 'relay-refused', 'relay-policy-route'): + assert f'[sealed-egress-e2e] ran {arm}' in text, f'sealed-egress E2E arm "{arm}" did not run — this lane is root with bwrap and the relay helper, so it must relay' + for method, rows in (('A_network_off_brokered_run_reaches_its_broker_and_nothing_else', 2), ('A_relayed_run_has_no_ipv6_path_to_the_worker_either', 1), ('The_same_live_agent_reaches_the_next_worker_through_its_socket_after_a_restart', 1), ('A_brokered_child_the_worker_cannot_relay_is_refused_and_would_have_reached_nothing', 1), ('A_host_policy_rule_that_discards_replies_from_the_broker_port_cannot_reach_a_relayed_run', 1)): cases = [r for r in results if 'SealedEgressE2ETests.' + method in r.get('testName', '')] assert len(cases) == rows and all(r.get('outcome') == 'Passed' for r in cases), f'{method}: all {rows} case(s) must pass' - print('All 7 sealed-egress arms ran and passed.') + print('All 6 sealed-egress arms ran and passed.') + + # The allowlist plan's host-routing arms, and the relayed allowlist launch, return early without ip and nft; require their markers. + for marker in ('[filtered-egress-e2e] ran restart-reissue', '[filtered-egress-e2e] ran policy-route-discard-dst', '[durable-egress-e2e] ran allowlist-relay'): + assert marker in text, f'"{marker}" is missing — this lane is root with ip and nft, so the allowlist plan must run' + for method in ('FilteredEgressNetnsE2ETests.A_30_still_held_by_a_run_that_outlived_its_worker_is_not_handed_to_the_next_run', 'FilteredEgressNetnsE2ETests.A_host_whose_policy_rule_discards_the_run_s_replies_fails_the_setup_and_leaks_nothing', 'DurableLaunchEgressE2ETests.An_allowlist_run_reaches_its_broker_through_the_relay_and_its_allowlist_still_holds'): + cases = [r for r in results if method in r.get('testName', '')] + assert len(cases) == 1 and cases[0].get('outcome') == 'Passed', f'{method}: must pass' + print('All 3 allowlist-plan arms ran and passed.') + + # The broker's socket, relay-revoke and legacy-gateway arms return early without bwrap or ip/nft; require each + # marker, the legacy survivor's teardown of the seal it carried included. + for arm in ('socket-channel', 'revoke-network-off', 'revoke-allowlist', 'legacy-gateway-rebind', 'legacy-sealed-teardown'): + assert f'[broker-socket-e2e] ran {arm}' in text, f'broker E2E arm "{arm}" did not run — this lane is root with bwrap, ip and nft' + print('All 5 broker arms ran.') # The bwrap probe E2E returns early off root, which reads as Passed; require each arm's marker. for arm in ('masked-proc', 'unmasked-proc'): @@ -310,10 +324,60 @@ jobs: print('All 4 bounded command capture kernel cases passed.') PY + - name: Test the non-root worker posture (uid 1654, no capabilities) + # The shipped worker runs as uid 1654 with no capabilities and may not build a network namespace of its own, + # and every step above runs as root. Category=SandboxNonRoot runs the relay's arms again as that uid; each one + # first asserts geteuid() != 0, that bubblewrap confines, and that no namespace can be built, so the lane cannot + # pass as root or on a host that could fall back to a veth namespace. RequireConfinement stays set. The runner's + # kernel restricts unprivileged user namespaces through AppArmor where it has that switch; lifting it is the + # same node setting operators give the worker. The root arms above leave their short-path socket roots behind + # owner-only, which uid 1654 could not enter, so those go first. + shell: bash + run: | + set -euo pipefail + if [ -e /proc/sys/kernel/apparmor_restrict_unprivileged_userns ]; then sysctl -w kernel.apparmor_restrict_unprivileged_userns=0; fi + rm -rf /tmp/cs-broker /tmp/cs-mcp + home=/tmp/nonroot-home + results=backend/TestResults/nonroot + mkdir -p "$home" "$results" + chown 1654:1654 "$home" "$results" + # The restore above ran as root under the job's HOME (/github/home in a container job, not /root): ask NuGet + # where it put the packages, and let uid 1654 traverse every directory down to them and read them. + packages=$(dotnet nuget locals global-packages --list | sed -E 's/^global-packages: *//; s:/+$::') + test -d "$packages" + dir=$(dirname "$packages"); while [ "$dir" != / ]; do chmod a+x "$dir"; dir=$(dirname "$dir"); done + chmod -R a+rX "$packages" + chmod -R a+rwX backend/tests/CodeSpace.SandboxTests/bin backend/tests/CodeSpace.SandboxTests/obj + setpriv --reuid 1654 --regid 1654 --clear-groups --inh-caps=-all --bounding-set=-all --no-new-privs \ + env HOME="$home" DOTNET_CLI_HOME="$home" NUGET_PACKAGES="$packages" \ + dotnet test backend/tests/CodeSpace.SandboxTests/CodeSpace.SandboxTests.csproj \ + --no-build \ + --filter "Category=SandboxNonRoot" \ + --logger "console;verbosity=detailed" \ + --logger "trx;LogFileName=sandbox-nonroot.trx" \ + --results-directory "$results" + + - name: Assert the non-root arms actually ran as uid 1654 + run: | + python3 - <<'PY' + import xml.etree.ElementTree as ET + path = 'backend/TestResults/nonroot/sandbox-nonroot.trx' + root = ET.parse(path).getroot() + counters = root.find('.//{*}Counters') + executed, passed = int(counters.get('executed')), int(counters.get('passed')) + assert executed >= 9 and passed == executed, f'expected all 9 non-root arms to run and pass, got executed={executed} passed={passed}' + text = open(path, encoding='utf-8').read() + for marker in ('[non-root-e2e] ran admission uid=1654', '[sealed-egress-e2e] ran non-root durable uid=1654', '[sealed-egress-e2e] ran non-root non-durable uid=1654', '[sealed-egress-e2e] ran non-root restart uid=1654', '[sealed-egress-e2e] ran non-root relay-refused uid=1654', '[sealed-egress-e2e] ran non-root relay-ipv6', '[broker-socket-e2e] ran non-root socket-channel uid=1654', '[review-diff-e2e] ran non-root network-off-relayed claude-code uid=1654', '[review-diff-e2e] ran non-root network-off-relayed codex-cli uid=1654'): + assert marker in text, f'non-root arm marker "{marker}" is missing — the arm returned early or did not run as the worker uid' + print(f'All {executed} non-root arms ran as uid 1654 and passed.') + PY + - name: Upload test results if: always() uses: actions/upload-artifact@v4 with: name: sandbox-isolation-trx - path: backend/TestResults/*.trx + path: | + backend/TestResults/*.trx + backend/TestResults/nonroot/*.trx if-no-files-found: ignore diff --git a/backend/Dockerfile.worker b/backend/Dockerfile.worker index e4c19e2d3..60ef85d7c 100644 --- a/backend/Dockerfile.worker +++ b/backend/Dockerfile.worker @@ -67,11 +67,12 @@ # arms it (Sandbox__RequireConfinement=true), so a lost grant refuses runs instead of unconfining them. # # NETWORK-OFF runs (Confined and Standard, the default tier) are severed by bubblewrap (--unshare-net: loopback only). -# One whose model is brokered still has to reach its broker. Once the model-broker relay ships (`codespace-mcp relay`), -# it does so through a per-run Unix socket bound read-only into the sandbox, which needs no root and nothing beyond -# the grants above. Without the relay it is SEALED to the broker through a per-run namespace instead (no route, no NAT, -# no DNS, one gateway port), which needs root + CAP_NET_ADMIN + CAP_SYS_ADMIN (FilteredEgressNetns.CanSeal); a worker -# that confines but cannot seal refuses such a run before it spends anything (sandbox_sealed_egress_unavailable). +# One whose model is brokered still reaches its broker: `codespace-mcp relay` runs inside the sandbox in front of the +# CLI, answers the CLI's 127.0.0.1:, and carries each call to a per-run Unix socket bound read-only into the +# sandbox. That needs no root and nothing beyond the grants above — only the codespace-mcp helper, which this image +# ships. A worker that confines but has no helper that can relay refuses such a run before it spends anything +# (sandbox_sealed_egress_unavailable): so does one whose CODESPACE_MCP_PROXY_PATH names a build from before the relay +# or a self-contained publish. An allowlist run reaches its broker through the same relay. # # EGRESS FILTERING (an allowlist run) needs root + CAP_NET_ADMIN + CAP_SYS_ADMIN + a writable net.ipv4.ip_forward on # top of these packages: FilteredEgressPlan runs `ip netns add` (CAP_SYS_ADMIN: it unshares a network namespace and diff --git a/backend/src/CodeSpace.Core/Services/Agents/AgentAutonomyPolicy.cs b/backend/src/CodeSpace.Core/Services/Agents/AgentAutonomyPolicy.cs index a5684c428..cc7cc696a 100644 --- a/backend/src/CodeSpace.Core/Services/Agents/AgentAutonomyPolicy.cs +++ b/backend/src/CodeSpace.Core/Services/Agents/AgentAutonomyPolicy.cs @@ -233,13 +233,13 @@ public static string DescribeApproval(AgentAutonomyLevel effective) /// materially different fact from a severed namespace and must not read like a milder version of it. /// is such a run too — no confinement was even attempted. /// Both unconfined verdicts also carry : egress is the loudest thing an - /// unconfined run loses, but not the only one. A run sealed to its broker holds a per-run /30 exactly as an - /// allowlisted one does, so it carries the same host subnet caveat where this host has proved it. + /// unconfined run loses, but not the only one. A run sealed to its broker reaches it through the relay and holds no + /// per-run /30, so it carries no host subnet caveat. /// private static string OffQualifier(SandboxConfinement? confinement) => confinement switch { null => ConfinementCaveat, - { Outcome: SandboxConfinementOutcome.Confined, EgressSealedToBroker: true } => WithHostSubnetPosture(SealedToBrokerQualifier, EgressSubnetAllocator.ObservedHostDegradation), + { Outcome: SandboxConfinementOutcome.Confined, EgressSealedToBroker: true } => SealedToBrokerQualifier, { Outcome: SandboxConfinementOutcome.Confined, NetworkSevered: true } => " — confined: egress severed", { Outcome: SandboxConfinementOutcome.Confined } => " — confined, but egress was NOT severed", { Outcome: SandboxConfinementOutcome.Unconfined } c => $" — OFF REQUESTED BUT UNCONFINED: this host cannot sever egress ({c.Reason ?? "unknown"}){UnconfinedIsolationCaveat}", diff --git a/backend/src/CodeSpace.Core/Services/Agents/AgentRunExecutor.cs b/backend/src/CodeSpace.Core/Services/Agents/AgentRunExecutor.cs index 32a70797d..adda7a55b 100644 --- a/backend/src/CodeSpace.Core/Services/Agents/AgentRunExecutor.cs +++ b/backend/src/CodeSpace.Core/Services/Agents/AgentRunExecutor.cs @@ -460,7 +460,7 @@ public async Task ExecuteAsync(Guid agentRunId, CancellationToken cancellationTo // receives the governed tools the endpoint serves (today the harness projects ONLY task.Tools, so a restricted // run couldn't call them). Additive + tier-filtered; a no-op when the author named no tools (the CLI default // already reaches a declared MCP server's tools). Drives BuildInvocation off the augmented task. - var hardening = new SpecHardening(modelBaseUrl, modelProvider, workspaceProvision, brokeredCredential?.RebindPort); + var hardening = new SpecHardening(modelBaseUrl, modelProvider, workspaceProvision, brokeredCredential); SandboxSpec BuildSpec(AgentTask built) => HardenSpec(harness.BuildInvocation(AugmentToolsForMcp(built, mcp, mcpWiring)) with { Mcp = mcpWiring }, built, hardening); @@ -484,10 +484,10 @@ public async Task ExecuteAsync(Guid agentRunId, CancellationToken cancellationTo spec = BuildSpec(RunCold(effectiveTask)); } - // A spec this runner could only launch with a network that leaves the agent unable to work — a network-off - // brokered run on a host that confines but cannot seal — is refused HERE, the last moment a refusal costs - // nothing: before the local acceptance is prepared, the spend admitted or a process started. - (runner as ISandboxEgressAdmission)?.EnsureEgressAdmissible(spec, brokeredCredential?.ReachableFromNamespace ?? false); + // A spec this runner could only launch with a network that leaves the agent unable to work — a brokered run + // with a network of its own that this host cannot relay to its broker — is refused HERE, the last moment a + // refusal costs nothing: before the local acceptance is prepared, the spend admitted or a process started. + (runner as ISandboxEgressAdmission)?.EnsureEgressAdmissible(spec); // Verification is judged against the contract the envelope persisted — never a goal amended for the // dispatch alone (this cold hint, or an unreadable checkpoint's) — or the contract hash cannot match. @@ -3766,7 +3766,7 @@ private async Task ResolveModelCredentialEnvAsync(Age var wouldInject = projector is not null && credential is not null; - var brokered = await OpenBrokeredCredentialAsync(harness, credential, teamId, brokerage, cancellationToken).ConfigureAwait(false); + var brokered = await OpenBrokeredCredentialAsync(harness, LeaseRequestFor(task, teamId, credential, brokerage), cancellationToken).ConfigureAwait(false); // Fail closed BEFORE the projection that would put the key in the env — a deployment that mandates // confinement refuses the run rather than handing out a credential it cannot withdraw. Skipped entirely on a @@ -3788,7 +3788,7 @@ private async Task ResolveModelCredentialEnvAsync(Age // in the egress allowlist (B3.3b) — the UPSTREAM ones even under brokerage, deliberately: the allowlist is // enforced inside the run's netns, the broker reaches the provider from the host outside it, and a brokered // run keeping the provider host reachable loses nothing — the token it holds is refused there. (A network-OFF - // brokered run is the one sealed to just its broker; see ApplySealedEgress.) DefaultModel flows out so a model-less ("auto") run + // brokered run reaches just its broker, through the relay; see ApplyModelBrokerChannel.) DefaultModel flows out so a model-less ("auto") run // falls back to one of the credential's own models instead of the CLI default. All null when no credential // resolved. CredentialId names the ROW whose key this run authenticates with (null for the operator-global // key, which has no row) — D3 bounds an escalation's candidate models to exactly that row. @@ -3806,26 +3806,42 @@ private async Task ResolveModelCredentialEnvAsync(Age /// passed over in silence: they decide whether this run is holding the tenant's long-lived key, and a deployment /// that believed itself brokered has no other way to find out that every run took the direct path. /// - private async Task OpenBrokeredCredentialAsync(IAgentHarness harness, ResolvedModelCredential? credential, Guid teamId, AgentRunOwnerToken? owner, CancellationToken cancellationToken) + private async Task OpenBrokeredCredentialAsync(IAgentHarness harness, ModelCredentialLeaseRequest? lease, CancellationToken cancellationToken) { - if (owner is null || credential is null) return null; // a redaction-only re-resolve, or no credential to front at all + if (lease is null) return null; // a redaction-only re-resolve, or no credential to front at all - if (_credentialBroker is not { } broker) { LogUnbrokered(owner.RunId, "no model-credential broker is registered on this worker"); return null; } - if (harness is not IBrokeredModelCredentialProjector) { LogUnbrokered(owner.RunId, $"the {harness.Kind} harness cannot be re-pointed at a broker"); return null; } + if (_credentialBroker is not { } broker) { LogUnbrokered(lease.RunId, "no model-credential broker is registered on this worker"); return null; } + if (harness is not IBrokeredModelCredentialProjector) { LogUnbrokered(lease.RunId, $"the {harness.Kind} harness cannot be re-pointed at a broker"); return null; } try { - return await broker.OpenAsync( - new() { RunId = owner.RunId, TeamId = teamId, Epoch = owner.Epoch, Upstream = credential, Ttl = Credentials.ModelCredentialLease.Ttl }, - cancellationToken).ConfigureAwait(false); + return await broker.OpenAsync(lease, cancellationToken).ConfigureAwait(false); } catch (Exception exception) when (exception is not OperationCanceledException) { - _logger.LogWarning(exception, "Agent run {RunId}: the model-credential broker could not open a lease; this run's credential is unbrokered", owner.RunId); + _logger.LogWarning(exception, "Agent run {RunId}: the model-credential broker could not open a lease; this run's credential is unbrokered", lease.RunId); return null; } } + /// The lease a LAUNCH asks the broker for — null for a redaction-only resolve (no ) or when there is no credential to front. It asks for a socket exactly when the run's child will need one (). + private static ModelCredentialLeaseRequest? LeaseRequestFor(AgentTask task, Guid teamId, ResolvedModelCredential? credential, AgentRunOwnerToken? owner) => + owner is null || credential is null ? null : new() { RunId = owner.RunId, TeamId = teamId, Epoch = owner.Epoch, Upstream = credential, Ttl = Credentials.ModelCredentialLease.Ttl, SocketPath = ModelBrokerSocketPathFor(task.Permissions, owner.RunId) }; + + /// + /// A fresh per-run socket path for the run's broker lease, or null when its child will never need one: a run whose + /// network is the worker's (network on, no allowlist) calls the lease's loopback port directly, and a host that is + /// not Linux never confines, so its child shares the worker's network whatever the run asked. Minted afresh on every + /// open (an unguessable segment under the run's spool), because a path serves one lease at a time. The permission + /// test is the lease-time spelling of , which reads the built spec, + /// and a unit test pins that the two agree. + /// + internal static string? ModelBrokerSocketPathFor(AgentPermissions permissions, Guid runId) => + OperatingSystem.IsLinux() && ChildNetworkIsPrivate(permissions) ? LocalProcessRunner.ModelBrokerSocketPathFor(runId.ToString("N"), McpRunToken.MintPathId()) : null; + + /// Whether a run with these permissions gets a network that is not the worker's: network off, or narrowed to an allowlist — which, off or on, never shares the worker's. + internal static bool ChildNetworkIsPrivate(AgentPermissions permissions) => permissions.Network != AgentNetworkAccess.On || permissions.Egress == AgentEgressPolicy.Allowlist; + /// Say WHY a run is taking the direct-credential path. Information, not Debug: it is the fact that decides what the run is holding, and a deployment reads its own posture off this line. private void LogUnbrokered(Guid runId, string reason) => _logger.LogInformation("Agent run {RunId}: model credential NOT brokered — {Reason}; the deployment's confinement policy decides whether the key may be injected directly", runId, reason); @@ -4069,21 +4085,26 @@ internal static SandboxSpec ApplyWriteScope(SandboxSpec spec, AgentPermissions p spec with { ReadOnlyWorkingDirectory = permissions.WriteScope != AgentWriteScope.Workspace }; /// - /// Stamp a network-off run's broker port onto its spec, so a confining runner able to build one runs it in a - /// namespace SEALED to that broker instead of severing it from everything — the broker included, which left such a - /// run unable to reach any model. Only network-off runs: a run with network reaches its broker already, and its - /// egress is 's business. Returns the spec itself when there is nothing to stamp. + /// Stamp the run's broker channel onto its spec when its child's network is its own — network off, or narrowed to + /// an allowlist — so a confining runner starts the CLI behind the relay that carries it to its broker, instead of + /// severing it from everything, the broker included. The port always; the socket path when the broker bound one + /// (a runner that confines refuses such a child without it, before it spends). A run on the worker's own network + /// reaches its broker already and is returned as it is, as is every spec with no lease to reach. Reads the spec + /// after , so an allowlist that came out empty — severed — counts as off. /// - internal static SandboxSpec ApplySealedEgress(SandboxSpec spec, AgentPermissions permissions, int? brokerPort) => - permissions.Network == AgentNetworkAccess.Off && brokerPort is { } port ? spec with { ModelBrokerPort = port } : spec; + internal static SandboxSpec ApplyModelBrokerChannel(SandboxSpec spec, BrokeredModelCredential? brokered) => + brokered?.RebindPort is { } port && ChildNetworkIsPrivate(spec) ? spec with { ModelBrokerPort = port, ModelBrokerSocketPath = brokered.SocketPath } : spec; + + /// Whether the built spec's child gets a network that is not the worker's — the spec-side spelling of . + private static bool ChildNetworkIsPrivate(SandboxSpec spec) => !spec.AllowNetwork || spec.EgressAllowlist is { Count: > 0 }; - /// What the executor's hardening reads beyond the task itself: the model endpoint and the workspace the egress allowlist is built from, and the port of the run's brokered model lease, if it has one. - private readonly record struct SpecHardening(string? ModelBaseUrl, string? ModelProvider, WorkspaceProvisionRequest? Workspace, int? ModelBrokerPort); + /// What the executor's hardening reads beyond the task itself: the model endpoint and the workspace the egress allowlist is built from, and the run's brokered model lease, if it has one. + private readonly record struct SpecHardening(string? ModelBaseUrl, string? ModelProvider, WorkspaceProvisionRequest? Workspace, BrokeredModelCredential? ModelBroker); - /// The harness invocation with every one of the executor's own spec post-processings applied — the egress posture, the broker seal, the write scope and the tier's resource ceilings. One name so the launch and each revise round cannot drift apart on which hardening they got. + /// The harness invocation with every one of the executor's own spec post-processings applied — the egress posture, the broker channel, the write scope and the tier's resource ceilings. One name so the launch and each revise round cannot drift apart on which hardening they got. private static SandboxSpec HardenSpec(SandboxSpec spec, AgentTask task, SpecHardening hardening) { - var egress = ApplySealedEgress(ApplyEgressPolicy(spec, task.Permissions, hardening.ModelBaseUrl, hardening.ModelProvider, hardening.Workspace), task.Permissions, hardening.ModelBrokerPort); + var egress = ApplyModelBrokerChannel(ApplyEgressPolicy(spec, task.Permissions, hardening.ModelBaseUrl, hardening.ModelProvider, hardening.Workspace), hardening.ModelBroker); return ApplyResourceCeilings(ApplyWriteScope(egress, task.Permissions), task.Autonomy, RuntimeSettings.Current.AgentMemoryCeilingMb); } diff --git a/backend/src/CodeSpace.Core/Services/Agents/Credentials/Broker/LoopbackModelCredentialBroker.cs b/backend/src/CodeSpace.Core/Services/Agents/Credentials/Broker/LoopbackModelCredentialBroker.cs index 214c9b7c1..34c079ccc 100644 --- a/backend/src/CodeSpace.Core/Services/Agents/Credentials/Broker/LoopbackModelCredentialBroker.cs +++ b/backend/src/CodeSpace.Core/Services/Agents/Credentials/Broker/LoopbackModelCredentialBroker.cs @@ -30,17 +30,16 @@ namespace CodeSpace.Core.Services.Agents.Credentials.Broker; /// withdrawn bearer presents either way depending on timing — refused while something is still bound, unanswered once /// it is not — and both say the same thing, which is that the bearer buys nothing. Nothing is relayed either way. /// -/// Where it listens, and why it cannot simply be loopback. A deny-by-default egress run executes inside -/// a per-run network namespace, so 127.0.0.1 there is the NAMESPACE's loopback — a broker bound only to the -/// host's would be unreachable by exactly the runs that most need one. The namespace does reach the worker: its -/// default route is the veth gateway (FilteredEgressPlan's .1), and a packet addressed to the host's -/// own address is delivered locally (INPUT) rather than FORWARDED, so the run's nftables allowlist — a forward-hook -/// filter — never sees it and no allowlist entry is needed. But that /30 is reserved DURING the launch, after this -/// lease was opened and its base URL projected, so the listener cannot be bound to it up front: on a host that can -/// build such namespaces it binds every address (http://+:port/, which also matches any Host header) and the -/// runner substitutes with the address that particular child can -/// reach. On a host that cannot (macOS development, a container with no ip/nft) it stays on loopback, -/// where nothing needs the wider bind. +/// Where it listens. On loopback. A child in a network namespace of its own — network off, or an +/// allowlist — cannot reach the host's 127.0.0.1, so such a lease is also served on a per-run Unix socket +/// () whose directory is bound into the sandbox, and the codespace-mcp relay +/// inside answers the CLI's 127.0.0.1:<port> and carries each call to that socket. The runner resolves +/// to loopback for every child. The one exception is the re-bind of a +/// handle written before the socket existed whose child is in a network namespace of its own: that child reaches this +/// worker at its namespace's veth gateway, so on a host that can build such namespaces that re-bind binds every +/// address (http://+:port/, which also matches any Host header), and the broker logs +/// for each one. A handle without a socket whose child shares the worker's network is +/// re-bound on loopback, where that child calls. /// /// What guards it. The 256-bit bearer, checked in constant time, is the capability; the run's route /// segment is a 128-bit CSPRNG id, so a caller cannot even find another run's route by holding its id; the lease @@ -149,6 +148,9 @@ internal void BreakListenerForTest(Guid runId) if (_byRun.TryGetValue(runId, out var lease)) CloseQuietly(lease.Listener); } + /// Test seam: the prefix a LIVE lease's listener is bound to (http://127.0.0.1:<port>/, or http://+:<port>/ for a wide bind), or null for a run with no lease — the one observation that says which addresses a lease exposes, on a host that has no second address to call it from. + internal string? ListenerPrefixForTest(Guid runId) => _byRun.TryGetValue(runId, out var lease) ? lease.Listener.Prefixes.Single() : null; + /// Test seam: kill the Unix-socket acceptor behind a LIVE lease without going through a revoke — 's counterpart for the lease's other door, whose failure must end the claim just the same. internal void BreakSocketForTest(Guid runId) { @@ -172,9 +174,7 @@ private static HttpMessageHandler DefaultUpstreamHandler() => { if (UpstreamRootFor(request.Upstream) is not { } upstreamRoot) return Task.FromResult(null); - var hosts = CandidateHosts(request.SocketPath); - - if (BindFresh(hosts, out var bindFailure) is not { } bound) + if (BindFresh(out var bindFailure) is not { } bound) { _logger.LogWarning(bindFailure, "Agent run {RunId}: the model-credential broker could not bind a listener on this worker after {Attempts} fresh ports, so the run falls back to whatever its deployment's confinement policy permits", request.RunId, BindAttempts); return Task.FromResult(null); @@ -187,9 +187,8 @@ private static HttpMessageHandler DefaultUpstreamHandler() => }, request.Ttl); _logger.LogDebug("Model credential brokered for agent run {RunId} on port {Port} (team {TeamId}, epoch {Epoch}) until {ExpiresAt:O}", lease.RunId, lease.Port, lease.TeamId, lease.Epoch, lease.ExpiresAt); - WarnIfUnreachableFromNetns(lease, hosts, bound.Host); - return Task.FromResult(new(BaseUrlFor(lease), lease.Token, lease.ExpiresAt) { RebindPort = lease.Port, RebindRoute = lease.PathId, ReachableFromNamespace = bound.Host == AnyHost, SocketPath = lease.SocketPath }); + return Task.FromResult(new(BaseUrlFor(lease), lease.Token, lease.ExpiresAt) { RebindPort = lease.Port, RebindRoute = lease.PathId, SocketPath = lease.SocketPath }); } /// @@ -268,7 +267,7 @@ public Task RebindAsync(ModelCredentialRebindRequest request, Cancellation // The PORT first, the socket only once it holds: the port is the lock between two workers on one host. A worker // still serving this run holds it, so this re-bind is refused here — before anything at the socket's path is // touched — and the live worker's sandboxed children keep their door. - var hosts = CandidateHosts(request.SocketPath); + var hosts = CandidateHosts(request.SocketPath, request.ChildInNetworkNamespace); if (BindPort(request.Port, hosts, out var bindFailure) is not { } bound) return RefuseRebind(request, "its port could not be bound here — something else is holding it, or this host refused the bind", bindFailure); @@ -483,10 +482,10 @@ private void SweepLapsedLeases() private static string BaseUrlFor(Lease lease) => $"http://{SandboxSpec.ModelBrokerHostToken}:{lease.Port}/{lease.PathId}"; /// - /// A host that CAN build per-run network namespaces but refused the wide bind can serve only its shared-network - /// runs: a sealed run's child reaches this worker at its namespace gateway, and nothing is listening there. Said - /// out loud because the failure it produces is a model call that times out, which reads like a provider problem - /// rather than a bind that fell back. + /// A legacy re-bind on a host that CAN build per-run network namespaces, but that refused the wide bind, serves + /// only a child on the worker's own network: a namespaced child with no socket reaches this worker at its namespace + /// gateway, and nothing is listening there. Said out loud because the failure it produces is a model call that + /// times out, which reads like a provider problem rather than a bind that fell back. /// private void WarnIfUnreachableFromNetns(Lease lease, IReadOnlyList candidateHosts, string host) { @@ -496,30 +495,29 @@ private void WarnIfUnreachableFromNetns(Lease lease, IReadOnlyList candi } /// - /// Bind a FRESH ephemeral port for a new lease: every address on a host that can build filtered-egress namespaces - /// (their children reach the worker on a per-run gateway IP, not on loopback), loopback otherwise. The wider bind - /// is TRIED FIRST and falls back, so a host that refuses it still brokers its shared-network runs. + /// Bind a FRESH ephemeral port for a new lease, on loopback only. A child on the worker's own network calls it + /// there; a child in a network of its own reaches it through the lease's socket, spliced to this same loopback + /// listener, so no new lease ever hands its port to the host's neighbours. /// - private static (HttpListener Listener, int Port, string Host)? BindFresh(IReadOnlyList candidateHosts, out Exception? failure) + private static (HttpListener Listener, int Port)? BindFresh(out Exception? failure) { failure = null; - foreach (var host in candidateHosts) - for (var attempt = 0; attempt < BindAttempts; attempt++) - { - var port = ReserveEphemeralPort(); + for (var attempt = 0; attempt < BindAttempts; attempt++) + { + var port = ReserveEphemeralPort(); - if (TryBind(host, port, out failure) is { } listener) return (listener, port, host); - } + if (TryBind(LoopbackHost, port, out failure) is { } listener) return (listener, port); + } return null; } /// /// Bind ONE GIVEN port — a re-bind's whole job. No fresh-port retry, deliberately: the address is not this - /// process's to choose, it is the one a detached agent already holds, so a substitute would answer nobody. The - /// same candidate hosts as , because the run whose port this is was launched on a host of - /// the same shape and its child reaches the worker the same way. + /// process's to choose, it is the one a detached agent already holds, so a substitute would answer nobody. On the + /// hosts names for the request, because the child whose port this is still reaches the + /// worker the way it did when it was launched. /// private static (HttpListener Listener, int Port, string Host)? BindPort(int port, IReadOnlyList candidateHosts, out Exception? failure) { @@ -537,12 +535,15 @@ private static (HttpListener Listener, int Port, string Host)? BindPort(int port private const string LoopbackHost = "127.0.0.1"; /// - /// Bind addresses in order of preference — see . Every address first only where a per-run - /// network namespace can exist to need it, and never for a lease served over a socket: its namespaced children come - /// in through the socket, spliced to loopback, so a wide bind would only hand its port to the host's neighbours. + /// Where a RE-BIND binds, in order of preference. A request with a socket path binds loopback: its namespaced child + /// comes in through the socket, spliced to loopback, so a wide bind would only hand its port to the host's + /// neighbours. So does one whose child shares the worker's network, which calls loopback. Only the legacy re-bind — + /// a child in a network namespace of its own on a handle written before the socket existed — keeps the wide bind + /// first where a per-run network namespace can exist, because such a child reaches this worker at its namespace + /// gateway. That branch goes once no such handle is left in flight (). /// - private static IReadOnlyList CandidateHosts(string? socketPath) => - socketPath is null && FilteredEgressNetns.IsSupported ? new[] { AnyHost, LoopbackHost } : new[] { LoopbackHost }; + private static IReadOnlyList CandidateHosts(string? socketPath, bool childInNetworkNamespace) => + socketPath is null && childInNetworkNamespace && FilteredEgressNetns.IsSupported ? new[] { AnyHost, LoopbackHost } : new[] { LoopbackHost }; /// /// Bind ONE prefix, or null plus the reason it could not. EVERY exception counts as "did not bind" — deliberately diff --git a/backend/src/CodeSpace.Core/Services/Agents/Sandbox/Exceptions/SealedEgressUnavailableException.cs b/backend/src/CodeSpace.Core/Services/Agents/Sandbox/Exceptions/SealedEgressUnavailableException.cs index a29051085..ae4de89b9 100644 --- a/backend/src/CodeSpace.Core/Services/Agents/Sandbox/Exceptions/SealedEgressUnavailableException.cs +++ b/backend/src/CodeSpace.Core/Services/Agents/Sandbox/Exceptions/SealedEgressUnavailableException.cs @@ -3,48 +3,37 @@ namespace CodeSpace.Core.Services.Agents.Sandbox.Exceptions; /// -/// A network-off run whose model is brokered, REFUSED because this host would confine it but cannot seal its network -/// to that broker. Severing it instead — what a host that cannot seal would otherwise do — cuts the broker off along -/// with everything else, so the agent reaches no model and the run burns its whole timeout and the CLI's retries on -/// failures that read like a provider outage. Refusing names the wall instead, and does it before anything is spent. +/// A run whose model is brokered and whose network is its own — off, or narrowed to an allowlist — REFUSED because this +/// host would confine it but has no way to carry it to that broker. Such a child reaches its broker only through the +/// codespace-mcp relay inside its sandbox and the lease's per-run socket; without either, the agent reaches no +/// model and the run burns its whole timeout and the CLI's retries on failures that read like a provider outage. +/// Refusing names the wall instead, and does it before anything is spent. /// /// Unavailable, like : nothing about the launch can be -/// changed to make it work, and the identical launch succeeds untouched once an operator grants the worker what a -/// sealed namespace needs — or, for a setup step that failed on a host that can seal (), -/// fixes what that step names. says which it was, in the same words the message uses. +/// changed to make it work, and the identical launch succeeds untouched on a worker that has the helper and can open +/// the socket. says which was missing, in the same words the message uses. The type and its code +/// keep the name of the namespace seal the relay replaced, because the supervisor and stored results key on it. /// public sealed class SealedEgressUnavailableException : Exception, IFailure { - /// The worker lacks the ip or nft binary a sealed namespace is built with. - public const string CauseMissingTools = "ip or nft is not installed on this worker"; + /// The codespace-mcp helper, which runs the relay inside the sandbox, is not where the worker looks for it, or is there and cannot run the relay: a build from before it, a self-contained publish, or a file that does not start. + public const string CauseRelayMissing = "no codespace-mcp helper on this worker can relay the CLI's broker address"; - /// The binaries are there, but this process could not build a throwaway namespace. - public const string CauseNoPrivilege = "this worker may not create a network namespace (it needs root with CAP_NET_ADMIN and CAP_SYS_ADMIN)"; + /// The run's model broker served its lease without the per-run socket the relay connects to. + public const string CauseBrokerSocketUnavailable = "the run's model broker could not open the socket the relay connects to"; - /// The run's model broker could only listen on loopback, which a sealed namespace cannot reach. - public const string CauseBrokerLoopbackOnly = "the run's model broker could only listen on loopback, which a sealed namespace cannot reach"; + private const string Remedy = "Install this release's codespace-mcp next to the worker's assembly, as a framework-dependent build or a single-file publish (or point CODESPACE_MCP_PROXY_PATH at one), or fix what stopped the broker's socket from binding (its Warning names the path); a retry lands on another worker, which may have both."; - /// What an operator does about a worker that cannot build a sealed namespace at all. - private const string GrantRemedy = "Grant the worker what a sealed namespace needs (see backend/Dockerfile.worker, EGRESS FILTERING); a retry on this host helps only once it can build one, which it re-checks at most once a minute."; - - /// What an operator does about one setup step that failed on a worker that can build a sealed namespace. - private const string SetupRemedy = "This worker can build a sealed namespace, but a step of this one failed on its host: fix what that step names (a route or policy rule that discards the run's /30, or a namespace or veth left behind under the run's name). Every launch runs the setup afresh."; - - public SealedEgressUnavailableException(string cause) : this(cause, GrantRemedy) { } - - private SealedEgressUnavailableException(string cause, string remedy) - : base($"This run's network is off and its model is reached through its broker; on this worker that is enforced with a network namespace sealed to that broker, but one cannot be built here: {cause}. Refusing to launch an agent that could not reach its model. {remedy}") + public SealedEgressUnavailableException(string cause) + : base($"This run's network is its own and its model is reached through its broker, which on this worker means a relay inside the sandbox and a per-run socket, but {cause}. Refusing to launch an agent that could not reach its model. {Remedy}") { Cause = cause; } - /// A sealed setup that failed at launch on a host that proved it can seal — a name collision, a route the kernel will not send the run's replies down — carrying the failed step's own error. - public static SealedEgressUnavailableException SetupFailed(string? setupError) => new($"the sealed namespace's setup failed: {setupError}", SetupRemedy); - - /// Which wall the host hit — one of the Cause* constants, or a failed setup step's own error. + /// Which wall the host hit — one of the Cause* constants, optionally followed by what was looked for. public string Cause { get; } FailureKind IFailure.Kind => FailureKind.Unavailable; string IFailure.Code => FailureCodes.SandboxSealedEgressUnavailable; - string? IFailure.ClientMessage => "This host cannot give a network-off run a sealed route to its model."; + string? IFailure.ClientMessage => "This host cannot give this run's sandbox a route to its model."; } diff --git a/backend/src/CodeSpace.Core/Services/Agents/Sandbox/ISandboxEgressAdmission.cs b/backend/src/CodeSpace.Core/Services/Agents/Sandbox/ISandboxEgressAdmission.cs index 4262a75e9..e7da5a02b 100644 --- a/backend/src/CodeSpace.Core/Services/Agents/Sandbox/ISandboxEgressAdmission.cs +++ b/backend/src/CodeSpace.Core/Services/Agents/Sandbox/ISandboxEgressAdmission.cs @@ -9,16 +9,12 @@ namespace CodeSpace.Core.Services.Agents.Sandbox; /// can actually build on this host, and the executor asks it at the one moment a refusal still costs nothing — after /// the spec is built, before the spend is admitted and the process started. /// -/// The first such spec is a network-off run whose model is brokered () -/// on a host that confines but cannot seal: severing it would cut its broker off too. A runner without this capability -/// simply launches, exactly as it always has. +/// The spec such a refusal is for is one whose model is brokered () +/// and whose child would run in a network of its own on this host, but which this runner has no way to carry to the +/// broker. A runner without this capability simply launches, exactly as it always has. /// public interface ISandboxEgressAdmission { - /// - /// Throws an naming the wall when this runner cannot give - /// the egress it needs; returns otherwise. - /// is whether the run's broker listens where a per-run namespace can reach it (BrokeredModelCredential.ReachableFromNamespace). - /// - void EnsureEgressAdmissible(SandboxSpec spec, bool modelBrokerReachableFromNamespace); + /// Throws an naming the wall when this runner cannot give the egress it needs; returns otherwise. + void EnsureEgressAdmissible(SandboxSpec spec); } diff --git a/backend/src/CodeSpace.Core/Services/Agents/Sandbox/Isolation/BubblewrapSandbox.cs b/backend/src/CodeSpace.Core/Services/Agents/Sandbox/Isolation/BubblewrapSandbox.cs index 39157826f..e7e55dd55 100644 --- a/backend/src/CodeSpace.Core/Services/Agents/Sandbox/Isolation/BubblewrapSandbox.cs +++ b/backend/src/CodeSpace.Core/Services/Agents/Sandbox/Isolation/BubblewrapSandbox.cs @@ -80,18 +80,26 @@ public static void EnsureSatisfiable(string? available, bool required) /// neither claim a severance the argv did not request NOR miss one it did (an unenforceable allowlist fails /// closed to severed even while the launch asked to share the network). /// - /// says the shared network IS a sealed namespace whose only destination is - /// the run's model broker: severed from everything else, so recorded as severed — and as sealed, so a reader knows - /// the one route it kept. + /// says the run's network is off but its model broker is still reached, + /// through the relay inside its sandbox: severed from everything else, so recorded as severed — and as sealed, so a + /// reader knows the one route it kept. /// public static SandboxConfinement DeriveConfinement(string? available, string? unavailableReason, bool shareNetwork, IReadOnlyList? egressAllowlist, bool sealedToBroker = false) { if (available is null) return new SandboxConfinement { Outcome = SandboxConfinementOutcome.Unconfined, Reason = unavailableReason ?? SandboxConfinement.ReasonNoBubblewrap }; - return new SandboxConfinement { Outcome = SandboxConfinementOutcome.Confined, NetworkSevered = sealedToBroker || EgressFor(shareNetwork, egressAllowlist).Mode != SandboxEgressMode.Full, EgressSealedToBroker = sealedToBroker }; + return new SandboxConfinement { Outcome = SandboxConfinementOutcome.Confined, NetworkSevered = sealedToBroker || SeversNetwork(shareNetwork, egressAllowlist), EgressSealedToBroker = sealedToBroker }; } + /// + /// Whether a launch with this network intent gets a fresh network namespace (--unshare-net) rather than the + /// one it was started in — the network off, or an allowlist this sandbox cannot enforce. + /// exposed as the one question every reader of it asks, so the argv, the record and the runner's broker relay + /// cannot disagree about whether a child's network is its own. + /// + internal static bool SeversNetwork(bool shareNetwork, IReadOnlyList? egressAllowlist) => EgressFor(shareNetwork, egressAllowlist).Mode != SandboxEgressMode.Full; + /// /// The ONE egress derivation for a launch's network intent — read by both the argv () and /// the record (), so a severance can never appear in one and not the other. @@ -128,7 +136,7 @@ public static IReadOnlyList BuildArgs(BwrapPlan plan) // FAIL CLOSED to --unshare-net (a fresh net namespace, loopback only — no cloud-metadata / LAN / internet). // Only Full shares the host network (the agent reaches its model API). Byte-identical for a run with no // allowlist: ShareNetwork true → Full → shared; false → None → severed. - if (EgressFor(plan.ShareNetwork, plan.EgressAllowlist).Mode != SandboxEgressMode.Full) args.Add("--unshare-net"); + if (SeversNetwork(plan.ShareNetwork, plan.EgressAllowlist)) args.Add("--unshare-net"); // Read-only minimal root: the runtime + harness binary are reachable, the rest of the host FS is invisible. foreach (var dir in ReadOnlyRootDirs) @@ -141,7 +149,7 @@ public static IReadOnlyList BuildArgs(BwrapPlan plan) // If the command is an absolute path outside the standard roots (an operator binary override), bind its dir // read-only so it stays reachable inside the otherwise-minimal root — unless the command is itself one of the // files bound below, whose directory must stay unbound (the codespace-mcp helper's is the worker's own app dir). - if (Path.IsPathRooted(plan.Command) && Path.GetDirectoryName(plan.Command) is { Length: > 0 } cmdDir && !IsUnderReadOnlyRoot(cmdDir) && !plan.ReadOnlyExtraPaths.Contains(plan.Command)) + if (CommandDirectoryToBind(plan.Command) is { } cmdDir && !plan.ReadOnlyExtraPaths.Contains(plan.Command)) { args.Add("--ro-bind-try"); args.Add(cmdDir); @@ -200,6 +208,10 @@ public static IReadOnlyList BuildArgs(BwrapPlan plan) return args; } + /// The directory an absolute command outside must have bound to stay reachable inside the minimal root, or null for a bare name or a command under those roots. + internal static string? CommandDirectoryToBind(string command) => + Path.IsPathRooted(command) && Path.GetDirectoryName(command) is { Length: > 0 } directory && !IsUnderReadOnlyRoot(directory) ? directory : null; + private static bool IsUnderReadOnlyRoot(string dir) => ReadOnlyRootDirs.Any(root => dir == root || dir.StartsWith(root + "/", StringComparison.Ordinal)); diff --git a/backend/src/CodeSpace.Core/Services/Agents/Sandbox/Isolation/FilteredEgressNetns.cs b/backend/src/CodeSpace.Core/Services/Agents/Sandbox/Isolation/FilteredEgressNetns.cs index b9a173e56..3db376935 100644 --- a/backend/src/CodeSpace.Core/Services/Agents/Sandbox/Isolation/FilteredEgressNetns.cs +++ b/backend/src/CodeSpace.Core/Services/Agents/Sandbox/Isolation/FilteredEgressNetns.cs @@ -6,15 +6,14 @@ namespace CodeSpace.Core.Services.Agents.Sandbox.Isolation; /// /// The privileged executor of a (B3.2 enforcement) — sets up a per-run filtered /// network namespace, runs a command INSIDE it (so its only egress is the nftables allowlist), and tears the -/// namespace down. A network-off run whose model is brokered gets the SEALED variant instead -/// (), whose only reachable destination is that broker. Needs ip + nft + CAP_NET_ADMIN/root, so it runs for real only in the +/// namespace down. Needs ip + nft + CAP_NET_ADMIN/root, so it runs for real only in the /// privileged sandbox-isolation CI job; gates it everywhere else. Teardown is BEST-EFFORT /// and ALWAYS runs (even on a setup failure mid-way), so a failed run never leaks a netns / veth / nft table. /// public static class FilteredEgressNetns { - /// How long a failed tools or seal probe stands before it is tried again (see ): caching a transient failure for the process lifetime would sever every later network-off brokered run on this worker from its model. + /// How long a failed tools or seal probe stands before it is tried again (see ): caching a transient failure for the process lifetime would misreport this worker's posture until it restarted. internal static readonly TimeSpan SealProbeRetryInterval = TimeSpan.FromMinutes(1); private static readonly long ProcessStart = System.Diagnostics.Stopwatch.GetTimestamp(); @@ -23,19 +22,20 @@ public static class FilteredEgressNetns private static readonly CapabilityProbe Seal = new(ProbeSeal, () => System.Diagnostics.Stopwatch.GetElapsedTime(ProcessStart), SealProbeRetryInterval); - /// True when ip + nft are present (the binaries the plan drives). Actual privilege to create a netns is exercised at run time — a setup failure fails closed. A failed probe is retried like the seal probe (): a fork that failed once at boot must not disable the broker's wide bind and every seal for the process lifetime. + /// True when ip + nft are present (the binaries the plan drives). Actual privilege to create a netns is exercised at run time — a setup failure fails closed. A failed probe is retried like the seal probe (): a fork that failed once at boot must not disable every allowlist, and the legacy re-bind's wide bind, for the process lifetime. public static bool IsSupported => Tools.Holds; /// /// True when this process has PROVED it can build a namespace: the binaries are present AND one throwaway /// namespace was created and deleted, and nftables answered. The binaries alone are not enough — an image can ship - /// them to a worker that runs without the privilege to use them — and a network-off run is sealed only where this - /// holds, severed everywhere else. A proof is kept for the process; a failure is kept for + /// them to a worker that runs without the privilege to use them. The boot posture line reports it; no launch keys + /// on it, since a namespaced run reaches its broker through a relay that needs no namespace of the worker's own. + /// A proof is kept for the process; a failure is kept for /// and then probed again, and says why in . /// public static bool CanSeal => Seal.Holds; - /// Why the last seal probe failed — the failed step and its output — or null when none has failed since the last proof. Read by whatever reports a run that could not be sealed, so the cause is not lost with the probe. + /// Why the last seal probe failed — the failed step and its output — or null when none has failed since the last proof. Read by the boot posture line, so the cause is not lost with the probe. public static string? SealUnavailableReason => Seal.UnavailableReason; /// The outcome of running a command inside the filtered netns: the command's exit code + its combined output, plus whether the netns setup itself succeeded. @@ -87,25 +87,10 @@ public static async Task SetupAsync(string runId, IReadOnlyList - /// Set up a SEALED netns for a network-off run whose model is brokered (): - /// its only reachable destination is on the returned . - /// The same fail-closed contract as , the same /30 reservation, and the same - /// — the names are the run id's either way. - /// - public static async Task SetupSealedAsync(string runId, int brokerPort, int timeoutSeconds, CancellationToken cancellationToken) - { - if (await ReadHostRoutesAsync(timeoutSeconds, cancellationToken).ConfigureAwait(false) is not { } routes) return RoutesUnreadable; - var (subnet, exhausted) = Reserve(EgressSubnetAllocator.Host, runId, routes); - if (subnet is null) return new SetupResult { SetupOk = false, SetupError = exhausted }; - - return await ApplyAsync(runId, FilteredEgressPlan.BuildSealed(runId, brokerPort, subnet), timeoutSeconds, cancellationToken).ConfigureAwait(false); - } - /// /// Reserve the run's /30, or say why the host has none free — every candidate routed or held. That outcome is a - /// failed SETUP, reported like any other step's, so a caller that types its setup failures (a sealed run's refusal) - /// types this one too. A host whose reservation directory is unusable still throws its own typed refusal. + /// failed SETUP, reported like any other step's. A host whose reservation directory is unusable still throws its own + /// typed refusal. /// internal static (EgressSubnetAllocator.Lease? Subnet, string? Exhausted) Reserve(EgressSubnetAllocator allocator, string runId, HostRoutedPrefixes routes) { diff --git a/backend/src/CodeSpace.Core/Services/Agents/Sandbox/Isolation/FilteredEgressPlan.cs b/backend/src/CodeSpace.Core/Services/Agents/Sandbox/Isolation/FilteredEgressPlan.cs index f57da4af0..bc5129df8 100644 --- a/backend/src/CodeSpace.Core/Services/Agents/Sandbox/Isolation/FilteredEgressPlan.cs +++ b/backend/src/CodeSpace.Core/Services/Agents/Sandbox/Isolation/FilteredEgressPlan.cs @@ -1,5 +1,3 @@ -using System.Globalization; - namespace CodeSpace.Core.Services.Agents.Sandbox.Isolation; /// @@ -40,11 +38,8 @@ public sealed record FilteredEgressPlan /// The argv that asks the kernel how the host reaches the namespace's end. Run after : /// the /30 was chosen from the routes the host lists (), but only the kernel's own /// lookup accounts for a policy rule, or the null route in a table one consults before main, that would - /// discard the run's replies after a clean setup. A sealed plan names what its broker's replies carry — TCP from the - /// broker port — so a rule keyed on the protocol or the source port is seen too. It cannot name the rest: each reply - /// goes to the agent's own ephemeral port and may carry a mark, so a rule keyed on the destination port or a mark - /// still gets past it. An allowlist plan has no one port to name, and its NAT'd replies are routed on input, so a - /// rule keyed on those selectors or on the uplink (iif) can still divert them. + /// discard the run's replies after a clean setup. The plan has no one port to name, and its NAT'd replies are + /// routed on input, so a rule keyed on the protocol, a port, a mark or the uplink (iif) can still divert them. /// public required IReadOnlyList RouteCheckArgv { get; init; } @@ -127,41 +122,7 @@ public static FilteredEgressPlan Build(string runId, IReadOnlyList allow }; } - /// - /// Build the plan for a network-off run whose model is reached through its broker: the same per-run namespace and - /// /30, but SEALED — no default route (a packet to anywhere but the /30 fails with ENETUNREACH at once), no - /// forwarding, no NAT and no DNS, and an input filter on the host veth that admits exactly one destination, the - /// broker's on the gateway. Everything else the worker listens on — its own API, - /// every other run's broker port — is dropped, which the allowlist plan, with no input filter at all, leaves - /// reachable through the gateway. The table is inet so the veth's IPv6 link-local address is covered too. - /// Teardown is the same , reconstructed from the run id alone. - /// - public static FilteredEgressPlan BuildSealed(string runId, int brokerPort, EgressSubnetAllocator.Lease subnet) - { - var ns = NamespaceFor(runId); - var slug = Slug(runId); - var vethHost = $"csh-{slug}"; - var vethNs = $"csn-{slug}"; - - return new FilteredEgressPlan - { - Namespace = ns, - VethHost = vethHost, - VethNs = vethNs, - HostAddrCidr = $"{subnet.HostIp}/30", - NsAddrCidr = $"{subnet.NsIp}/30", - HostIp = subnet.HostIp, - NsIp = subnet.NsIp, - RouteCheckArgv = RouteCheck(subnet.NsIp, subnet.HostIp, "ipproto", "6", "sport", brokerPort.ToString(CultureInfo.InvariantCulture)), - NsSubnetCidr = subnet.Cidr, - SetupCommands = NamespaceSetup(ns, vethHost, vethNs, subnet), - NftRuleset = BuildSealedNftRuleset(ns, vethHost, subnet.HostIp, brokerPort), - ExecPrefix = new[] { "ip", "netns", "exec", ns }, - TeardownCommands = TeardownCommandsFor(runId), - }; - } - - /// The namespace, its veth pair and the /30 on both ends, with loopback up — what both plans share. Routing and forwarding are each plan's own. + /// The namespace, its veth pair and the /30 on both ends, with loopback up. The plan's routing and forwarding follow it. private static List> NamespaceSetup(string ns, string vethHost, string vethNs, EgressSubnetAllocator.Lease subnet) => new() { new[] { "ip", "netns", "add", ns }, @@ -174,8 +135,8 @@ public static FilteredEgressPlan BuildSealed(string runId, int brokerPort, Egres new[] { "ip", "netns", "exec", ns, "ip", "link", "set", "lo", "up" }, }; - /// The route lookup to the namespace's end from the gateway, narrowed by . The protocol is named by number: the name tcp needs /etc/protocols, which minimal images lack. - private static IReadOnlyList RouteCheck(string nsIp, string hostIp, params string[] selectors) => ["ip", "route", "get", nsIp, "from", hostIp, .. selectors]; + /// The route lookup to the namespace's end from the gateway. + private static IReadOnlyList RouteCheck(string nsIp, string hostIp) => ["ip", "route", "get", nsIp, "from", hostIp]; /// The per-run netns / nft-table name — derived PURELY from , so a reaper / teardown reconstructs it with no setup-time state. public static string NamespaceFor(string runId) => $"cs-egr-{Slug(runId)}"; @@ -195,7 +156,7 @@ public static IReadOnlyList> TeardownCommandsFor(string ru new[] { "ip", "netns", "del", ns }, // removes the ns + its veth end new[] { "ip", "link", "del", vethHost }, // best-effort: del may already be gone with the ns new[] { "nft", "delete", "table", "ip", ns }, - new[] { "nft", "delete", "table", "inet", ns }, // the sealed plan's table; best-effort, absent for an allowlist run + new[] { "nft", "delete", "table", "inet", ns }, // the table of a network-off run sealed to its broker before the relay replaced the seal; best-effort, absent otherwise }; } @@ -228,35 +189,6 @@ internal static string BuildNftRuleset(string table, string subnet, IReadOnlyLis return string.Join("\n", lines) + "\n"; } - /// - /// The sealed ruleset fed to nft -f -: on the INPUT hook, traffic arriving from this run's host veth may - /// reach only : over TCP (plus the replies to it); on the - /// FORWARD hook it is dropped outright. Keyed on the veth, never on the subnet, so another run's namespace — even - /// one handed the same /30 by a degraded allocator — is untouched by this table. - /// - /// It REPLACES any table of the same name rather than adding to it: every revise round of a run gets the same - /// names, so a round whose teardown failed to delete its table would otherwise have the next round's rules appended - /// after its own drop, cutting that round off from its broker. Declared, deleted and redefined in one nft -f - /// transaction, which the kernel applies atomically. - /// - internal static string BuildSealedNftRuleset(string table, string vethHost, string hostIp, int brokerPort) => string.Join("\n", new[] - { - $"table inet {table} {{}}", - $"delete table inet {table}", - $"table inet {table} {{", - " chain input {", - " type filter hook input priority 0;", - $" iifname \"{vethHost}\" ct state established,related accept", - $" iifname \"{vethHost}\" ip daddr {hostIp} tcp dport {brokerPort} accept", - $" iifname \"{vethHost}\" drop", - " }", - " chain forward {", - " type filter hook forward priority 0;", - $" iifname \"{vethHost}\" drop", - " }", - "}", - }) + "\n"; - private static string Slug(string runId) { var clean = new string((runId ?? "").Where(char.IsLetterOrDigit).ToArray()).ToLowerInvariant(); diff --git a/backend/src/CodeSpace.Core/Services/Agents/Sandbox/Isolation/ModelBrokerRelay.cs b/backend/src/CodeSpace.Core/Services/Agents/Sandbox/Isolation/ModelBrokerRelay.cs new file mode 100644 index 000000000..89f4942f2 --- /dev/null +++ b/backend/src/CodeSpace.Core/Services/Agents/Sandbox/Isolation/ModelBrokerRelay.cs @@ -0,0 +1,114 @@ +using System.Collections.Concurrent; +using System.ComponentModel; +using System.Diagnostics; +using System.Globalization; +using System.Net; +using System.Net.Sockets; +using CodeSpace.Core.Services.Agents.Mcp; + +namespace CodeSpace.Core.Services.Agents.Sandbox.Isolation; + +/// +/// The command rewrite that puts a CLI behind the codespace-mcp relay: <helper> relay <port> +/// <socket> -- <command> <args>. A child whose network namespace is not the worker's cannot reach +/// its broker's loopback port, so the relay — run INSIDE whatever confines the CLI — listens on +/// 127.0.0.1:<port> in the child's own namespace, the address already in the CLI's base URL, and carries +/// each connection to the lease's per-run socket. It starts the CLI itself, through /bin/sh's PATH lookup, and +/// exits with its status, so the chain around it sees the CLI exactly as before. is pure, the same +/// shape as , so the argv is testable on a host that cannot confine; +/// asks the helper itself. +/// +public static class ModelBrokerRelay +{ + /// The helper's verb for the relay — its argv[0]. The helper's own parser owns the other end, and a unit test pins that it accepts what builds. + public const string Verb = "relay"; + + /// The status the relay exits with when it cannot listen on its port, having started nothing. The helper's own constant owns the value; a unit test pins that the two agree. + public const int ListenFailedExitCode = 125; + + /// How long the helper has to answer . The relay binds one socket and exits, so a helper still running past this is not the relay. + private static readonly TimeSpan AnswerTimeout = TimeSpan.FromSeconds(10); + + /// The socket the question names. Never connected to: the relay exits before it accepts anything. + private const string UnusedSocketPath = "/nonexistent/cs-relay-probe"; + + /// The CLI the question names. Never started by a helper that answers as the relay. + private const string UnusedCli = "/bin/true"; + + /// The variables the MCP proxy connects with, kept from the question: a helper from before the relay reads any argv as the proxy's, and must fail at once rather than connect to whatever the worker's environment names. + private static readonly string[] McpProxyConnectVariables = [McpDeclarationWriter.SocketEnvVar, McpDeclarationWriter.TokenEnvVar]; + + /// The helper files that answered as the relay, by path, write time and length. + private static readonly ConcurrentDictionary<(string Path, DateTime WrittenUtc, long Length), bool> AnsweredAsRelay = new(); + + /// Rewrite to run behind the relay at , listening on and carrying each connection to . + public static (string Command, IReadOnlyList Args) Wrap(string helper, int port, string socketPath, string command, IReadOnlyList args) => + (helper, [Verb, port.ToString(CultureInfo.InvariantCulture), socketPath, "--", command, .. args]); + + /// + /// Whether the helper at runs the relay — asked, because the file being there does not + /// say so. The CODESPACE_MCP_PROXY_PATH override can name a build from before the relay, whose MCP proxy reads + /// any argv as its own and exits with its usage error, and a file can be there and not start at all; either way a + /// CLI put behind it never starts, and its run has already spent. The helper is started once, outside any sandbox, + /// and told to listen on a loopback port this process already holds: the relay cannot bind it, so it exits with + /// having started nothing. Only a yes is remembered, and only for the file that + /// gave it, so a replaced helper is asked again, and a no (a fork refused under the uid's shared task cap, say) is + /// not held against the next run. + /// + public static bool HelperRunsRelay(string helperPath) + { + if (IdentityOf(helperPath) is not { } identity) return false; + + if (AnsweredAsRelay.ContainsKey(identity)) return true; + + if (!AnswersAsRelay(helperPath)) return false; + + AnsweredAsRelay[identity] = true; + + return true; + } + + /// The helper file's identity for , or null when there is no file. + private static (string, DateTime, long)? IdentityOf(string helperPath) + { + var file = new FileInfo(helperPath); + + return file.Exists ? (helperPath, file.LastWriteTimeUtc, file.Length) : null; + } + + /// Ask the helper to relay on a loopback port held here for the length of the question, and whether it answered with . + private static bool AnswersAsRelay(string helperPath) + { + using var held = new TcpListener(IPAddress.Loopback, 0); + held.Start(); + + var (command, args) = Wrap(helperPath, ((IPEndPoint)held.LocalEndpoint).Port, UnusedSocketPath, UnusedCli, []); + + return ExitCodeOf(command, args) == ListenFailedExitCode; + } + + /// The exit status of , or null when it could not start or did not exit within , which kills it. It reads an empty stdin, and its output is drained and dropped. + private static int? ExitCodeOf(string command, IReadOnlyList args) + { + var start = new ProcessStartInfo(command) { UseShellExecute = false, RedirectStandardInput = true, RedirectStandardOutput = true, RedirectStandardError = true }; + + foreach (var arg in args) start.ArgumentList.Add(arg); + foreach (var name in McpProxyConnectVariables) start.Environment.Remove(name); + + try + { + using var process = Process.Start(start)!; + + process.StandardInput.Close(); + _ = process.StandardOutput.ReadToEndAsync(); + _ = process.StandardError.ReadToEndAsync(); + + if (process.WaitForExit(AnswerTimeout)) return process.ExitCode; + + process.Kill(entireProcessTree: true); + + return null; + } + catch (Exception exception) when (exception is Win32Exception or InvalidOperationException) { return null; } + } +} diff --git a/backend/src/CodeSpace.Core/Services/Agents/Sandbox/Isolation/SandboxEgressPolicy.cs b/backend/src/CodeSpace.Core/Services/Agents/Sandbox/Isolation/SandboxEgressPolicy.cs index 6a06964f4..60b65d19d 100644 --- a/backend/src/CodeSpace.Core/Services/Agents/Sandbox/Isolation/SandboxEgressPolicy.cs +++ b/backend/src/CodeSpace.Core/Services/Agents/Sandbox/Isolation/SandboxEgressPolicy.cs @@ -11,9 +11,6 @@ public enum SandboxEgressMode /// Egress filtered to only — the deny-by-default allowlist. The host filtering (privileged netns + nftables / proxy) is enforced by a later sandbox slice. Filtered, - - /// No egress, except the one TCP port on the namespace's gateway — the run's own model-credential broker. A network-off run whose model is brokered, on a host able to build the sealed namespace; everywhere else that run is . - Sealed, } /// @@ -21,9 +18,7 @@ public enum SandboxEgressMode /// 底座. It turns the binary AllowNetwork + an optional host allowlist into a single explicit policy, and is /// FAIL-CLOSED by construction: an allowlist requested on a runner that cannot ENFORCE filtering degrades to /// (severed), NEVER to — so a narrowing -/// can never silently widen to "any host". A network-off run is never widened either: sealing it to its broker -/// keeps it off everything else, and a run that cannot be sealed stays severed. No I/O — the actual host filtering -/// is a later slice that reads this. +/// can never silently widen to "any host". No I/O — the actual host filtering is a later slice that reads this. /// public sealed record SandboxEgressPolicy { @@ -32,9 +27,6 @@ public sealed record SandboxEgressPolicy /// The hosts reachable under — normalized (trimmed, lower-cased, de-duped, blanks dropped). Empty for None / Full. public IReadOnlyList AllowedHosts { get; init; } = Array.Empty(); - /// The one gateway port reachable under ; null for every other mode. - public int? BrokerPort { get; init; } - /// No egress. public static SandboxEgressPolicy Denied { get; } = new() { Mode = SandboxEgressMode.None }; @@ -42,13 +34,12 @@ public sealed record SandboxEgressPolicy public static SandboxEgressPolicy Shared { get; } = new() { Mode = SandboxEgressMode.Full }; /// - /// Derive the egress policy: no network ⇒ None, or Sealed to when the caller - /// has one (a brokered run on a host that can seal); network without an allowlist ⇒ Full (today's behaviour); + /// Derive the egress policy: no network ⇒ None; network without an allowlist ⇒ Full (today's behaviour); /// network WITH an allowlist ⇒ Filtered when the runner can enforce it, else None (FAIL-CLOSED — never Full). /// - public static SandboxEgressPolicy Derive(bool allowNetwork, IReadOnlyList? allowlist, bool canEnforceAllowlist, int? sealableBrokerPort = null) + public static SandboxEgressPolicy Derive(bool allowNetwork, IReadOnlyList? allowlist, bool canEnforceAllowlist) { - if (!allowNetwork) return sealableBrokerPort is { } port ? new SandboxEgressPolicy { Mode = SandboxEgressMode.Sealed, BrokerPort = port } : Denied; + if (!allowNetwork) return Denied; var hosts = NormalizeHosts(allowlist); diff --git a/backend/src/CodeSpace.Core/Services/Agents/Sandbox/Runners/LocalProcessRunner.Commands.cs b/backend/src/CodeSpace.Core/Services/Agents/Sandbox/Runners/LocalProcessRunner.Commands.cs index 3784e94cb..884059ee3 100644 --- a/backend/src/CodeSpace.Core/Services/Agents/Sandbox/Runners/LocalProcessRunner.Commands.cs +++ b/backend/src/CodeSpace.Core/Services/Agents/Sandbox/Runners/LocalProcessRunner.Commands.cs @@ -8,7 +8,7 @@ namespace CodeSpace.Core.Services.Agents.Sandbox.Runners; public sealed partial class LocalProcessRunner { - private sealed record CommandIsolationContext(SandboxSpec Spec, string? ConfigHome, string? McpDeclarationPath, IReadOnlyList EgressPrefix, IReadOnlyList CgroupPrefix); + internal sealed record CommandIsolationContext(SandboxSpec Spec, string? ConfigHome, string? McpDeclarationPath, IReadOnlyList EgressPrefix, IReadOnlyList CgroupPrefix); private async Task PrepareCommandAsync(SandboxSpec spec, CancellationToken cancellationToken) { @@ -24,11 +24,11 @@ private async Task PrepareCommandAsync(SandboxSpec spec, Canc var egress = await SetupEgressNetnsAsync(spec, key, cancellationToken).ConfigureAwait(false); invocation.EgressKey = egress.Key; - // Re-layer the spec with the broker host resolved (the start info was built before the run's /30 existed). - // The ARGV needs it as much as the env does — a harness whose CLI ignores its base-URL env var carries - // that URL on the command line instead — so the resolved spec, not `spec`, is what builds the child - // command below. A no-op for every run that does not mention the token: the values are identical. - var launched = ResolveModelBrokerHost(spec, egress.GatewayIp); + // Re-layer the spec with the broker host resolved. The ARGV needs it as much as the env does — a harness + // whose CLI ignores its base-URL env var carries that URL on the command line instead — so the resolved + // spec, not `spec`, is what builds the child command below. A no-op for every run that does not mention the + // token: the values are identical. + var launched = ResolveModelBrokerHost(spec); foreach (var (name, value) in launched.Environment) invocation.StartInfo.Environment[name] = value; WithoutProxiesWhenSealed(invocation.StartInfo.Environment, spec, egress.ExecPrefix); diff --git a/backend/src/CodeSpace.Core/Services/Agents/Sandbox/Runners/LocalProcessRunner.Durable.cs b/backend/src/CodeSpace.Core/Services/Agents/Sandbox/Runners/LocalProcessRunner.Durable.cs index 358e1e66a..ee3392e91 100644 --- a/backend/src/CodeSpace.Core/Services/Agents/Sandbox/Runners/LocalProcessRunner.Durable.cs +++ b/backend/src/CodeSpace.Core/Services/Agents/Sandbox/Runners/LocalProcessRunner.Durable.cs @@ -91,6 +91,10 @@ private static void CreateOwnerOnlyConfigHome(string? configHome) /// framework-dependent build output (the apphost beside its .dll, .deps.json and /// .runtimeconfig.json) or a single-file publish. A self-contained publish of several files cannot start /// inside the sandbox, and the executor refuses it with a Warning (). + /// + /// It must also be a build of this release. A brokered run whose network is its own starts its CLI behind + /// the helper's relay verb (), which a build from before it does not have; the + /// admission asks the helper and refuses such a run before it spends (). /// public const string McpProxyPathEnvVar = "CODESPACE_MCP_PROXY_PATH"; @@ -326,19 +330,17 @@ private static bool TryFileLength(string path, out long length) } /// - /// Derive this run's egress posture and, when it is an enforceable Filtered allowlist or a Sealed broker route, set - /// up the per-run netns and return the ip netns exec prefix the supervisor chain runs behind plus the - /// teardown key. None/Full need no netns (empty prefix, null key). Fail-closed: an allowlist requested on a runner - /// that cannot enforce it degrades to None (no netns) via , a network-off run this - /// host cannot seal stays severed, and a netns whose setup fails throws. + /// Derive this run's egress posture and, when it is an enforceable Filtered allowlist, set up the per-run netns and + /// return the ip netns exec prefix the supervisor chain runs behind plus the teardown key. None/Full need no + /// netns (empty prefix, null key). Fail-closed: an allowlist requested on a runner that cannot enforce it degrades + /// to None (no netns) via , and a netns whose setup fails throws. A network-off run + /// never gets one: it reaches its broker, if it has one, through the relay (). /// - private static async Task<(IReadOnlyList ExecPrefix, string? Key, string? GatewayIp)> SetupEgressNetnsAsync(SandboxSpec spec, string spoolKey, CancellationToken ct) + private static async Task<(IReadOnlyList ExecPrefix, string? Key)> SetupEgressNetnsAsync(SandboxSpec spec, string spoolKey, CancellationToken ct) { - var policy = SandboxEgressPolicy.Derive(spec.AllowNetwork, spec.EgressAllowlist, FilteredEgressNetns.IsSupported, SealableBrokerPort(spec)); - - if (policy.Mode == SandboxEgressMode.Sealed) return await SetupSealedNetnsAsync(policy.BrokerPort!.Value, spoolKey, ct).ConfigureAwait(false); + var policy = SandboxEgressPolicy.Derive(spec.AllowNetwork, spec.EgressAllowlist, FilteredEgressNetns.IsSupported); - if (policy.Mode != SandboxEgressMode.Filtered) return (Array.Empty(), null, null); + if (policy.Mode != SandboxEgressMode.Filtered) return (Array.Empty(), null); // The allowlist carries host NAMES (+ IP literals); the IPv4-only netns pins IPs, so resolve at setup on the // host that builds the namespace. Best-effort/fail-closed: an unresolvable host is dropped (the resulting set, @@ -351,50 +353,58 @@ private static bool TryFileLength(string path, out long length) if (!setup.SetupOk) throw new InvalidOperationException($"Filtered-egress netns setup failed (fail-closed — run aborted rather than launched unfiltered): {setup.SetupError}"); - return (setup.ExecPrefix, spoolKey, setup.HostIp); + return (setup.ExecPrefix, spoolKey); } /// - /// The broker port a network-off run may be sealed to — only where bubblewrap confines the command (exactly where - /// it would otherwise sever it with --unshare-net) and this host has proved it can build a namespace. Null - /// everywhere else, so an unconfined host keeps the launch it always had and a host that cannot seal keeps severing. + /// Refuse, before anything is spent, a brokered run whose child would run in a network of its own on this host but + /// could not be carried to its broker: its lease came back without the socket, or the codespace-mcp helper + /// at cannot run the relay there (). A spec with no + /// broker port, or a child that shares the worker's network (an unconfined host, a network-granting run with no + /// allowlist), is admitted untouched: nothing about its launch changes. The admission reads the same + /// the launch does, with the namespace predicted rather than built. /// - private static int? SealableBrokerPort(SandboxSpec spec) => - spec.ModelBrokerPort is { } port && BubblewrapSandbox.Available is not null && FilteredEgressNetns.CanSeal ? port : null; + public void EnsureEgressAdmissible(SandboxSpec spec) => EnsureEgressAdmissible(spec, BubblewrapSandbox.Available is not null, McpProxyBinaryPath()); - private static async Task<(IReadOnlyList ExecPrefix, string? Key, string? GatewayIp)> SetupSealedNetnsAsync(int brokerPort, string spoolKey, CancellationToken ct) + /// over whether this host and where it keeps the relay's helper, so a test can stand in for a confining host on any host. + internal static void EnsureEgressAdmissible(SandboxSpec spec, bool confines, string helperPath) { - var setup = await FilteredEgressNetns.SetupSealedAsync(spoolKey, brokerPort, EgressSetupTimeoutSeconds, ct).ConfigureAwait(false); + if (RelayRefusal(spec, ChildNetworkIsPrivate(spec, WouldFilterEgress(spec), confines), helperPath) is { } cause) + throw new SealedEgressUnavailableException(cause); + } - // The same refusal EnsureEgressAdmissible raises before any spend, for the rarer case the probe could not - // foresee — a setup step that fails on a host that proved it can seal (a name collision, a kernel refusal). - if (!setup.SetupOk) - throw SealedEgressUnavailableException.SetupFailed(setup.SetupError); + /// + /// Why a brokered child with a network of its own could not reach its broker from this host, or null when it can + /// or needs nothing: no broker port, a network it shares with the worker, or both halves of the relay in place. + /// Pure over the host's facts, so every cause is testable on a host that has none of them. + /// + internal static string? RelayRefusal(SandboxSpec spec, bool childNetworkIsPrivate, string helperPath) + { + if (spec.ModelBrokerPort is null || !childNetworkIsPrivate) return null; + + if (spec.ModelBrokerSocketPath is not { Length: > 0 }) return SealedEgressUnavailableException.CauseBrokerSocketUnavailable; - return (setup.ExecPrefix, spoolKey, setup.HostIp); + return RelayHelperProblem(helperPath) is { } problem ? $"{SealedEgressUnavailableException.CauseRelayMissing} ({problem})" : null; } /// - /// Refuse, before anything is spent, a network-off brokered run this host would confine but cannot seal — the - /// mirror of , which would otherwise quietly sever it from its broker. A spec with - /// no broker port, or a host that does not confine, is admitted untouched: nothing about its launch changes. + /// What stops the helper at from running the relay inside a sandbox, or null when + /// nothing does: it is not there; it is a self-contained publish, which cannot start from the files the sandbox + /// binds (); or, asked, it does not answer as the relay + /// () — a build from before the relay, which the + /// override can name, or a file that cannot start. /// - public void EnsureEgressAdmissible(SandboxSpec spec, bool modelBrokerReachableFromNamespace) + private static string? RelayHelperProblem(string helperPath) { - if (spec.ModelBrokerPort is null || BubblewrapSandbox.Available is null) return; + if (!File.Exists(helperPath)) return $"looked for {helperPath}"; - if (SealRefusal(FilteredEgressNetns.IsSupported, FilteredEgressNetns.CanSeal, modelBrokerReachableFromNamespace) is not { } cause) return; + if (McpProxyNeedsItsDirectory(helperPath)) return $"{helperPath} is a self-contained publish of several files, which cannot start from the files the sandbox binds"; - // The probe keeps the step that failed and its output; an operator needs that more than the category. - throw new SealedEgressUnavailableException(cause == SealedEgressUnavailableException.CauseNoPrivilege && FilteredEgressNetns.SealUnavailableReason is { } probe ? $"{cause}; the probe: {probe}" : cause); + return ModelBrokerRelay.HelperRunsRelay(helperPath) ? null : $"{helperPath} did not answer as the relay: it predates it, or cannot start"; } - /// Why a confining host cannot seal a brokered network-off run, or null when it can. Pure over the host's three facts, so every cause is testable on a host that has none of them. - internal static string? SealRefusal(bool haveTools, bool canSeal, bool brokerReachableFromNamespace) => - !haveTools ? SealedEgressUnavailableException.CauseMissingTools - : !canSeal ? SealedEgressUnavailableException.CauseNoPrivilege - : !brokerReachableFromNamespace ? SealedEgressUnavailableException.CauseBrokerLoopbackOnly - : null; + /// Whether this host launches inside a filtered-egress namespace — the prefix builds, predicted for the admission that runs before any of it exists. + private static bool WouldFilterEgress(SandboxSpec spec) => SandboxEgressPolicy.Derive(spec.AllowNetwork, spec.EgressAllowlist, FilteredEgressNetns.IsSupported).Mode == SandboxEgressMode.Filtered; /// /// Create this run's cgroup-v2 resource-cap leaf (B4) when a memory/cpu cap is requested AND the operator delegated @@ -959,10 +969,11 @@ internal static ProcessStartInfo BuildDurableStartInfo(SandboxSpec spec, string } /// - /// Resolve — everywhere it can appear in a spec — to the address - /// THIS child can actually reach the worker at: the filtered netns's own gateway when it runs inside one, else - /// loopback (a run sharing the host network). Total by construction: the token never survives into a child, - /// because a base URL still carrying it would be a broken URL rather than a visibly refused one. + /// Resolve — everywhere it can appear in a spec — to the address the + /// child reaches its broker at: 127.0.0.1, always. A child sharing the worker's network finds the lease's + /// own loopback listener there; a child in a network of its own finds the relay, which listens on that address + /// inside the child's namespace and carries the call to the lease's socket. Total by construction: the token never + /// survives into a child, because a base URL still carrying it would be a broken URL rather than a visibly refused one. /// /// The ENV is not the only carrier, and assuming it was is what broke brokered Codex runs: a harness whose /// CLI ignores its base-URL env var re-emits the value on the ARGV instead (CodexHarness's @@ -973,38 +984,35 @@ internal static ProcessStartInfo BuildDurableStartInfo(SandboxSpec spec, string /// Returns the spec UNCHANGED when nothing mentions the token, which is every run whose credential was /// not brokered — byte-identical command, argv and env, and no allocation. /// - /// A network-off brokered run on a host that seals runs inside a namespace SEALED to its broker, and resolves - /// to that namespace's gateway like any other netns run. One that is severed instead (no netns, no shared network — - /// a host that cannot seal) resolves to loopback and cannot reach the broker — nor could it reach the provider - /// directly, so brokerage neither adds nor removes anything for it. - /// /// scans only , /// and — a harness that instead wrote the /// broker base URL into would ship the token unresolved into that file, /// so any such projection must route the base URL through the command, argv, or env carriers this pass covers. /// - internal static SandboxSpec ResolveModelBrokerHost(SandboxSpec spec, string? gatewayIp) + internal static SandboxSpec ResolveModelBrokerHost(SandboxSpec spec) { if (!MentionsModelBrokerHost(spec)) return spec; - var host = gatewayIp is { Length: > 0 } reachable ? reachable : "127.0.0.1"; - var environment = spec.Environment.ToDictionary(entry => entry.Key, entry => WithModelBrokerHost(entry.Value, host), StringComparer.Ordinal); + var environment = spec.Environment.ToDictionary(entry => entry.Key, entry => WithModelBrokerHost(entry.Value, ModelBrokerHost), StringComparer.Ordinal); - ExemptBrokerFromProxies(environment, host); + ExemptBrokerFromProxies(environment, ModelBrokerHost); return spec with { - Command = WithModelBrokerHost(spec.Command, host), - Args = spec.Args.Select(arg => WithModelBrokerHost(arg, host)).ToList(), + Command = WithModelBrokerHost(spec.Command, ModelBrokerHost), + Args = spec.Args.Select(arg => WithModelBrokerHost(arg, ModelBrokerHost)).ToList(), Environment = environment, }; } + /// What resolves to in every child — see . + private const string ModelBrokerHost = "127.0.0.1"; + private static readonly string[] NoProxyVariables = { "NO_PROXY", "no_proxy" }; /// - /// Keep a child that was handed a proxy from sending its model calls there. The broker is reached at an address no - /// operator's NO_PROXY can name ahead of time — a per-run gateway, or a loopback a NO_PROXY may simply omit. Both + /// Keep a child that was handed a proxy from sending its model calls there. The broker is reached on loopback, which + /// an operator's NO_PROXY may simply omit — and inside a namespace of its own, loopback is the relay. Both /// spellings get the union of what the task or the worker set, because readers prefer different ones (curl, Python /// and Claude Code lowercase; reqwest and Go uppercase) and a spelling created with the broker alone would hide the /// operator's own exemptions from half of them. A child with no proxy is left as it was: a NO_PROXY written there @@ -1054,20 +1062,48 @@ private static bool MentionsModelBrokerHost(SandboxSpec spec) => private static readonly string[] ProxyVariables = { "HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "http_proxy", "https_proxy", "all_proxy" }; /// - /// Drop the proxy variables from a SEALED launch's environment. Its only destination is its broker, on the - /// namespace's own gateway — a proxy is unreachable from there, and a CLI that honours the variables would send its - /// every model call to it and fail as if the provider were down. The per-run gateway is never in an operator's - /// NO_PROXY, so the variables must go rather than be amended. The worker's own hop to the provider keeps its proxy. + /// Drop the proxy variables from a SEALED launch's environment. Its only destination is its broker, through the + /// relay on its own loopback — a proxy is unreachable from a network that is off, and a CLI that honours the + /// variables would send every call it does not exempt there and fail as if the provider were down. The worker's own + /// hop to the provider keeps its proxy. An allowlist run keeps its proxy, which its allowlist may well admit, and + /// reaches the relay past it through the loopback exemption in NO_PROXY. /// internal static void WithoutProxiesWhenSealed(IDictionary environment, SandboxSpec spec, IReadOnlyList egressExecPrefix) { - if (!SealedEgress(spec, egressExecPrefix)) return; + if (!SealedEgress(spec, egressExecPrefix.Count > 0, BubblewrapSandbox.Available is not null)) return; foreach (var name in ProxyVariables) environment.Remove(name); } - /// Whether this launch runs inside a SEALED netns: a namespace prefix for a run whose network is off can only be the sealed one, since an allowlist is read only when network is granted. Read by the launch's confinement record, beside . - private static bool SealedEgress(SandboxSpec spec, IReadOnlyList egressExecPrefix) => !spec.AllowNetwork && egressExecPrefix.Count > 0; + /// Whether this launch is SEALED to its broker: its network is off and the relay carries it to that broker, its one destination. Read by the proxy strip and the launch's confinement record, beside ; over the same two host facts as . + internal static bool SealedEgress(SandboxSpec spec, bool inNamespace, bool confines) => !spec.AllowNetwork && RelaysModelBroker(spec, inNamespace, confines); + + /// + /// What this launch's confinement RECORD says, from the same derivations its argv is built from: whether + /// confined it at all, whether its network was severed, and whether it was sealed to its + /// broker. Pure over the host's facts, so the record for every posture is testable on any host. + /// + internal static SandboxConfinement LaunchConfinement(SandboxSpec spec, IReadOnlyList egressExecPrefix, string? bwrap, string? unavailableReason) => + BubblewrapSandbox.DeriveConfinement(bwrap, unavailableReason, ShareNetwork(spec, egressExecPrefix), EgressAllowlist(spec, egressExecPrefix), SealedEgress(spec, egressExecPrefix.Count > 0, bwrap is not null)); + + /// + /// Whether this launch starts its CLI behind the codespace-mcp relay (): its + /// model is brokered, the lease has a socket, and the child's network is its own on this host — the chain runs in a + /// filtered-egress namespace (), or bubblewrap confines it () + /// into a fresh one. The ONE predicate the argv, the proxy strip, the confinement record and the admission all + /// read, so no two of them can disagree about which children reach their broker through the socket. + /// + internal static bool RelaysModelBroker(SandboxSpec spec, bool inNamespace, bool confines) => + spec.ModelBrokerPort is not null && spec.ModelBrokerSocketPath is { Length: > 0 } && ChildNetworkIsPrivate(spec, inNamespace, confines); + + /// + /// Whether the child's network is not the worker's: it runs in a filtered-egress namespace, or bubblewrap confines it + /// and gives it a fresh one ( — network off, or an allowlist bwrap + /// cannot enforce). Such a child cannot reach the broker's loopback port. An unconfined child shares the worker's + /// network whatever the spec asked, and reaches it directly. + /// + internal static bool ChildNetworkIsPrivate(SandboxSpec spec, bool inNamespace, bool confines) => + inNamespace || confines && BubblewrapSandbox.SeversNetwork(spec.AllowNetwork, spec.EgressAllowlist); /// /// The allowlist bwrap's OWN egress policy sees. Inside a filtered netns the namespace IS the enforcement, so the @@ -1086,54 +1122,76 @@ internal static void WithoutProxiesWhenSealed(IDictionary envir /// private static void AppendChildCommand(System.Collections.ObjectModel.Collection argv, CommandIsolationContext context) { - var (spec, configHome, mcpDeclarationPath, egressExecPrefix, cgroupExecPrefix) = context; // Fail-closed: a deployment that mandates isolation (Sandbox:RequireConfinement) must never run unconfined. BubblewrapSandbox.EnsureSatisfiable(BubblewrapSandbox.Available, BubblewrapSandbox.IsRequired); - var command = spec.Command; + foreach (var arg in ChildCommand(context, BubblewrapSandbox.Available, ProcessRlimits.Available)) argv.Add(arg); + } - // The declaration the write above laid down is inside the config-home, which bwrap binds writable at its own - // absolute path below — so the path on the argv resolves inside the sandbox exactly as it does outside it. - IReadOnlyList args = ArgsWithMcpDeclaration(spec, mcpDeclarationPath); + /// + /// The whole "$@" chain for this launch, given the host's and + /// (null where absent): cgroup → netns → prlimit → bwrap → relay → cli, each layer + /// present only where it applies. Pure over those two facts, so every host's chain is testable on any host. + /// + internal static IReadOnlyList ChildCommand(CommandIsolationContext context, string? bwrap, string? prlimit) + { + var (spec, _, _, egressExecPrefix, cgroupExecPrefix) = context; - // 1. Filesystem + namespace confinement (bubblewrap), innermost. A CLI that brings an OS sandbox of its own has - // it stood down HERE and nowhere else — the one decision that wraps the command — so it cannot lose its own - // sandbox on a launch that did not get ours. - if (BubblewrapSandbox.Available is { } bwrap) - { - args = BubblewrapSandbox.BuildArgs(PlanFor(spec, WithRunnerConfinement(args, spec.WhenRunnerConfines), configHome, egressExecPrefix)); - command = bwrap; - } + // 1. Filesystem + namespace confinement (bubblewrap), innermost, with the broker relay just inside it. + var (command, args) = ConfinedCommand(context, bwrap); // 2. Resource caps (prlimit) — outermost WITHIN "$@" ONLY: it wraps the agent chain, never the supervisor shell // that owns the spool. So its RLIMIT_FSIZE bounds files the AGENT writes, and cannot bound out.log/err.log at // all: those are written by the supervisor's own (unlimited) FIFO copier children, and RLIMIT_FSIZE does not // apply to pipe writes either. The spool's byte budget is enforced by those bounded copiers (CSP_MAX_BYTES, // SpoolCapBytes) instead. Fork-bomb + runaway-agent-file caps; memory-RSS + total-disk need the cgroup tier. - if (ProcessRlimits.Available is { } prlimit) + if (prlimit is not null) (command, args) = ProcessRlimits.Wrap(prlimit, command, args, ProcessRlimits.EffectiveMaxProcesses(spec.MaxProcesses), ProcessRlimits.EffectiveMaxFileSizeMb(spec.MaxFileSizeMb)); // 4. cgroup self-add (sh -c 'echo $$ > procs && exec "$@"'), OUTERMOST — places the WHOLE chain into the per-run // resource-capped cgroup on the HOST before entering the netns; cgroup membership is inherited across the // netns/prlimit/bwrap unshares, so the agent + every descendant are capped. Empty ⇒ byte-identical. - foreach (var p in cgroupExecPrefix) argv.Add(p); - // 3. Filtered-egress netns (ip netns exec ) — enters the per-run filtered network namespace before // prlimit/bwrap/agent, so the entire chain's only egress is the nftables allowlist. Empty (no prefix) // ⇒ byte-identical to a run without an enforceable allowlist. - foreach (var p in egressExecPrefix) argv.Add(p); + return [.. cgroupExecPrefix, .. egressExecPrefix, command, .. args]; + } + + /// + /// The CLI as whatever confines it runs it. Under bubblewrap: the CLI's own sandbox stood down (a CLI that brings + /// an OS sandbox of its own has it stood down HERE and nowhere else — the one decision that wraps the command — so + /// it cannot lose it on a launch that did not get ours), then the broker relay, then bubblewrap around both, so the + /// relay binds the sandbox's loopback and the stand-down still lands on the CLI's own argv. Without bubblewrap, the + /// relay alone, for a child that is in a namespace all the same (an allowlist on a host with no userns). + /// + private static (string Command, IReadOnlyList Args) ConfinedCommand(CommandIsolationContext context, string? bwrap) + { + var (spec, configHome, mcpDeclarationPath, egressExecPrefix, _) = context; + var inNamespace = egressExecPrefix.Count > 0; + + // The declaration the write above laid down is inside the config-home, which bwrap binds writable at its own + // absolute path below — so the path on the argv resolves inside the sandbox exactly as it does outside it. + var args = ArgsWithMcpDeclaration(spec, mcpDeclarationPath); + + if (bwrap is null) return WithModelBrokerRelay(spec, RelaysModelBroker(spec, inNamespace, confines: false), spec.Command, args); - argv.Add(command); - foreach (var arg in args) argv.Add(arg); + var (command, relayed) = WithModelBrokerRelay(spec, RelaysModelBroker(spec, inNamespace, confines: true), spec.Command, WithRunnerConfinement(args, spec.WhenRunnerConfines)); + + return (bwrap, BubblewrapSandbox.BuildArgs(PlanFor(spec, command, relayed, configHome, egressExecPrefix))); } + /// The CLI behind the broker relay when (); otherwise exactly the command and argv it was given. + private static (string Command, IReadOnlyList Args) WithModelBrokerRelay(SandboxSpec spec, bool relays, string command, IReadOnlyList args) => + relays ? ModelBrokerRelay.Wrap(McpProxyBinaryPath(), spec.ModelBrokerPort!.Value, spec.ModelBrokerSocketPath!, command, args) : (command, args); + /// /// What bubblewrap confines this launch to: the ONLY writable host paths are the config home and — unless the spec - /// may only read it — the workspace, which is otherwise mounted read-only. The MCP socket's dedicated dir and the - /// helper's own files are read-only. Pure over its inputs, so the spec-to-mount mapping is testable on a host that - /// cannot confine. + /// may only read it — the workspace, which is otherwise mounted read-only. The MCP socket's dedicated dir, the + /// broker socket's, and the helper's own files are read-only. and + /// are what runs inside — the CLI, or the relay in front of it. Pure over its inputs, so the spec-to-mount mapping is + /// testable on a host that cannot confine. /// - internal static BwrapPlan PlanFor(SandboxSpec spec, IReadOnlyList args, string? configHome, IReadOnlyList egressExecPrefix) + internal static BwrapPlan PlanFor(SandboxSpec spec, string command, IReadOnlyList args, string? configHome, IReadOnlyList egressExecPrefix) { var writable = new List(); if (!string.IsNullOrEmpty(spec.WorkingDirectory) && !spec.ReadOnlyWorkingDirectory) writable.Add(spec.WorkingDirectory); @@ -1155,9 +1213,11 @@ internal static BwrapPlan PlanFor(SandboxSpec spec, IReadOnlyList args, readOnlyExtra.AddRange(McpProxyFiles(McpProxyBinaryPath())); } + readOnlyExtra.AddRange(ModelBrokerRelayPaths(spec, egressExecPrefix)); + return new BwrapPlan { - Command = spec.Command, + Command = command, Args = args, WorkingDirectory = spec.WorkingDirectory, WorkingDirectoryReadOnly = spec.ReadOnlyWorkingDirectory, @@ -1172,6 +1232,26 @@ internal static BwrapPlan PlanFor(SandboxSpec spec, IReadOnlyList args, }; } + /// + /// What a relayed launch needs bound read-only beyond the rest ( runs only where bubblewrap + /// confines): the broker socket's own directory, the same rule as the MCP socket's; the helper that runs the relay, + /// file by file; and the CLI's own directory when it is an absolute path outside the read-only roots, which + /// bubblewrap would have bound had the CLI been the command, and the relay now is. Empty for every other launch, + /// so its argv is exactly what it was. + /// + private static IEnumerable ModelBrokerRelayPaths(SandboxSpec spec, IReadOnlyList egressExecPrefix) + { + if (!RelaysModelBroker(spec, egressExecPrefix.Count > 0, confines: true)) return Array.Empty(); + + var paths = new List { Path.GetDirectoryName(spec.ModelBrokerSocketPath!)! }; + + paths.AddRange(McpProxyFiles(McpProxyBinaryPath())); + + if (BubblewrapSandbox.CommandDirectoryToBind(spec.Command) is { } cliDirectory) paths.Add(cliDirectory); + + return paths; + } + /// /// The ABSOLUTE host path of the codespace-mcp proxy binary: the override /// when set (an air-gapped mirror), else codespace-mcp next to the running assembly. Identity-bound into the diff --git a/backend/src/CodeSpace.Core/Services/Agents/Sandbox/Runners/LocalProcessRunner.NativeLaunch.cs b/backend/src/CodeSpace.Core/Services/Agents/Sandbox/Runners/LocalProcessRunner.NativeLaunch.cs index 63f743d55..1fbd1d88f 100644 --- a/backend/src/CodeSpace.Core/Services/Agents/Sandbox/Runners/LocalProcessRunner.NativeLaunch.cs +++ b/backend/src/CodeSpace.Core/Services/Agents/Sandbox/Runners/LocalProcessRunner.NativeLaunch.cs @@ -220,11 +220,6 @@ private static async Task BindLaunchAsync(SandboxLaunchReque private async Task StartBrokerAsync(BrokerStart request, CancellationToken cancellationToken) { var binary = RunnerHostBinaryPath(); - - // The bootstrap's admission window opens at "owned" and must cover the whole cgroup + namespace setup below, so - // the one slow first-use cost — the seal probe — is paid before the window opens rather than inside it. - _ = SealableBrokerPort(request.Spec); - var info = new ProcessStartInfo(binary) { UseShellExecute = false, CreateNoWindow = true, RedirectStandardInput = true, RedirectStandardOutput = true, RedirectStandardError = true }; info.ArgumentList.Add("broker"); info.ArgumentList.Add(request.Directory); using var process = new Process { StartInfo = info }; @@ -244,15 +239,15 @@ private async Task StartBrokerAsync(BrokerStart request, CancellationToken cance cgroupKey = cgroup.Key; var egress = await SetupEgressNetnsAsync(request.Spec, request.SpoolKey, cancellationToken).ConfigureAwait(false); egressKey = egress.Key; - // The child's env is built from the spec with the broker host resolved to the address THIS launch can - // reach the worker at — known only now, after the run's /30 was reserved above. - var command = BuildDurableStartInfo(ResolveModelBrokerHost(request.Spec, egress.GatewayIp), request.Spool, egress.ExecPrefix, cgroup.ExecPrefix, bootstrapSession: true); + // The child's env is built from the spec with the broker host resolved to loopback, where the lease — or, + // for a child with a network of its own, the relay in front of it — answers. + var command = BuildDurableStartInfo(ResolveModelBrokerHost(request.Spec), request.Spool, egress.ExecPrefix, cgroup.ExecPrefix, bootstrapSession: true); var invocation = new NativeLaunchInvocation { Spec = request.Spec, ReadOnlyPaths = request.Spec.ReadOnlyPaths, CaptureBudget = request.Spec.CaptureBudget, Command = command.FileName, Args = command.ArgumentList.ToArray(), WorkingDirectory = command.WorkingDirectory, Environment = command.Environment.ToDictionary(pair => pair.Key, pair => pair.Value), EgressNetnsKey = egressKey, CgroupRunKey = cgroupKey, - Confinement = BubblewrapSandbox.DeriveConfinement(BubblewrapSandbox.Available, BubblewrapSandbox.UnavailableReason, ShareNetwork(request.Spec, egress.ExecPrefix), EgressAllowlist(request.Spec, egress.ExecPrefix), SealedEgress(request.Spec, egress.ExecPrefix)), + Confinement = LaunchConfinement(request.Spec, egress.ExecPrefix, BubblewrapSandbox.Available, BubblewrapSandbox.UnavailableReason), }; // Measured BEFORE transmission is marked started, so a frame no pipe can carry is refused while the catch // below can still tear the netns and cgroup down, and the broker reads EOF and releases its slot as rejected. diff --git a/backend/src/CodeSpace.Core/Services/Supervisor/Deciders/LlmSupervisorDecider.cs b/backend/src/CodeSpace.Core/Services/Supervisor/Deciders/LlmSupervisorDecider.cs index de0cedde6..6e117c68d 100644 --- a/backend/src/CodeSpace.Core/Services/Supervisor/Deciders/LlmSupervisorDecider.cs +++ b/backend/src/CodeSpace.Core/Services/Supervisor/Deciders/LlmSupervisorDecider.cs @@ -1775,7 +1775,7 @@ private static void AppendFailedVerdict(StringBuilder builder, SupervisorAgentRe Messages.Failures.FailureCodes.ModelCredentialBrokerUnavailable => "RETRY this exact subtask once, in case another worker can broker its model credential; if it ends the same way again, 'ask_human' — that is a deployment setting only an operator can change. Either way do NOT re-plan it and do NOT amend its check — there is nothing wrong with either.", Messages.Failures.FailureCodes.SandboxSealedEgressUnavailable => - "RETRY this exact subtask once, in case another worker can seal its network to its model broker; if it ends the same way again, 'ask_human' — that is a deployment setting only an operator can change. Either way do NOT re-plan it and do NOT amend its check — there is nothing wrong with either.", + "RETRY this exact subtask once, in case another worker can relay its sandbox to its model broker; if it ends the same way again, 'ask_human' — that is a deployment setting only an operator can change. Either way do NOT re-plan it and do NOT amend its check — there is nothing wrong with either.", _ => "This is an infrastructure fault with no recorded remedy: 'ask_human' to rule. Do NOT re-plan it and do NOT amend its check — neither is where the fault is.", }; diff --git a/backend/src/CodeSpace.Messages/Agents/BrokeredModelCredential.cs b/backend/src/CodeSpace.Messages/Agents/BrokeredModelCredential.cs index 06b95b52d..7dab1f017 100644 --- a/backend/src/CodeSpace.Messages/Agents/BrokeredModelCredential.cs +++ b/backend/src/CodeSpace.Messages/Agents/BrokeredModelCredential.cs @@ -34,19 +34,13 @@ public sealed record BrokeredModelCredential(string BaseUrl, string RunToken, Da /// The unguessable route segment of , for the same reason as : a re-bind has to install the run's OWN route, never mint a fresh one, or the address the agent holds resolves to nothing. Null exactly when is. public string? RebindRoute { get; init; } - /// - /// Whether the lease listens on every address, so a child inside a per-run network namespace — which reaches the - /// worker at its namespace gateway, never on loopback — can reach it. False (the default) is the fail-closed - /// answer: a broker that could only bind loopback, or one that does not say, cannot serve a sealed network-off run, - /// and such a run is refused before launch rather than left calling an address nothing answers. - /// - public bool ReachableFromNamespace { get; init; } - /// /// The per-run Unix socket this lease is ALSO served on — the path the request asked for, once bound — or null when /// none was asked for or it was not bound: its bind failed, or another live lease on this worker still serves that - /// path. The lease then serves TCP alone, and the broker says so in a Warning. - /// Stamped on the durable handle beside , so a re-attaching worker re-opens the same path. + /// path. The lease then serves loopback TCP alone, and the broker says so in a Warning. It is the only way a child + /// in a network of its own reaches the lease, so a confining runner refuses such a child whose lease came back + /// without one. Stamped on the durable handle beside , so a re-attaching worker re-opens the + /// same path. /// [System.Text.Json.Serialization.JsonIgnore(Condition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull)] public string? SocketPath { get; init; } diff --git a/backend/src/CodeSpace.Messages/Agents/ModelCredentialRebindRequest.cs b/backend/src/CodeSpace.Messages/Agents/ModelCredentialRebindRequest.cs index b08b5bd04..51b060bb7 100644 --- a/backend/src/CodeSpace.Messages/Agents/ModelCredentialRebindRequest.cs +++ b/backend/src/CodeSpace.Messages/Agents/ModelCredentialRebindRequest.cs @@ -45,8 +45,8 @@ public sealed record ModelCredentialRebindRequest /// The Unix socket the launch's lease was also served on, read off the same handle, or null when it recorded none. /// With a path, is bound on LOOPBACK FIRST and the socket re-opened at this exact path only once /// that bind holds: the port is the lock between two workers on one host, so a worker that loses it never touches - /// the other's socket. Null is the legacy re-bind, unchanged — the recorded port on every candidate host, wide - /// first, for a child that reaches this worker at its namespace gateway. + /// the other's socket. Null binds loopback too, unless holds: that is the + /// legacy re-bind — the recorded port wide first, for a child that reaches this worker at its namespace gateway. /// [System.Text.Json.Serialization.JsonIgnore(Condition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull)] public string? SocketPath { get; init; } @@ -54,9 +54,10 @@ public sealed record ModelCredentialRebindRequest /// /// Whether the same handle recorded a per-run network namespace for the child (SandboxHandle.EgressNetnsKey). /// Without a , such a child reaches this worker at its namespace's gateway rather than on - /// loopback — the one kind of run that still needs the legacy re-bind, so the broker names exactly these re-binds in - /// a fixed log line that the retirement of the gateway path waits on. A run on the worker's own network calls - /// loopback and is not one of them, whatever its handle lacks. + /// loopback — the one kind of run that still needs the legacy re-bind, so the broker binds wide for exactly these + /// re-binds and names them in a fixed log line that the retirement of the gateway path waits on. A run on the + /// worker's own network calls loopback and is not one of them, whatever its handle lacks: its re-bind binds + /// loopback alone. /// public bool ChildInNetworkNamespace { get; init; } } diff --git a/backend/src/CodeSpace.Messages/Agents/SandboxConfinement.cs b/backend/src/CodeSpace.Messages/Agents/SandboxConfinement.cs index 57d7deec6..5018e588a 100644 --- a/backend/src/CodeSpace.Messages/Agents/SandboxConfinement.cs +++ b/backend/src/CodeSpace.Messages/Agents/SandboxConfinement.cs @@ -45,14 +45,17 @@ public sealed record SandboxConfinement /// Why the host could not confine — one of the Reason* constants. Null for every outcome but . public string? Reason { get; init; } - /// Whether the launch severed the agent from the network — a fresh EMPTY net namespace (--unshare-net), or one SEALED to its model broker (, which says which) — true when confinement applied AND its egress policy came out anything but full: the run's network was off, OR it asked for an allowlist the sandbox cannot yet enforce and so failed closed. False for a plain shared-network run and for every unconfined one. + /// Whether the launch severed the agent from the network — a fresh EMPTY net namespace (--unshare-net), with or without its model broker reachable through a relay (, which says which) — true when confinement applied AND its egress policy came out anything but full: the run's network was off, OR it asked for an allowlist the sandbox cannot yet enforce and so failed closed. False for a plain shared-network run and for every unconfined one. public bool NetworkSevered { get; init; } /// - /// Whether the run's severed network was a SEALED namespace rather than an empty one: no route, no NAT and no DNS, - /// with exactly one reachable destination — the run's own model-credential broker on the namespace's gateway. The - /// shape a network-off run with a brokered model gets where the host can build it, because an empty namespace - /// would cut that run off from its model too. Implies ; false for every other run. + /// Whether the run's severed network still reached exactly one destination: the run's own model-credential broker. + /// The shape a network-off run with a brokered model gets on a confining host — an empty namespace with only + /// loopback, where the codespace-mcp relay answers the CLI's broker address and carries each connection to + /// the lease's socket bound read-only into the sandbox — because an empty namespace alone would cut that run off + /// from its model too. A run launched before the relay reached the broker through a namespace sealed to it (no + /// route, no NAT, no DNS, one gateway port) and recorded the same fact. Implies ; false + /// for every other run. /// public bool EgressSealedToBroker { get; init; } diff --git a/backend/src/CodeSpace.Messages/Agents/SandboxHandle.cs b/backend/src/CodeSpace.Messages/Agents/SandboxHandle.cs index a441a0510..0423db144 100644 --- a/backend/src/CodeSpace.Messages/Agents/SandboxHandle.cs +++ b/backend/src/CodeSpace.Messages/Agents/SandboxHandle.cs @@ -175,11 +175,12 @@ public sealed record SandboxHandle /// that authenticates to nothing else and expires — not the tenant's key, which never leaves the broker. /// /// Per-lease ports change the shape of that exposure rather than its nature: the number of squattable - /// addresses is now the number of concurrent brokered runs instead of one per worker, and on a host that builds - /// filtered-egress namespaces each is a WIDE (+) bind rather than loopback — the pre-existing design, since - /// a sealed run reaches the worker at its namespace gateway and not on loopback, so there are now N of those where - /// there was 1. Reaching them still buys nothing without a live run's bearer, and the broker refuses any source - /// outside loopback and the allocator's 10/8 space. + /// addresses is now the number of concurrent brokered runs instead of one per worker. Every lease binds loopback; a + /// child in a network of its own reaches it through . Only the re-bind of a + /// handle with no socket path — a namespaced run launched before the socket existed, which reaches the worker at its + /// namespace gateway — still binds WIDE (+) where the host builds filtered-egress namespaces. Reaching any of + /// them still buys nothing without a live run's bearer, and the broker refuses any source outside loopback and the + /// allocator's 10/8 space. /// public int? ModelBrokerPort { get; init; } @@ -211,8 +212,9 @@ public sealed record SandboxHandle /// /// The key of the filtered-egress network namespace this run was launched inside (B3.2b) — non-null ONLY when a - /// deny-by-default allowlist was enforceable, or a network-off run was sealed to its model broker, and a netns was - /// set up. It is the teardown handle: the netns / veth / + /// deny-by-default allowlist was enforceable and a netns was set up (or, on a handle written before a namespaced + /// run reached its broker through a relay, when a network-off run was sealed to its broker through one). It is the + /// teardown handle: the netns / veth / /// nft-table names are derived purely from it, so a reap (or a re-attach after a restart, from a DIFFERENT worker /// process on the same host) tears the namespace down with no setup-time state — the tools it drives are local, so /// the same-host boundary in the type remarks applies. Null when the run had no allowlist or the runner couldn't diff --git a/backend/src/CodeSpace.Messages/Agents/SandboxSpec.cs b/backend/src/CodeSpace.Messages/Agents/SandboxSpec.cs index f10f1632a..c22ef1bd5 100644 --- a/backend/src/CodeSpace.Messages/Agents/SandboxSpec.cs +++ b/backend/src/CodeSpace.Messages/Agents/SandboxSpec.cs @@ -82,8 +82,8 @@ public sealed record SandboxSpec /// /// Whether the command may reach the network. false (the DEFAULT) → the sandbox runner severs egress - /// entirely (a fresh network namespace with only loopback — or, for a run whose model is brokered, one sealed to - /// that broker: see ), so a confined agent cannot reach cloud-metadata, the LAN, or + /// entirely (a fresh network namespace with only loopback — a run whose model is brokered still reaches that + /// broker, through its socket: see ), so a confined agent cannot reach cloud-metadata, the LAN, or /// exfiltrate over the internet. true → the host network is shared, UNLESS /// narrows it. Enforced only by a sandboxing runner; a bare-process runner cannot /// honour it. @@ -107,18 +107,29 @@ public sealed record SandboxSpec public IReadOnlyList? EgressAllowlist { get; init; } /// - /// The port of this run's model-credential broker lease, set only for a run whose network is OFF and whose model - /// is reached through that broker. Such a run cannot be severed from everything the way - /// otherwise asks — the broker would be cut off with the rest, and the agent could reach no model at all — so a - /// confining runner able to build one runs it in a SEALED namespace instead: no route, no NAT, no DNS, and exactly - /// one reachable destination, this port on the namespace's gateway. A runner that cannot seal keeps severing. + /// The port of this run's model-credential broker lease, set only for a run whose network is not the worker's — + /// network off, or narrowed to an — and whose model is reached through that broker. + /// Such a child cannot reach the broker's loopback port from its own namespace, so a confining runner starts it + /// behind the codespace-mcp relay, which listens on 127.0.0.1:<this port> inside the sandbox + /// and carries each connection to . The network stays exactly as severed or + /// filtered as and the allowlist ask. /// - /// Set by AgentRunExecutor.ApplySealedEgress at the executor's one spec choke point. Null (every other - /// spec) ⇒ omitted from the JSON, so the spec serializes and hashes as it did before the field existed. + /// Set by AgentRunExecutor.ApplyModelBrokerChannel at the executor's one spec choke point. Null (every + /// other spec) ⇒ omitted from the JSON, so the spec serializes and hashes as it did before the field existed. /// [System.Text.Json.Serialization.JsonIgnore(Condition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull)] public int? ModelBrokerPort { get; init; } + /// + /// The per-run Unix socket the run's broker lease is also served on — the other end of the relay + /// describes. The runner binds its directory read-only into the sandbox. Null when the + /// broker bound none (a host that mints none, or a bind that failed): a confining runner then has no way to reach + /// such a child's broker, and refuses it before anything is spent. Omitted from the JSON while null, so a spec + /// without one serializes and hashes as it did before the field existed. + /// + [System.Text.Json.Serialization.JsonIgnore(Condition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull)] + public string? ModelBrokerSocketPath { get; init; } + /// /// Max processes the command + its descendants may spawn (RLIMIT_NPROC) — a fork-bomb cap so a runaway agent /// cannot exhaust the worker's process table. 0 = unlimited. Enforced by a sandboxing runner. diff --git a/backend/src/CodeSpace.Messages/Failures/FailureCodes.cs b/backend/src/CodeSpace.Messages/Failures/FailureCodes.cs index 492b5e929..dd6d551b8 100644 --- a/backend/src/CodeSpace.Messages/Failures/FailureCodes.cs +++ b/backend/src/CodeSpace.Messages/Failures/FailureCodes.cs @@ -117,7 +117,7 @@ public static class FailureCodes /// This host cannot reserve a filtered-egress run's own /30 subnet, so the run is refused rather than handed one nothing reserved. Remedy: make the reservation directory under the agent-run spool root writable by the worker — a retry on the same host cannot help. public const string SandboxEgressReservationUnavailable = "sandbox_egress_reservation_unavailable"; - /// A network-off run whose model is brokered was refused before it spent anything, because this host would confine it but cannot seal its network to that broker — no ip/nft, no privilege to build a namespace, or a broker that could only listen on loopback — and severing it instead would leave its agent unable to reach any model. Remedy: grant the worker what a sealed namespace needs (root with CAP_NET_ADMIN and CAP_SYS_ADMIN, ip and nft; see backend/Dockerfile.worker) — a retry on the same host helps only once it can build one, which it re-checks at most once a minute. + /// A run whose model is brokered and whose network is its own — off, or narrowed to an allowlist — was refused before it spent anything, because this host would confine it but has no way to carry it to that broker: the codespace-mcp helper that relays the CLI's broker address is missing or cannot run the relay (a build from before it, a self-contained publish), or the broker could not open the run's socket. Launching it anyway would leave its agent unable to reach any model. The wire name predates the relay and is kept, because the supervisor and stored results key on it. Remedy: install this release's helper where the worker looks for it (next to the worker's assembly, or CODESPACE_MCP_PROXY_PATH), or fix what stopped the socket from binding (the spool directory's permissions, a path past the socket-length cap) — a retry lands on another worker, which may have both. public const string SandboxSealedEgressUnavailable = "sandbox_sealed_egress_unavailable"; /// A run's model credential could not be brokered on a deployment that requires confinement, so the run is refused rather than handed the tenant's long-lived provider key. Remedy: make the worker able to bind a broker listener, use a harness that honours a base-URL override, or store an upstream endpoint on the credential — a retry on the same host cannot help. diff --git a/backend/tests/CodeSpace.IntegrationTests/Workflows/AgentRunExecutorCredentialBrokerTests.cs b/backend/tests/CodeSpace.IntegrationTests/Workflows/AgentRunExecutorCredentialBrokerTests.cs index 2f7db264a..737a1499f 100644 --- a/backend/tests/CodeSpace.IntegrationTests/Workflows/AgentRunExecutorCredentialBrokerTests.cs +++ b/backend/tests/CodeSpace.IntegrationTests/Workflows/AgentRunExecutorCredentialBrokerTests.cs @@ -104,28 +104,32 @@ public async Task A_brokered_run_records_that_it_was_brokered_and_says_nothing_a } [Theory] - [InlineData(AgentAutonomyLevel.Standard, true)] // network off: the port a confining runner seals the run to its broker with - [InlineData(AgentAutonomyLevel.Trusted, false)] // network on already reaches its broker; nothing to seal + [InlineData(AgentAutonomyLevel.Standard, true)] // network off: the port and socket the runner's relay carries the run to its broker with + [InlineData(AgentAutonomyLevel.Trusted, false)] // network on already reaches its broker on loopback; nothing to relay public async Task A_brokered_network_off_launch_carries_its_lease_port_to_the_runner(AgentAutonomyLevel autonomy, bool expectPort) { if (OperatingSystem.IsWindows()) return; - // The WIRING pin: ApplySealedEgress's unit tests would pass even if HardenSpec never fed it the lease, and a - // network-off brokered run would then be severed from its broker on every host that confines. + // The WIRING pin: ApplyModelBrokerChannel's unit tests would pass even if HardenSpec never fed it the lease, and a + // network-off brokered run would then be severed from its broker on every host that confines. The socket is the + // executor's own on Linux — it asks for one only there, and only for a run whose network is its own. var teamId = await SeedTeamAsync(); var credId = await SeedModelCredentialAsync(teamId, BrokeredProvider, "sk-sealed-port-fixture"); var runId = await CreateTaskRunAsync(teamId, new AgentTask { Goal = "scripted", Harness = "scripted-projector", Model = "test-model", ModelCredentialId = credId, Autonomy = autonomy, Permissions = AgentAutonomyPolicy.Derive(autonomy) }); var runner = new SpecRecordingDurableRunner(); - using var broker = new LoopbackModelCredentialBroker(); + using var broker = new RecordingBroker(new LoopbackModelCredentialBroker()); var harness = new BrokerableScriptedHarness(BrokeredProvider, "echo done"); await ExecuteAsync(runId, harness, runners: new SandboxRunnerRegistry(new ISandboxRunner[] { runner }), credentialBroker: broker); var launched = runner.Launched.ShouldNotBeNull("the executor must have launched — a null spec means it failed before reaching the runner"); var leasePort = new Uri(harness.BuiltTask!.Environment["SCRIPTED_BASE_URL"].Replace(SandboxSpec.ModelBrokerHostToken, "127.0.0.1", StringComparison.Ordinal)).Port; + var expectSocket = expectPort && OperatingSystem.IsLinux(); launched.ModelBrokerPort.ShouldBe(expectPort ? leasePort : null, $"a {autonomy} brokered run must {(expectPort ? "" : "not ")}hand the runner the port of the lease its CLI was pointed at"); + broker.Opens.ShouldHaveSingleItem().SocketPath.ShouldBe(expectSocket ? launched.ModelBrokerSocketPath : null, $"the lease asks for a socket exactly where its child will need one (linux={OperatingSystem.IsLinux()})"); + (launched.ModelBrokerSocketPath is not null).ShouldBe(expectSocket, "and the runner is handed the socket the lease bound, so its relay knows where to carry the CLI"); } [Fact] @@ -168,21 +172,29 @@ public async Task A_sealed_launch_s_record_survives_the_column_and_reads_back_as .ShouldBe("Network: off (Standard) — confined: egress sealed to the run's model broker"); } - [Fact] - public async Task A_network_off_brokered_run_its_runner_cannot_seal_is_refused_before_it_spends_or_launches() + [Theory] + [InlineData(false)] // no helper where the worker looks + [InlineData(true)] // a helper built before the relay, which CODESPACE_MCP_PROXY_PATH can name: its MCP proxy reads the relay's argv as its own + public async Task A_network_off_brokered_run_a_confining_runner_cannot_relay_is_refused_before_it_spends_or_launches(bool preRelayHelper) { if (OperatingSystem.IsWindows()) return; // The ORDER is the claim: the runner is asked while a refusal still costs nothing. A run owned by a workflow run // records a spend row the moment it is admitted — even with no cap — so an absent row proves the refusal came - // first; a null launch proves no process started; and the lease must be withdrawn like any finished run's. + // first; a null launch proves no process started; and the lease must be withdrawn like any finished run's. The + // runner confines and its admission is the local runner's own, over a helper path that does not exist, or over + // one that is there and does not answer as the relay — admitted, its CLI would never start, after the spend. + using var helper = new TempDir(); + var helperPath = Path.Combine(helper.Path, "codespace-mcp"); + if (preRelayHelper) await WritePreRelayHelperAsync(helperPath); + var teamId = await SeedTeamAsync(); - var credId = await SeedModelCredentialAsync(teamId, BrokeredProvider, "sk-unsealable-fixture"); + var credId = await SeedModelCredentialAsync(teamId, BrokeredProvider, "sk-unrelayable-fixture"); var workflowRunId = await SeedCappedWorkflowRunAsync(teamId, capUsd: null); var runId = await CreateTaskRunInWorkflowAsync(teamId, workflowRunId, new AgentTask { Goal = "scripted", Harness = "scripted-projector", Model = "claude-opus-4-8", ModelCredentialId = credId, MaxCostUsd = 5m }); - var runner = new SealRefusingRunner(); + var runner = new ConfiningRunner(helperPath); - using var broker = new LoopbackModelCredentialBroker(); + using var broker = new RecordingBroker(new LoopbackModelCredentialBroker(), SocketStandIn(runId)); var harness = new BrokerableScriptedHarness(BrokeredProvider, "echo done"); await ExecuteAsync(runId, harness, runners: new SandboxRunnerRegistry(new ISandboxRunner[] { runner }), credentialBroker: broker); @@ -194,8 +206,11 @@ public async Task A_network_off_brokered_run_its_runner_cannot_seal_is_refused_b run.Status.ShouldBe(AgentRunStatus.Failed); result.ExitReason.ShouldBe(CodeSpace.Messages.Failures.FailureCodes.SandboxSealedEgressUnavailable, "the refusal lands under its own code, which the supervisor steers on"); - runner.Asked.ShouldHaveSingleItem().Port.ShouldBe(leasePort, "the runner is asked about the spec the run would have launched, lease port and all"); - runner.Asked[0].Reachable.ShouldBe(CodeSpace.Core.Services.Agents.Sandbox.Isolation.FilteredEgressNetns.IsSupported, "and is told whether the lease bound where a namespace can reach it"); + run.Error.ShouldNotBeNull().ShouldContain(CodeSpace.Core.Services.Agents.Sandbox.Exceptions.SealedEgressUnavailableException.CauseRelayMissing, customMessage: "and names the wall: no helper that can run the relay"); + run.Error.ShouldContain(preRelayHelper ? "did not answer as the relay" : "looked for", customMessage: "and says what was wrong with the helper where it looked"); + var asked = runner.Asked.ShouldHaveSingleItem(); + asked.ModelBrokerPort.ShouldBe(leasePort, "the runner is asked about the spec the run would have launched, lease port and all"); + asked.ModelBrokerSocketPath.ShouldNotBeNull("and the socket its lease bound, so only the helper is what refuses it"); runner.Launched.ShouldBeNull("a refused run starts no process"); (await scope.Resolve().BudgetReservation.AsNoTracking().Where(r => r.TeamId == teamId).ToListAsync()) .ShouldBeEmpty("a refusal before admission claims nothing — an admitted run here would have recorded an unbudgeted row"); @@ -203,22 +218,72 @@ public async Task A_network_off_brokered_run_its_runner_cannot_seal_is_refused_b } [Fact] - public async Task An_unbrokered_network_off_run_is_admitted_by_a_runner_that_cannot_seal() + public async Task A_runner_that_confines_but_cannot_build_a_namespace_admits_a_brokered_network_off_run_and_its_handle_names_the_socket() { if (OperatingSystem.IsWindows()) return; - // Nothing to seal, nothing to refuse: a run with no broker lease — model-less, or keyless — launches exactly as + // The shipped worker's posture, before the relay the one this code refused: bubblewrap confines, but the worker + // is non-root and may not build a network namespace of its own. Its admission is the local runner's own over the + // helper this release builds, which it asks; nothing it decides reads whether a namespace can be built. The run + // is admitted, spends, launches with no namespace key, and its durable handle carries the socket a re-attach will + // re-open. + var helperPath = BuiltMcpProxy.ExecutablePathOrNull().ShouldNotBeNull("fixture: the codespace-mcp apphost is built beside its dll (the build-only ProjectReference in CodeSpace.IntegrationTests.csproj)"); + + var teamId = await SeedTeamAsync(); + var credId = await SeedModelCredentialAsync(teamId, BrokeredProvider, "sk-relayable-fixture"); + var workflowRunId = await SeedCappedWorkflowRunAsync(teamId, capUsd: null); + var runId = await CreateTaskRunInWorkflowAsync(teamId, workflowRunId, new AgentTask { Goal = "scripted", Harness = "scripted-projector", Model = "claude-opus-4-8", ModelCredentialId = credId, MaxCostUsd = 5m }); + var runner = new ConfiningRunner(helperPath); + + using var broker = new RecordingBroker(new LoopbackModelCredentialBroker(), SocketStandIn(runId)); + + await ExecuteAsync(runId, new BrokerableScriptedHarness(BrokeredProvider, "echo done"), runners: new SandboxRunnerRegistry(new ISandboxRunner[] { runner }), credentialBroker: broker); + + using var scope = _fixture.BeginScope(); + var run = await scope.Resolve().GetAsync(runId, CancellationToken.None); + var launched = runner.Launched.ShouldNotBeNull($"a confining runner that can relay launches the run; it ended {run.Status}: {run.Error}"); + var socketPath = launched.ModelBrokerSocketPath.ShouldNotBeNull("the runner is handed the socket the lease bound"); + + runner.Asked.ShouldHaveSingleItem().ModelBrokerSocketPath.ShouldBe(socketPath, "admitted because the lease bound its socket and the helper runs the relay"); + (await scope.Resolve().BudgetReservation.AsNoTracking().Where(r => r.TeamId == teamId).ToListAsync()).ShouldNotBeEmpty("an admitted run is admitted to spend"); + + var handle = HandleOf(runId).ShouldNotBeNull(); + handle.ModelBrokerSocketPath.ShouldBe(socketPath, "the durable handle is the only record of the lease's socket a later worker has"); + handle.EgressNetnsKey.ShouldBeNull("the run got no namespace of the worker's: its broker is reached through the relay, which needs none"); + } + + [Fact] + public async Task An_unbrokered_network_off_run_is_admitted_by_a_runner_that_cannot_relay() + { + if (OperatingSystem.IsWindows()) return; + + // Nothing to reach, nothing to refuse: a run with no broker lease — model-less, or keyless — launches exactly as // it did before, even where a brokered one would be refused. var teamId = await SeedTeamAsync(); var runId = await CreateScriptedRunAsync(teamId); - var runner = new SealRefusingRunner(); + var runner = new ConfiningRunner(helperPath: Path.Combine(Path.GetTempPath(), "cs-no-relay-" + Guid.NewGuid().ToString("N"), "codespace-mcp")); await ExecuteAsync(runId, new ScriptedHarness("printf 'one\\n'"), runners: new SandboxRunnerRegistry(new ISandboxRunner[] { runner })); - runner.Asked.ShouldHaveSingleItem().Port.ShouldBeNull(); + runner.Asked.ShouldHaveSingleItem().ModelBrokerPort.ShouldBeNull(); runner.Launched.ShouldNotBeNull("an unbrokered network-off run is launched, not refused"); } + /// A codespace-mcp from before the relay, as far as the admission's question goes: its MCP proxy reads any argv as its own and exits with its usage error. + private static async Task WritePreRelayHelperAsync(string path) + { + await File.WriteAllTextAsync(path, "#!/bin/sh\necho 'The MCP proxy requires a socket path in CODESPACE_MCP_SOCKET.' >&2\nexit 2\n"); + File.SetUnixFileMode(path, UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute); + } + + /// + /// The socket a network-off run's lease is served on, where the executor mints none itself: it asks for one only on + /// Linux, and these tests run on developers' macOS hosts too. Null on Linux, so there the socket under test is the + /// executor's own. + /// + private static string? SocketStandIn(Guid runId) => + OperatingSystem.IsLinux() ? null : LocalProcessRunner.ModelBrokerSocketPathFor(runId.ToString("N"), CodeSpace.Core.Services.Agents.Mcp.McpRunToken.MintPathId()); + [Fact] public async Task A_run_whose_credential_cannot_be_brokered_discloses_the_direct_injection() { @@ -878,31 +943,35 @@ await WaitUntilAsync(() => !ProcessIsAlive(handle.ProcessId), TimeSpan.FromSecon } [Theory] - [InlineData(true)] // the lease was served over a socket: the handle records it, and the re-attach re-opens it - [InlineData(false)] // it was not: the handle is exactly a pre-field handle, and the re-attach takes the legacy re-bind + [InlineData(true)] // a network-off run: its lease is served over a socket, the handle records it, and the re-attach re-opens it + [InlineData(false)] // a network-on run: no socket, so the handle is exactly a pre-field handle, and the re-attach takes the legacy re-bind public async Task A_reattach_re_opens_the_broker_socket_its_handle_recorded_and_a_handle_without_one_takes_the_legacy_rebind(bool socket) { if (OperatingSystem.IsWindows()) return; var teamId = await SeedTeamAsync(); var credId = await SeedModelCredentialAsync(teamId, BrokeredProvider, "sk-socket-rebind-fixture"); - var runId = await CreateRunWithCredentialAsync(teamId, credId); + var autonomy = socket ? AgentAutonomyLevel.Standard : AgentAutonomyLevel.Trusted; + var runId = await CreateTaskRunAsync(teamId, new AgentTask { Goal = "scripted", Harness = "scripted-projector", Model = "test-model", ModelCredentialId = credId, Autonomy = autonomy, Permissions = AgentAutonomyPolicy.Derive(autonomy) }); - // Nothing in production mints a broker socket yet, so the launch's lease is asked for one here — at the path the - // production layout gives it — which is the one thing this test substitutes. Everything after the open is the - // real executor: the handle it stamps, the jsonb column it persists to, and the re-attach that reads it back. - var socketPath = socket ? LocalProcessRunner.ModelBrokerSocketPathFor(runId.ToString("N"), CodeSpace.Core.Services.Agents.Mcp.McpRunToken.MintPathId()) : null; + // On Linux the executor asks for the socket itself; elsewhere it asks for none, so a network-off run's lease is + // asked for one here, at the path the production layout gives it. Everything after the open is the real + // executor: the handle it stamps, the jsonb column it persists to, and the re-attach that reads it back. + var standIn = socket ? SocketStandIn(runId) : null; using var release = new TempDir(); var releaseFile = Path.Combine(release.Path, "release"); var harness = new BrokerableScriptedHarness(BrokeredProvider, $"while [ ! -f '{releaseFile}' ]; do sleep 0.2; done; echo done"); + string? socketPath = null; try { - var (handle, childBaseUrl) = await DrainLeavingTheAgentRunningAsync(runId, harness, socketPath); + var (handle, childBaseUrl) = await DrainLeavingTheAgentRunningAsync(runId, harness, standIn); var runToken = handle.ModelBrokerRunToken.ShouldNotBeNull(); + socketPath = handle.ModelBrokerSocketPath; - handle.ModelBrokerSocketPath.ShouldBe(socketPath, "the durable handle is the only record of the lease's socket a later worker has — it must carry exactly the path the lease bound, and nothing for a lease that bound none"); + (socketPath is not null).ShouldBe(socket, "the durable handle is the only record of the lease's socket a later worker has — it must carry the path the lease bound, and nothing for a lease that bound none"); + if (standIn is not null) socketPath.ShouldBe(standIn, "exactly the path the lease bound"); if (!socket) RunnerHandleJsonOf(runId).ShouldNotContain("modelBrokerSocketPath", customMessage: "a handle whose lease had no socket must be stored exactly as one written before the field existed"); using var workerB = new RecordingBroker(LoopbackModelCredentialBroker.ForTest(new AlwaysOkUpstream())); @@ -957,16 +1026,22 @@ private static async Task ReachesUpstreamOverSocketAsync(string socketPath } /// - /// The production broker, with the two things the socket test needs from its side of the seam: a socket asked for - /// on the lease the launch opens (production mints none yet — so the helper's path fills in only where the executor - /// asked for none, and never overwrites one it did ask for), and a record of every re-bind the re-attach asks for — - /// which is the executor's reading of the stored handle, observed exactly where it leaves the executor. + /// The production broker, with what the socket tests need from its side of the seam: a record of every open the + /// launch asks for and every re-bind the re-attach asks for — the executor's own requests, observed exactly where + /// they leave the executor — and, where the executor asks for no socket (it mints one only on Linux), a socket + /// asked for on the lease the launch opens, which fills in only there and never overwrites one it did ask for. /// private sealed class RecordingBroker(LoopbackModelCredentialBroker inner, string? socketPath = null) : IModelCredentialBroker, IDisposable { + public List Opens { get; } = []; + public List Rebinds { get; } = []; - public Task OpenAsync(ModelCredentialLeaseRequest request, CancellationToken cancellationToken) => inner.OpenAsync(request with { SocketPath = socketPath ?? request.SocketPath }, cancellationToken); + public Task OpenAsync(ModelCredentialLeaseRequest request, CancellationToken cancellationToken) + { + Opens.Add(request); + return inner.OpenAsync(request with { SocketPath = request.SocketPath ?? socketPath }, cancellationToken); + } public Task RebindAsync(ModelCredentialRebindRequest request, CancellationToken cancellationToken) { @@ -1531,20 +1606,25 @@ private async Task CreateTaskRunInWorkflowAsync(Guid teamId, Guid workflow return run.Id; } - /// A durable runner that refuses admission to any spec carrying a broker port — the shape the local runner takes on a host that confines but cannot seal — recording what it was asked and what it launched. - private sealed class SealRefusingRunner : ISandboxRunner, ISandboxDurableRunner, ISandboxEgressAdmission + /// + /// A durable runner on a host that CONFINES, whatever host the test runs on: its admission is the local runner's + /// own (LocalProcessRunner.EnsureEgressAdmissible) with bubblewrap taken as present and the relay's helper at + /// . It builds no namespace, so its handles carry no namespace key — the non-root + /// worker's posture. Records what it was asked and what it launched. + /// + private sealed class ConfiningRunner(string helperPath) : ISandboxRunner, ISandboxDurableRunner, ISandboxEgressAdmission { public string Kind => LocalProcessRunner.LocalKind; - public List<(int? Port, bool Reachable)> Asked { get; } = new(); + public List Asked { get; } = new(); public SandboxSpec? Launched { get; private set; } - public void EnsureEgressAdmissible(SandboxSpec spec, bool modelBrokerReachableFromNamespace) + public void EnsureEgressAdmissible(SandboxSpec spec) { - Asked.Add((spec.ModelBrokerPort, modelBrokerReachableFromNamespace)); + Asked.Add(spec); - if (spec.ModelBrokerPort is not null) throw new CodeSpace.Core.Services.Agents.Sandbox.Exceptions.SealedEgressUnavailableException(CodeSpace.Core.Services.Agents.Sandbox.Exceptions.SealedEgressUnavailableException.CauseNoPrivilege); + LocalProcessRunner.EnsureEgressAdmissible(spec, confines: true, helperPath); } public Task RunAsync(SandboxSpec spec, CancellationToken cancellationToken) => @@ -1581,10 +1661,12 @@ private sealed class BrokerableScriptedHarness(string provider, string script) : public AgentTask? BuiltTask { get; private set; } + // Network as the real harnesses derive it (ClaudeCodeHarness, CodexHarness): on exactly when the tier grants it, + // which is what the executor's broker channel reads off the built spec. public SandboxSpec BuildInvocation(AgentTask task) { BuiltTask = task; - return new SandboxSpec { Command = "/bin/sh", Args = new[] { "-c", script }, WorkingDirectory = task.WorkspaceDirectory, Environment = task.Environment, TimeoutSeconds = task.TimeoutSeconds }; + return new SandboxSpec { Command = "/bin/sh", Args = new[] { "-c", script }, WorkingDirectory = task.WorkspaceDirectory, Environment = task.Environment, TimeoutSeconds = task.TimeoutSeconds, AllowNetwork = task.Permissions.Network == AgentNetworkAccess.On }; } // KEEPS the line's structured root, as every real harness's parse does — AgentRunFacts reads only diff --git a/backend/tests/CodeSpace.IntegrationTests/Workflows/AgentRunExecutorTests.cs b/backend/tests/CodeSpace.IntegrationTests/Workflows/AgentRunExecutorTests.cs index 796da9d1b..2d12fc9c2 100644 --- a/backend/tests/CodeSpace.IntegrationTests/Workflows/AgentRunExecutorTests.cs +++ b/backend/tests/CodeSpace.IntegrationTests/Workflows/AgentRunExecutorTests.cs @@ -1815,6 +1815,7 @@ public ValueTask DisposeAsync() private async Task ExecuteAsync(Guid runId, IAgentHarness harness, IAgentRunLogCaptureBridge? logCapture = null, IAgentRunCompletionNotifier? notifier = null, ISandboxRunnerRegistry? runners = null, CodeSpace.Core.Services.Agents.Credentials.IModelCredentialBroker? credentialBroker = null, CodeSpace.Core.Services.Review.IStructuredCritic? critic = null, CancellationToken cancellationToken = default, Microsoft.Extensions.Hosting.IHostApplicationLifetime? lifetime = null, bool productionCapturePlanes = false) { + using var relay = credentialBroker is null ? null : RelayHelper.UseBuilt(); using var scope = _fixture.BeginScope(); await NewExecutor(scope, harness, logCapture, notifier, runners, credentialBroker, critic, lifetime, productionCapturePlanes).ExecuteAsync(runId, cancellationToken); @@ -1823,11 +1824,53 @@ private async Task ExecuteAsync(Guid runId, IAgentHarness harness, IAgentRunLogC /// Drive the RE-ATTACH entry point with the same executor wiring uses — the terminal a run reaches when it finishes on a worker that never saw its launch. The broker is the NEXT worker's, never the launching one's: a re-attach that shared a broker with the launch would never exercise the re-bind at all. private async Task ReattachAsync(AgentRunReattachReservation reservation, IAgentHarness harness, CodeSpace.Core.Services.Agents.Credentials.IModelCredentialBroker? credentialBroker = null, CancellationToken cancellationToken = default) { + using var relay = credentialBroker is null ? null : RelayHelper.UseBuilt(); using var scope = _fixture.BeginScope(); await NewExecutor(scope, harness, credentialBroker: credentialBroker).ReattachAsync(reservation, cancellationToken); } + /// + /// Points the runner at the codespace-mcp helper this build produced for as long as a brokered run executes, and + /// restores what was there. On a host that confines, a brokered run whose network is its own reaches its broker + /// through that helper's relay, as production does with the helper beside the worker's assembly — and this + /// assembly's bin deliberately carries no helper (see the csproj), so without this the admission would refuse every + /// such run here. Nothing changes on a host that does not confine, where no run is relayed. The collection runs its + /// tests one at a time, so the process-wide variable is this test's alone while it is set; within one test a + /// launch and a re-attach can overlap in any order, so the variable is set by the first user and restored by the + /// last. + /// + private sealed class RelayHelper : IDisposable + { + private static readonly object Gate = new(); + private static int _users; + private static string? _previous; + + private RelayHelper() { } + + public static RelayHelper UseBuilt() + { + var built = BuiltMcpProxy.ExecutablePathOrNull() ?? throw new InvalidOperationException("The codespace-mcp apphost was not built beside its dll; the build-only ProjectReference in CodeSpace.IntegrationTests.csproj builds it, and a brokered run on a confining host cannot reach its broker without it."); + + lock (Gate) + { + if (_users++ == 0) _previous = Environment.GetEnvironmentVariable(LocalProcessRunner.McpProxyPathEnvVar); + + Environment.SetEnvironmentVariable(LocalProcessRunner.McpProxyPathEnvVar, built); + } + + return new RelayHelper(); + } + + public void Dispose() + { + lock (Gate) + { + if (--_users == 0) Environment.SetEnvironmentVariable(LocalProcessRunner.McpProxyPathEnvVar, _previous); + } + } + } + private AgentRunExecutor NewExecutor(Autofac.ILifetimeScope scope, IAgentHarness harness, IAgentRunLogCaptureBridge? logCapture = null, IAgentRunCompletionNotifier? notifier = null, ISandboxRunnerRegistry? runners = null, CodeSpace.Core.Services.Agents.Credentials.IModelCredentialBroker? credentialBroker = null, CodeSpace.Core.Services.Review.IStructuredCritic? critic = null, Microsoft.Extensions.Hosting.IHostApplicationLifetime? lifetime = null, bool productionCapturePlanes = false, IAgentRunService? runs = null, CodeSpace.Core.Services.RunData.IRunDataCompletenessWriter? completeness = null, TimeProvider? clock = null) { var executor = new AgentRunExecutor( diff --git a/backend/tests/CodeSpace.SandboxTests/DurableLaunchEgressE2ETests.cs b/backend/tests/CodeSpace.SandboxTests/DurableLaunchEgressE2ETests.cs index ab67a18fe..fc7ec41c2 100644 --- a/backend/tests/CodeSpace.SandboxTests/DurableLaunchEgressE2ETests.cs +++ b/backend/tests/CodeSpace.SandboxTests/DurableLaunchEgressE2ETests.cs @@ -1,8 +1,11 @@ using System.Diagnostics; +using CodeSpace.Core.Services.Agents; +using CodeSpace.Core.Services.Agents.Credentials.Broker; using CodeSpace.Core.Services.Agents.Sandbox.Isolation; using CodeSpace.Core.Services.Agents.Sandbox.Runners; using CodeSpace.Messages.Agents; using Shouldly; +using Xunit.Abstractions; namespace CodeSpace.SandboxTests; @@ -18,7 +21,7 @@ namespace CodeSpace.SandboxTests; /// Uses raw IPs over plain HTTP so the signal is purely the egress filter — not DNS, not TLS. /// [Trait("Category", "Sandbox")] -public sealed class DurableLaunchEgressE2ETests +public sealed class DurableLaunchEgressE2ETests(ITestOutputHelper output) { private const string Allowed = "1.1.1.1"; // Cloudflare — allowlisted private const string Denied = "8.8.8.8"; // Google — NOT allowlisted, must be dropped @@ -47,6 +50,83 @@ public async Task The_durable_launch_runs_the_agent_inside_the_filtered_netns_an (await NetnsExistsAsync(NamespaceOf(denyKey))).ShouldBeFalse("the denied run's netns is reaped on the terminal path too"); } + [Fact] + public async Task An_allowlist_run_reaches_its_broker_through_the_relay_and_its_allowlist_still_holds() + { + // An allowlist run's namespace is its own, so its lease's loopback port is not its loopback: it reaches its + // broker through the relay in front of its CLI and the lease's socket — which bubblewrap, sharing the filtered + // namespace, binds read-only — while the allowlist that namespace enforces decides everything else. + if (!FilteredEgressNetns.IsSupported || BubblewrapSandbox.Available is null) return; // the root lane, with ip, nft and bwrap, is authoritative + + var runId = Guid.NewGuid(); + var permissions = new AgentPermissions { Network = AgentNetworkAccess.On, Egress = AgentEgressPolicy.Allowlist }; + var socketPath = AgentRunExecutor.ModelBrokerSocketPathFor(permissions, runId).ShouldNotBeNull("the executor mints a socket for an allowlist run on Linux"); + using var broker = LoopbackModelCredentialBroker.ForTest(new OkUpstream()); + var brokered = (await broker.OpenAsync(new() { RunId = runId, TeamId = Guid.NewGuid(), Epoch = 1, Upstream = new() { Provider = "Anthropic", ApiKey = "sk-allowlist-e2e" }, Ttl = TimeSpan.FromMinutes(5), SocketPath = socketPath }, CancellationToken.None)).ShouldNotBeNull(); + var spec = AgentRunExecutor.ApplyModelBrokerChannel(new SandboxSpec + { + Command = "/usr/bin/python3", Args = ["-c", AllowlistProbe], AllowNetwork = true, EgressAllowlist = [Allowed], TimeoutSeconds = 60, + Environment = new Dictionary { ["BROKER_URL"] = brokered.BaseUrl, ["RUN_TOKEN"] = brokered.RunToken, ["SOCK_PATH"] = socketPath }, + }, brokered); + + spec.ModelBrokerSocketPath.ShouldBe(socketPath, "fixture check: the executor's own hardening stamps an allowlist run with its lease's socket"); + + var key = Guid.NewGuid().ToString("N"); + var runner = new LocalProcessRunner(); + var lines = new List(); + + try + { + var handle = await runner.LaunchAsync(spec, key, CancellationToken.None); + handle.EgressNetnsKey.ShouldBe(key, "an enforceable allowlist still launches the run inside its filtered netns"); + handle.Confinement.ShouldNotBeNull().EgressSealedToBroker.ShouldBeFalse("an allowlist run is filtered, not sealed: it has more than one destination"); + + var result = await runner.AttachAsync(handle, (frame, _) => { lines.Add(frame.Text); return Task.CompletedTask; }, CancellationToken.None); + var probe = string.Join(' ', lines); + + result.Status.ShouldBe(SandboxStatus.Success, $"the probe must run to its end; stderr: {result.Stderr}"); + probe.ShouldContain("broker=200", customMessage: $"the allowlist run's broker answers through the relay; check `ls -la {Path.GetDirectoryName(socketPath)}`; probe: {probe}"); + probe.ShouldContain("allowed=open", customMessage: $"the allowlisted IP is still reachable through the namespace's NAT; probe: {probe}"); + probe.ShouldNotContain("denied=open", customMessage: $"and a host outside the allowlist is still dropped; probe: {probe}"); + SealedEgressE2ETests.AssertSocketDirectoryIsReadOnly(ProbeValue(probe, "sock_unlink"), ProbeValue(probe, "sock_plant"), socketPath); + + output.WriteLine($"[durable-egress-e2e] ran allowlist-relay {probe}"); + } + finally + { + try { Directory.Delete(LocalProcessRunner.SpoolDirectoryFor(key), recursive: true); } catch { /* best-effort */ } + } + + (await NetnsExistsAsync(NamespaceOf(key))).ShouldBeFalse("the run's filtered netns is reaped on completion"); + } + + /// The broker through the relay, the two writes the socket's read-only directory must refuse, and the allowlisted IP and a denied one, from inside the run. + private const string AllowlistProbe = SealedEgressE2ETests.SocketDirectoryWrites + "\n" + """ + import os, socket, urllib.request + req = urllib.request.Request(os.environ['BROKER_URL'] + '/v1/messages', data=b'{}', method='POST', headers={'Authorization': 'Bearer ' + os.environ['RUN_TOKEN'], 'content-type': 'application/json'}) + try: + broker = str(urllib.request.urlopen(req, timeout=10).status) + except Exception as e: + broker = type(e).__name__ + unlink, plant = sock_writes(os.environ['SOCK_PATH']) + def tcp(host): + try: + socket.create_connection((host, 80), timeout=6).close(); return 'open' + except OSError as e: + return type(e).__name__ + print('broker=%s sock_unlink=%s sock_plant=%s allowed=%s denied=%s' % (broker, unlink, plant, tcp('1.1.1.1'), tcp('8.8.8.8'))) + """; + + /// The value the probe printed for (key=value), or ? when it printed none. + private static string ProbeValue(string probe, string key) => + probe.Split(' ').FirstOrDefault(pair => pair.StartsWith(key + "=", StringComparison.Ordinal))?[(key.Length + 1)..] ?? "?"; + + private sealed class OkUpstream : HttpMessageHandler + { + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) => + Task.FromResult(new HttpResponseMessage(System.Net.HttpStatusCode.OK) { Content = new StringContent("{\"ok\":true}") }); + } + /// Launch a real durable run that curls with an allowlist of , observe it to completion, and return the result. Also asserts the run was launched inside a netns keyed by the run. private static async Task DurableCurlAsync(LocalProcessRunner runner, string runKey, string allow, string target) { diff --git a/backend/tests/CodeSpace.SandboxTests/FilteredEgressNetnsE2ETests.cs b/backend/tests/CodeSpace.SandboxTests/FilteredEgressNetnsE2ETests.cs index 4ab2be1e2..6d3f5ce8f 100644 --- a/backend/tests/CodeSpace.SandboxTests/FilteredEgressNetnsE2ETests.cs +++ b/backend/tests/CodeSpace.SandboxTests/FilteredEgressNetnsE2ETests.cs @@ -1,5 +1,8 @@ +using System.Globalization; +using System.Security.Cryptography; using CodeSpace.Core.Services.Agents.Sandbox.Isolation; using Shouldly; +using Xunit.Abstractions; namespace CodeSpace.SandboxTests; @@ -13,10 +16,16 @@ namespace CodeSpace.SandboxTests; /// /// Class-level [Trait("Category", "Sandbox")] — runs in the same privileged gate as the bwrap /// confinement tests. The teardown is the executor's own best-effort netns/table cleanup (no leak between runs). +/// +/// The two arms about the host's own routing (a /30 still held by a namespace that outlived its worker, a policy +/// rule that discards the run's replies) print , which the lane requires. /// [Trait("Category", "Sandbox")] -public sealed class FilteredEgressNetnsE2ETests +public sealed class FilteredEgressNetnsE2ETests(ITestOutputHelper output) { + /// Printed by the arms that must not pass by returning early; the sandbox lane requires one per arm. + public const string RanMarker = "[filtered-egress-e2e] ran"; + // Cloudflare 1.1.1.1 + Google 8.8.8.8 both serve HTTPS on the open internet — so if the filter did NOT enforce, // BOTH would be reachable. The allowlist permits ONLY 1.1.1.1, so 8.8.8.8 being unreachable proves the drop. private const string Allowed = "1.1.1.1"; @@ -68,6 +77,116 @@ public async Task The_durable_setup_teardown_split_enforces_the_filter_and_teard await FilteredEgressNetns.TeardownAsync(runId, CancellationToken.None); } + [Fact] + public async Task A_30_still_held_by_a_run_that_outlived_its_worker_is_not_handed_to_the_next_run() + { + if (!BuildsNamespaces()) return; + + // A run survives its worker by design, and so do its namespace and veth; the reservation lock does not. A fresh + // worker that trusted the lock alone would hand the survivor's /30 to its next allowlist launch, and the kernel + // would split the two runs' replies between two veths. Releasing the survivor's reservation without tearing its + // namespace down is exactly what the restart leaves behind. + var survivor = Guid.NewGuid().ToString("N"); + var next = Guid.NewGuid().ToString("N"); + var first = await FilteredEgressNetns.SetupAsync(survivor, new[] { Allowed }, timeoutSeconds: 20, CancellationToken.None); + + try + { + first.SetupOk.ShouldBeTrue($"the survivor's allowlist namespace must set up on this host: {first.SetupError}"); + EgressSubnetAllocator.Host.Release(survivor); + + var second = await FilteredEgressNetns.SetupAsync(next, new[] { Allowed }, timeoutSeconds: 20, CancellationToken.None); + + try + { + second.SetupOk.ShouldBeTrue($"the next run's allowlist namespace must set up: {second.SetupError}"); + second.HostIp.ShouldNotBe(first.HostIp, "the survivor's /30 is still on its veth; handing it out again routes one run's replies into the other's namespace"); + + output.WriteLine($"{RanMarker} restart-reissue survivor={first.HostIp} next={second.HostIp}"); + } + finally { await FilteredEgressNetns.TeardownAsync(next, CancellationToken.None); } + } + finally { await FilteredEgressNetns.TeardownAsync(survivor, CancellationToken.None); } + } + + [Fact] + public async Task A_host_whose_policy_rule_discards_the_run_s_replies_fails_the_setup_and_leaks_nothing() + { + if (!BuildsNamespaces()) return; + + // The allocator skips what the host's route listing covers, but a null route in a table that a policy rule + // consults before main wins by rule ORDER, not prefix length: every step of the setup succeeds, and then every + // reply to the namespace is discarded. The /30 here is from TEST-NET-1 (RFC 5737), which the allocator never + // hands out, and the rule covers only that /30, so it cannot reach another run on this host. + var third = RandomNumberGenerator.GetInt32(0, 64) * 4; + var lease = new EgressSubnetAllocator.Lease { Cidr = $"192.0.2.{third}/30", HostIp = $"192.0.2.{third + 1}", NsIp = $"192.0.2.{third + 2}" }; + var table = RandomNumberGenerator.GetInt32(10_000, 1_000_000).ToString(CultureInfo.InvariantCulture); + string[] rule = ["pref", "100", "to", lease.Cidr, "lookup", table]; + var runId = Guid.NewGuid().ToString("N"); + var plan = FilteredEgressPlan.Build(runId, new[] { Allowed }, lease); + + // A run of this test killed between its rule add and its cleanup leaves a rule for its /30 in a table this run + // cannot name; left there, it would fail this run's control and blame the check. + for (var stale = 0; stale < 8 && await RunHostExitAsync(["ip", "rule", "del", "pref", "100", "to", lease.Cidr]) == 0; stale++) { } + + try + { + var control = await FilteredEgressNetns.ApplyAsync(runId, plan, timeoutSeconds: 20, CancellationToken.None); + control.SetupOk.ShouldBeTrue($"control: the same plan must set up on a host with no such rule, or the check refuses what it should admit: {control.SetupError}"); + await FilteredEgressNetns.TeardownAsync(runId, CancellationToken.None); + + (await RunHostExitAsync(["ip", "route", "add", "unreachable", "192.0.2.0/24", "table", table])).ShouldBe(0, "setup: the null route — broader than a /30, which the route listing ignores — must be installable in its own table"); + (await RunHostExitAsync(["ip", "rule", "add", .. rule])).ShouldBe(0, "setup: the rule that consults it before main must be installable"); + + var refused = await FilteredEgressNetns.ApplyAsync(runId, plan, timeoutSeconds: 20, CancellationToken.None); + + refused.SetupOk.ShouldBeFalse("a namespace the host can never answer must fail its setup, not admit a run that spends its timeout unanswered"); + refused.SetupError.ShouldNotBeNull().ShouldContain(string.Join(' ', plan.RouteCheckArgv), customMessage: "the refusal names the lookup that found it, so an operator can rerun it"); + (await RunHostAsync(["ip", "netns", "list"])).Split('\n').ShouldNotContain(line => line.Trim().Split(' ')[0] == plan.Namespace, "a setup that failed its route check tears its namespace down"); + (await RunHostExitAsync(["ip", "link", "show", plan.VethHost])).ShouldNotBe(0, "and the host end of its veth, with the address on it"); + + output.WriteLine($"{RanMarker} policy-route-discard-dst {refused.SetupError}"); + } + finally + { + await RunHostExitAsync(["ip", "rule", "del", .. rule]); + await RunHostExitAsync(["ip", "route", "flush", "table", table]); + await FilteredEgressNetns.TeardownAsync(runId, CancellationToken.None); + } + } + + /// A lane with ip and nft builds namespaces (the root lane); there, one that cannot be built is a failure, not a skip. + private static bool BuildsNamespaces() + { + if (!FilteredEgressNetns.IsSupported) return false; + + FilteredEgressNetns.CanSeal.ShouldBeTrue("ip and nft are here, but this process could not build a throwaway namespace — an allowlist run could not be filtered on this host"); + return true; + } + + private static async Task RunHostExitAsync(IReadOnlyList argv) + { + var psi = new System.Diagnostics.ProcessStartInfo { FileName = argv[0], UseShellExecute = false, RedirectStandardOutput = true, RedirectStandardError = true }; + foreach (var arg in argv.Skip(1)) psi.ArgumentList.Add(arg); + + using var process = System.Diagnostics.Process.Start(psi)!; + await process.WaitForExitAsync(); + + return process.ExitCode; + } + + private static async Task RunHostAsync(IReadOnlyList argv) + { + var psi = new System.Diagnostics.ProcessStartInfo { FileName = argv[0], UseShellExecute = false, RedirectStandardOutput = true, RedirectStandardError = true }; + foreach (var arg in argv.Skip(1)) psi.ArgumentList.Add(arg); + + using var process = System.Diagnostics.Process.Start(psi)!; + var stdout = await process.StandardOutput.ReadToEndAsync(); + await process.WaitForExitAsync(); + + return stdout; + } + private static Task CurlInFilteredNetnsAsync(string allow, string target) => FilteredEgressNetns.RunAsync( runId: Guid.NewGuid().ToString("N"), diff --git a/backend/tests/CodeSpace.SandboxTests/ModelCredentialBrokerNetnsE2ETests.cs b/backend/tests/CodeSpace.SandboxTests/ModelCredentialBrokerNetnsE2ETests.cs index f7310bddf..48ed4438d 100644 --- a/backend/tests/CodeSpace.SandboxTests/ModelCredentialBrokerNetnsE2ETests.cs +++ b/backend/tests/CodeSpace.SandboxTests/ModelCredentialBrokerNetnsE2ETests.cs @@ -1,4 +1,5 @@ using System.Diagnostics; +using System.Security.Cryptography; using CodeSpace.Core.Services.Agents.Credentials.Broker; using CodeSpace.Core.Services.Agents.Sandbox.Isolation; using CodeSpace.Core.Services.Agents.Sandbox.Runners; @@ -9,24 +10,21 @@ namespace CodeSpace.SandboxTests; /// -/// 🟢 Sandbox isolation E2E (high fidelity, Rule 12): the REAL model-credential broker reached by a REAL process -/// inside a REAL deny-by-default network namespace, over a live kernel. Needs ip + nft + CAP_NET_ADMIN, so it runs -/// for real ONLY in the privileged sandbox-isolation CI job; elsewhere -/// is false and it degrade-skips. +/// 🟢 Sandbox isolation E2E (high fidelity, Rule 12): the REAL model-credential broker reached by a REAL process in a +/// network namespace of its own, over a live kernel. /// -/// The claim it settles. A sealed run's whole point is that its broker is the only way out — so "the -/// broker is reachable from inside" cannot be argued from the code, it has to be observed. The namespace is the -/// production sealed one (): no route, no NAT, no DNS, and an input -/// filter admitting only the lease's own port on the gateway. The broker still answers, because a packet addressed to -/// the host's own veth address is delivered locally (INPUT) and that one port is what the filter admits. If that ever -/// stops being true, every sealed brokered run loses its model and this test is where it shows. +/// The claim it settles. A namespaced run cannot reach the broker's loopback port, so "the broker is +/// reachable from inside" cannot be argued from the code, it has to be observed: through the per-run socket, bound +/// read-only into the sandbox (, +/// which needs only bubblewrap and so runs as root and as the unprivileged worker uid alike), through the relay the +/// production chain puts in front of the CLI, for a network-off and an allowlist child, until the lease is revoked — +/// and, for a run launched before the relay, at its namespace's gateway on a legacy re-bind, until its teardown by name +/// removes the namespace and the seal it carried. /// /// The second claim is the flip side: the bearer the sandbox holds is NOT the tenant's key. Sent straight to the /// provider it buys nothing, so a token that escapes a run is not a credential. /// -/// The third is the per-run socket (), -/// which needs only bubblewrap — no ip, no nft, no privilege — so it runs as root and as an unprivileged worker uid -/// alike, and prints with the uid it ran as. +/// Every arm that ran prints with the uid it ran as, which the lanes require. /// [Trait("Category", "Sandbox")] public sealed class ModelCredentialBrokerNetnsE2ETests(ITestOutputHelper output) @@ -50,7 +48,10 @@ public sealed class ModelCredentialBrokerNetnsE2ETests(ITestOutputHelper output) /// reports the run restored. /// [Fact] - public async Task A_severed_child_reaches_its_broker_through_the_lease_socket_and_the_same_child_reaches_the_next_worker() + public Task A_severed_child_reaches_its_broker_through_the_lease_socket_and_the_same_child_reaches_the_next_worker() => SocketChannelAsync(lane: "root"); + + /// The socket-channel arm, for either lane: see the test above. + internal async Task SocketChannelAsync(string lane) { if (BubblewrapSandbox.Available is not { } bwrap) { @@ -68,9 +69,9 @@ public async Task A_severed_child_reaches_its_broker_through_the_lease_socket_an var brokered = (await workerA.OpenAsync(LeaseFor(runId, teamId) with { SocketPath = context.SocketPath }, CancellationToken.None)).ShouldNotBeNull("the broker must be able to open a lease on a host that confines"); brokered.SocketPath.ShouldBe(context.SocketPath, $"the lease must bind its socket at the path it was given; check `ls -la {context.SocketDirectory}`"); - brokered.ReachableFromNamespace.ShouldBeFalse("a lease served over a socket binds its TCP listener on loopback only, even on a host that could build namespaces — its namespaced children come in through the socket"); + workerA.ListenerPrefixForTest(runId).ShouldBe($"http://127.0.0.1:{brokered.RebindPort}/", "a lease served over a socket binds its TCP listener on loopback only, even on a host that could build namespaces — its namespaced children come in through the socket"); - using var child = SeveredChild.Start(bwrap, context, ReachableUrl(brokered, gatewayIp: null), brokered.RunToken); + using var child = SeveredChild.Start(bwrap, context, ReachableUrl(brokered), brokered.RunToken); (await child.AskAsync("call")).ShouldBe("200", $"a severed child must reach its broker through the socket its sandbox binds read-only; diagnose by hand with `curl --unix-socket {context.SocketPath} -X POST -H 'Authorization: Bearer ' http://127.0.0.1//v1/messages`. Child stderr: {child.Stderr}"); (await child.AskAsync("tcp")).ShouldBe("refused", "the child's network is its own: the broker's TCP port on the host's loopback does not exist in it, so the socket is its one door"); @@ -83,7 +84,7 @@ public async Task A_severed_child_reaches_its_broker_through_the_lease_socket_an (await child.AskAsync("call")).ShouldStartWith("error:", customMessage: "with worker A gone the child's call must fail — that is the restart this arm is about"); var upstreamB = new AlwaysOkUpstream(); - using var workerB = LoopbackModelCredentialBroker.ForTest(upstreamB); + using var workerB = LoopbackModelCredentialBroker.ForTest(upstreamB, logger: new TestOutputLogger(output)); (await workerB.RebindAsync(RebindOf(brokered, runId, teamId, epoch: 2) with { SocketPath = context.SocketPath }, CancellationToken.None)).ShouldBeTrue("worker B must re-open the run's recorded address, socket and all"); @@ -96,7 +97,7 @@ public async Task A_severed_child_reaches_its_broker_through_the_lease_socket_an (await child.AskAsync("call")).ShouldStartWith("error:", customMessage: "a revoked lease's socket must answer nothing"); upstreamB.Calls.ShouldBe(1, "and nothing more reached the provider"); - output.WriteLine($"{RanMarker} socket-channel uid={EffectiveUid()}"); + output.WriteLine($"{RanMarker} {(lane == "root" ? "" : lane + " ")}socket-channel uid={EffectiveUid()}"); } finally { workerA.Dispose(); } } @@ -227,118 +228,211 @@ def unlink(): } [Theory] - [InlineData(true)] // the network-off run's sealed namespace — its one destination is this lease's port - [InlineData(false)] // an allowlist run's namespace with nothing allowed — reaches the worker through the same gateway - public async Task A_namespaced_run_reaches_its_broker_and_is_refused_the_moment_the_lease_is_revoked(bool sealedToBroker) + [InlineData(false)] // a network-off run: bubblewrap gives it a fresh namespace with only loopback + [InlineData(true)] // an allowlist run: the filtered namespace, which bubblewrap shares + public async Task A_namespaced_run_reaches_its_broker_and_is_refused_the_moment_the_lease_is_revoked(bool allowlist) { - if (!FilteredEgressNetns.IsSupported) return; // no ip/nft (macOS dev / non-privileged) → the privileged CI job is authoritative + if (BubblewrapSandbox.Available is not { } bwrap || allowlist && !FilteredEgressNetns.IsSupported) return; // the sandbox lane, with bwrap (and ip + nft for the allowlist arm), is authoritative var upstream = new AlwaysOkUpstream(); using var broker = LoopbackModelCredentialBroker.ForTest(upstream); + using var context = new SocketChannelContext(); var runId = Guid.NewGuid(); - - var brokered = await broker.OpenAsync( - new() { RunId = runId, TeamId = Guid.NewGuid(), Epoch = 1, Upstream = new() { Provider = "Anthropic", ApiKey = "sk-e2e-upstream-key" }, Ttl = TimeSpan.FromMinutes(5) }, - CancellationToken.None); - - brokered.ShouldNotBeNull("the broker must be able to listen on a host that can build filtered-egress namespaces — a sealed run has no other route to a model"); - - // Both production namespaces a brokered run is launched into: the sealed one, and the allowlist one, whose - // broker is reached as a local delivery the forward filter never sees. See the class remarks. + var brokered = (await broker.OpenAsync(LeaseFor(runId, Guid.NewGuid()) with { SocketPath = context.SocketPath }, CancellationToken.None)).ShouldNotBeNull("the broker must be able to lease on a host that confines"); var netnsKey = Guid.NewGuid().ToString("N"); - var setup = sealedToBroker - ? await FilteredEgressNetns.SetupSealedAsync(netnsKey, brokered!.RebindPort!.Value, timeoutSeconds: 20, CancellationToken.None) - : await FilteredEgressNetns.SetupAsync(netnsKey, Array.Empty(), timeoutSeconds: 20, CancellationToken.None); - - var plan = sealedToBroker ? "sealed" : "allowlist"; - var table = sealedToBroker ? $"inet {FilteredEgressPlan.NamespaceFor(netnsKey)}" : $"ip {FilteredEgressPlan.NamespaceFor(netnsKey)}"; - var why = sealedToBroker - ? "a host-destined packet is INPUT, and the sealed input filter must admit exactly this lease's port" - : "a host-destined packet is INPUT, which the allowlist plan's forward filter never sees, so no allowlist entry is needed"; + var prefix = Array.Empty() as IReadOnlyList; try { - setup.SetupOk.ShouldBeTrue($"the {plan} netns must set up cleanly; setup error: {setup.SetupError}"); - setup.HostIp.ShouldNotBeNullOrWhiteSpace("the setup must report its gateway address — it is the only address a process inside the namespace can reach this worker at"); + if (allowlist) + { + var setup = await FilteredEgressNetns.SetupAsync(netnsKey, Array.Empty(), timeoutSeconds: 20, CancellationToken.None); + setup.SetupOk.ShouldBeTrue($"the allowlist netns must set up cleanly; setup error: {setup.SetupError}"); + prefix = setup.ExecPrefix; + } - // Resolve the broker's address through the PRODUCTION substitution the runner performs at launch, so the - // URL the test curls is the one a real child would be handed. - var url = ReachableUrl(brokered!, setup.HostIp!) + "/v1/messages"; + // The PRODUCTION chain around the child — the runner's own composition of namespace, bubblewrap and relay — + // with curl as the CLI, so each call is one relayed child from start to exit. + var spec = new SandboxSpec { Command = "/usr/bin/curl", AllowNetwork = allowlist, EgressAllowlist = allowlist ? ["api.anthropic.com"] : null, ModelBrokerPort = brokered.RebindPort, ModelBrokerSocketPath = brokered.SocketPath, WorkingDirectory = context.WorkingDirectory }; + var url = ReachableUrl(brokered) + "/v1/messages"; - (await CurlInNetnsAsync(setup.ExecPrefix, url, brokered!.RunToken)).ShouldBe("200", - customMessage: $"a run in the {plan} netns must reach its broker at {setup.HostIp} — if this is not 200, check by hand: `ip netns exec {FilteredEgressPlan.NamespaceFor(netnsKey)} curl -v {url}` and `nft list table {table}`. {why}"); + (await RelayedCurlAsync(spec, prefix, bwrap, url, brokered.RunToken)).ShouldBe((0, "200"), + customMessage: $"a {(allowlist ? "allowlist" : "network-off")} run must reach its broker at {url} through the relay and its socket; check `ls -la {context.SocketDirectory}`"); var relayedBeforeRevoke = upstream.Calls; await broker.RevokeAsync(runId, "e2e-revoke", fencedToEpoch: null, CancellationToken.None); - // A revoke withdraws the ADDRESS, not just the routing entry: every lease owns its own listener, and - // closing it is what stops one finished run from holding a port for the life of the worker. So what the - // sealed process observes is a refused CONNECTION, not an HTTP 401 — a strictly stronger withdrawal, and - // the shape this arm pins. (It used to read 401 back when one listener served every run and only the route - // was removed.) - var (exit, status) = await CurlAsync(setup.ExecPrefix, url, brokered.RunToken); - - exit.ShouldBe(CurlCouldNotConnect, - customMessage: $"after a revoke nothing may answer at {url} from inside the namespace — curl must fail to connect (7), and got exit {exit} (status '{status}'). Exit 0 means something is STILL LISTENING on the revoked lease's port; exit 28 means the packet is being dropped rather than rejected — neither plan's filter drops this port, so that is a netns/filter change, not a brokerage one. Check by hand: `ip netns exec {FilteredEgressPlan.NamespaceFor(netnsKey)} curl -v {url}`"); + // A revoke withdraws the ADDRESS, not just the routing entry: the lease's socket and its listener both go. + // What the relayed child observes is a connection the relay resets, never an HTTP answer. + var (exit, status) = await RelayedCurlAsync(spec, prefix, bwrap, url, brokered.RunToken); + exit.ShouldNotBe(0, $"after a revoke nothing may answer the relayed child — curl got status '{status}'"); upstream.Calls.ShouldBe(relayedBeforeRevoke, "and nothing may reach the provider after the withdrawal — that, not which error the sandbox sees, is what decides whether a cancelled run can still spend the tenant's key"); + + output.WriteLine($"{RanMarker} revoke-{(allowlist ? "allowlist" : "network-off")} uid={EffectiveUid()} exitAfterRevoke={exit}"); } - finally { await FilteredEgressNetns.TeardownAsync(netnsKey, CancellationToken.None); } + finally { if (allowlist) await FilteredEgressNetns.TeardownAsync(netnsKey, CancellationToken.None); } } - /// curl's "Failed to connect to host" — what a sealed process gets once a revoked lease's listener is closed and its port stops existing. - private const int CurlCouldNotConnect = 7; - [Fact] - public async Task A_sealed_run_reaches_its_broker_again_after_the_worker_that_minted_it_restarts() + public async Task A_gateway_addressed_run_launched_before_the_relay_is_re_bound_wide_and_reaches_its_broker() { - if (!FilteredEgressNetns.IsSupported) return; // no ip/nft (macOS dev / non-privileged) → the privileged CI job is authoritative + // The in-flight survivor this deploy must not strand: a namespaced run launched by the code before the relay, + // whose child froze a base URL at its namespace's GATEWAY and whose handle recorded no socket. Its re-bind is the + // legacy one: the recorded port on every address, wide first, where this host can build namespaces; and the + // broker's source gate admits the child's 10.x address. The namespace is the allowlist plan's, applied on a 10.x + // lease as the old allocator handed them out, so the arm keeps meaning what it means when the pool moves. A + // network-off survivor was sealed through that veth by an inet table of its own, which only the teardown by name + // still deletes, so the arm stages that table too and ends by tearing the namespace down. + if (!FilteredEgressNetns.IsSupported) return; // the root lane, with ip + nft, is authoritative + + FilteredEgressNetns.CanSeal.ShouldBeTrue("ip and nft are here, but this process could not build a namespace — the survivor this arm stands for could not exist either"); var runId = Guid.NewGuid(); var teamId = Guid.NewGuid(); var netnsKey = Guid.NewGuid().ToString("N"); + var third = RandomNumberGenerator.GetInt32(0, 64) * 4; + var second = RandomNumberGenerator.GetInt32(0, 256); + var lease = new EgressSubnetAllocator.Lease { Cidr = $"10.254.{second}.{third}/30", HostIp = $"10.254.{second}.{third + 1}", NsIp = $"10.254.{second}.{third + 2}" }; try { - BrokeredModelCredential brokered; - FilteredEgressNetns.SetupResult setup; - string url; - - // Worker A mints the address, proves it works from inside the sealed namespace, and then GOES AWAY. The - // namespace is sealed to the port worker A's lease holds — the port the re-bind below must take again. - using (var workerA = LoopbackModelCredentialBroker.ForTest(new AlwaysOkUpstream())) - { - brokered = (await workerA.OpenAsync(LeaseFor(runId, teamId), CancellationToken.None)).ShouldNotBeNull(); - setup = await FilteredEgressNetns.SetupSealedAsync(netnsKey, brokered.RebindPort!.Value, timeoutSeconds: 20, CancellationToken.None); - - setup.SetupOk.ShouldBeTrue($"the sealed netns must set up cleanly; setup error: {setup.SetupError}"); - setup.HostIp.ShouldNotBeNullOrWhiteSpace("the setup must report its gateway address — it is the only address a process inside the namespace can reach this worker at"); + var plan = FilteredEgressPlan.Build(netnsKey, Array.Empty(), lease); + var setup = await FilteredEgressNetns.ApplyAsync(netnsKey, plan, timeoutSeconds: 20, CancellationToken.None); + setup.SetupOk.ShouldBeTrue($"the allowlist-plan netns must set up on {lease.Cidr}; setup error: {setup.SetupError}"); - url = ReachableUrl(brokered, setup.HostIp!) + "/v1/messages"; + // What the survivor's handle recorded: a port, a route and a bearer — and no socket. + var brokered = new BrokeredModelCredential("unused", McpRunTokenMint(), DateTimeOffset.UtcNow) { RebindPort = FreeLoopbackPort(), RebindRoute = McpPathIdMint() }; + var url = $"http://{setup.HostIp}:{brokered.RebindPort}/{brokered.RebindRoute}/v1/messages"; + var sealTable = plan.Namespace; - (await CurlInNetnsAsync(setup.ExecPrefix, url, brokered.RunToken)).ShouldBe("200", "precondition: the sealed run reaches its broker while the worker that minted it holds the address"); - } + (await RunHostAsync(["nft", "-f", "-"], RetiredSealRuleset(sealTable, plan.VethHost, plan.HostIp, brokered.RebindPort!.Value))).ShouldBe(0, "setup: the seal a network-off survivor carries must load on this host"); + (await RunHostAsync(["nft", "list", "table", "inet", sealTable])).ShouldBe(0, "control: the survivor's seal is there before its run ends"); - (await CurlAsync(setup.ExecPrefix, url, brokered.RunToken)).Exit.ShouldNotBe(0, - "precondition: with worker A gone the address answers nothing at all — that is the deploy this test is about"); + (await CurlAsync(setup.ExecPrefix, url, brokered.RunToken)).Exit.ShouldNotBe(0, "precondition: the worker that minted the address is gone, so nothing answers at the gateway"); using var workerB = LoopbackModelCredentialBroker.ForTest(new AlwaysOkUpstream()); - (await workerB.RebindAsync(RebindOf(brokered, runId, teamId, epoch: 2), CancellationToken.None)).ShouldBeTrue( - "worker B must be able to re-open the address the sealed run is still calling"); + (await workerB.RebindAsync(RebindOf(brokered, runId, teamId, epoch: 2) with { ChildInNetworkNamespace = true }, CancellationToken.None)).ShouldBeTrue("the new worker must re-open the survivor's recorded address"); + workerB.ListenerPrefixForTest(runId).ShouldBe($"http://+:{brokered.RebindPort}/", "a re-bind with no socket on a host that builds namespaces takes the wide bind, first"); - // The claim this lane exists for, and one no unit test can make: a re-bind has to take the WIDE address, - // because a sealed child reaches this worker at its namespace GATEWAY and never on loopback. Fall back to - // a loopback-only bind here and curl cannot connect at all — the re-bind reports success onto an address - // nobody calls, which is strictly worse than the honest refusal it replaced. (await CurlInNetnsAsync(setup.ExecPrefix, url, brokered.RunToken)).ShouldBe("200", - customMessage: $"a sealed run must reach its RE-BOUND broker at {setup.HostIp}. If curl cannot connect, the re-bind took loopback instead of the wide bind; check by hand: `ip netns exec {FilteredEgressPlan.NamespaceFor(netnsKey)} curl -v {url}`"); + customMessage: $"the survivor must reach its re-bound broker at its gateway {setup.HostIp}; if curl cannot connect the re-bind took loopback instead of the wide bind — check by hand: `ip netns exec {FilteredEgressPlan.NamespaceFor(netnsKey)} curl -v {url}`"); + + output.WriteLine($"{RanMarker} legacy-gateway-rebind lease={lease.Cidr}"); + + // The run's terminal path: the teardown by name, reconstructed from the run key alone, as a later worker does. + await FilteredEgressNetns.TeardownAsync(netnsKey, CancellationToken.None); + + (await RunHostAsync(["nft", "list", "table", "inet", sealTable])).ShouldNotBe(0, $"the survivor's seal must go with its run, or every sealed run in flight at the deploy leaks an nft table on the worker — check `nft list tables | grep {sealTable}`"); + (await RunHostAsync(["ip", "netns", "pids", sealTable])).ShouldNotBe(0, $"and so must its namespace — check `ip netns list | grep {sealTable}`"); + + output.WriteLine($"{RanMarker} legacy-sealed-teardown table={sealTable}"); } finally { await FilteredEgressNetns.TeardownAsync(netnsKey, CancellationToken.None); } } + /// + /// The inet table the veth seal (#2035) loaded for a network-off run: input from the run's veth only to the broker's + /// port at the gateway, nothing forwarded. A copy of the ruleset that change shipped, kept only to stage what a run + /// sealed before the relay still carries; the plan that built it is gone, so there is no live source for it to drift + /// from, and the teardown under test deletes the table by its name whatever it holds. + /// + private static string RetiredSealRuleset(string table, string vethHost, string hostIp, int brokerPort) => string.Join("\n", new[] + { + $"table inet {table} {{", + " chain input {", + " type filter hook input priority 0;", + $" iifname \"{vethHost}\" ct state established,related accept", + $" iifname \"{vethHost}\" ip daddr {hostIp} tcp dport {brokerPort} accept", + $" iifname \"{vethHost}\" drop", + " }", + " chain forward {", + " type filter hook forward priority 0;", + $" iifname \"{vethHost}\" drop", + " }", + "}", + }) + "\n"; + + /// Run one host command as this worker, feeding when given, and return its exit code. + private static async Task RunHostAsync(IReadOnlyList argv, string? stdin = null) + { + var psi = new ProcessStartInfo { FileName = argv[0], UseShellExecute = false, RedirectStandardInput = true, RedirectStandardOutput = true, RedirectStandardError = true }; + foreach (var argument in argv.Skip(1)) psi.ArgumentList.Add(argument); + + using var process = Process.Start(psi)!; + + await process.StandardInput.WriteAsync(stdin ?? ""); + process.StandardInput.Close(); + await process.StandardOutput.ReadToEndAsync(); + await process.StandardError.ReadToEndAsync(); + await process.WaitForExitAsync(); + + return process.ExitCode; + } + + /// + /// One relayed child under the PRODUCTION chain (): the namespace prefix + /// if any, bubblewrap, the relay, and curl POSTing to . Returns curl's exit code and the HTTP + /// status it saw. + /// + /// Forked from a thread of its own that lives until the chain is done: --die-with-parent is + /// PR_SET_PDEATHSIG, which fires when the forking THREAD exits, and a pool thread can retire mid-call — a + /// SIGKILL'd chain, exit 137, whatever the broker answered. The production runner launches from a thread that + /// outlives the command for the same reason, and so does BrokerRelayE2ETests. + /// + private static Task<(int Exit, string Status)> RelayedCurlAsync(SandboxSpec spec, IReadOnlyList prefix, string bwrap, string url, string token) + { + var withArgs = spec with { Args = ["-s", "-m", "15", "-o", "/dev/null", "-w", "%{http_code}", "-X", "POST", "-H", "content-type: application/json", "-H", $"Authorization: Bearer {token}", "-d", "{}", url] }; + var argv = LocalProcessRunner.ChildCommand(new LocalProcessRunner.CommandIsolationContext(withArgs, null, null, prefix, Array.Empty()), bwrap, prlimit: null); + + argv.ShouldContain(ModelBrokerRelay.Verb, "fixture check: the production chain put the relay in front of curl"); + + var done = new TaskCompletionSource<(int Exit, string Status)>(TaskCreationOptions.RunContinuationsAsynchronously); + var launcher = new Thread(() => + { + try { done.SetResult(RunChainToExit(argv)); } + catch (Exception exception) { done.SetException(exception); } + }) { IsBackground = true, Name = "relayed-curl-launcher" }; + + launcher.Start(); + + return done.Task; + } + + /// Start on the calling thread and wait for it there, bounded (Rule 12.10): curl's own -m 15 ends any call well inside the deadline, so a chain still running past it is the relay waiting on something after its CLI. + private static (int Exit, string Status) RunChainToExit(IReadOnlyList argv) + { + var psi = new ProcessStartInfo { FileName = argv[0], UseShellExecute = false, RedirectStandardOutput = true, RedirectStandardError = true }; + foreach (var argument in argv.Skip(1)) psi.ArgumentList.Add(argument); + + using var process = Process.Start(psi)!; + var stdout = process.StandardOutput.ReadToEndAsync(); + var stderr = process.StandardError.ReadToEndAsync(); + + if (!Task.WhenAll(process.WaitForExitAsync(), stdout, stderr).Wait(TimeSpan.FromSeconds(60))) + { + try { process.Kill(entireProcessTree: true); } catch (InvalidOperationException) { /* already exited */ } + throw new TimeoutException("the relayed curl chain did not finish within 60s although curl gives up at 15s — check `ps -ef | grep -e bwrap -e codespace-mcp` for a relay left waiting"); + } + + return (process.ExitCode, stdout.Result.Trim()); + } + + private static int FreeLoopbackPort() + { + using var probe = new System.Net.Sockets.TcpListener(System.Net.IPAddress.Loopback, 0); + probe.Start(); + + return ((System.Net.IPEndPoint)probe.LocalEndpoint).Port; + } + + private static string McpRunTokenMint() => CodeSpace.Core.Services.Agents.Mcp.McpRunToken.Mint(); + + private static string McpPathIdMint() => CodeSpace.Core.Services.Agents.Mcp.McpRunToken.MintPathId(); + private static ModelCredentialLeaseRequest LeaseFor(Guid runId, Guid teamId) => new() { RunId = runId, TeamId = teamId, Epoch = 1, Upstream = new() { Provider = "Anthropic", ApiKey = "sk-e2e-upstream-key" }, Ttl = TimeSpan.FromMinutes(5) }; @@ -372,11 +466,11 @@ public async Task A_run_token_presented_to_the_provider_directly_is_refused() customMessage: $"the provider must REFUSE the run token outright (got {status})"); } - private static string ReachableUrl(BrokeredModelCredential brokered, string? gatewayIp) + private static string ReachableUrl(BrokeredModelCredential brokered) { var spec = new SandboxSpec { Command = "curl", Environment = new Dictionary { ["URL"] = brokered.BaseUrl } }; - return LocalProcessRunner.ResolveModelBrokerHost(spec, gatewayIp).Environment["URL"]; + return LocalProcessRunner.ResolveModelBrokerHost(spec).Environment["URL"]; } private static async Task CurlInNetnsAsync(IReadOnlyList execPrefix, string url, string token) diff --git a/backend/tests/CodeSpace.SandboxTests/NonRootWorker.cs b/backend/tests/CodeSpace.SandboxTests/NonRootWorker.cs new file mode 100644 index 000000000..aa5ca13d0 --- /dev/null +++ b/backend/tests/CodeSpace.SandboxTests/NonRootWorker.cs @@ -0,0 +1,40 @@ +using System.Runtime.InteropServices; +using CodeSpace.Core.Services.Agents.Sandbox.Isolation; +using Shouldly; + +namespace CodeSpace.SandboxTests; + +/// +/// The shipped worker's posture, asserted rather than assumed: a non-root uid with no capabilities, on a host where +/// bubblewrap confines through unprivileged user namespaces and this process may NOT build a network namespace of its +/// own (ip netns add needs CAP_SYS_ADMIN). Every test in the SandboxNonRoot lane calls +/// first, so the lane cannot pass by running as root, or on a host that could fall back to a +/// veth namespace, and the trait selects the lane without any environment flag. +/// +internal static class NonRootWorker +{ + /// The trait value the non-root lane filters on (--filter Category=SandboxNonRoot). + public const string Category = "SandboxNonRoot"; + + /// + /// True on Linux once the posture is proved; false on any other OS, where there is no such lane (Rule 12.1). On + /// Linux a missing piece of the posture FAILS the test: a non-root arm that ran as root proves nothing about the + /// posture it is named for. + /// + public static bool Require() + { + if (!OperatingSystem.IsLinux()) return false; + + GetEuid().ShouldNotBe(0u, "this is the non-root lane: run it as the worker's uid with no capabilities (setpriv --reuid 1654 --regid 1654 --clear-groups --inh-caps=-all --bounding-set=-all --no-new-privs); as root it proves nothing about the shipped posture"); + BubblewrapSandbox.Available.ShouldNotBeNull($"bubblewrap must confine as this uid ({BubblewrapSandbox.UnavailableReason}); check `sysctl kernel.apparmor_restrict_unprivileged_userns` and `bwrap --unshare-user --unshare-net true` as this user"); + FilteredEgressNetns.CanSeal.ShouldBeFalse("this uid must NOT be able to build a network namespace — that is the posture the relay exists for; a worker that can was given CAP_SYS_ADMIN"); + + return true; + } + + /// The effective uid this test process runs as, for the lane's markers. + public static uint EffectiveUid() => GetEuid(); + + [DllImport("libc", EntryPoint = "geteuid")] + private static extern uint GetEuid(); +} diff --git a/backend/tests/CodeSpace.SandboxTests/NonRootWorkerE2ETests.cs b/backend/tests/CodeSpace.SandboxTests/NonRootWorkerE2ETests.cs new file mode 100644 index 000000000..c34fd833a --- /dev/null +++ b/backend/tests/CodeSpace.SandboxTests/NonRootWorkerE2ETests.cs @@ -0,0 +1,100 @@ +using CodeSpace.Core.Services.Agents; +using CodeSpace.Core.Services.Agents.Harnesses.Claude; +using CodeSpace.Core.Services.Agents.Harnesses.Codex; +using CodeSpace.Core.Services.Agents.Sandbox.Runners; +using CodeSpace.Messages.Agents; +using Shouldly; +using Xunit.Abstractions; + +namespace CodeSpace.SandboxTests; + +/// +/// 🟢 Sandbox isolation E2E (high fidelity, Rule 12), the NON-ROOT lane: the shipped worker's posture — uid 1654, no +/// capabilities, no_new_privs — on a host where bubblewrap confines through unprivileged user namespaces but this +/// process may not build a network namespace of its own. Until the relay, a confining host that could not build one +/// refused every network-off brokered run before it spent; here the REAL runner admits it, and the real chain carries +/// it to its broker through the relay and the lease's socket. Each arm asserts the posture first +/// (), then runs the SAME arm the root lane runs, so both lanes pin one behaviour. +/// +/// Selected by its trait alone (--filter Category=SandboxNonRoot), never by the root lane's +/// Category=Sandbox. Every arm that ran prints its class's marker with non-root and its uid, which the +/// lane requires. +/// +[Trait("Category", NonRootWorker.Category)] +public sealed class NonRootWorkerE2ETests(ITestOutputHelper output) +{ + /// Printed by the admission arm when it actually ran. + public const string RanMarker = "[non-root-e2e] ran"; + + private const string Lane = "non-root"; + + [Fact] + public void A_worker_that_cannot_build_a_namespace_admits_a_brokered_network_off_run() + { + if (!NonRootWorker.Require()) return; + + var runId = Guid.NewGuid(); + var spec = new SandboxSpec { Command = "agent", ModelBrokerPort = 43121, ModelBrokerSocketPath = AgentRunExecutor.ModelBrokerSocketPathFor(new AgentPermissions { Network = AgentNetworkAccess.Off }, runId) }; + + Should.NotThrow(() => new LocalProcessRunner().EnsureEgressAdmissible(spec), "a network-off brokered run whose lease has its socket and whose helper is installed reaches its broker through the relay, which needs no namespace of the worker's"); + + output.WriteLine($"{RanMarker} admission uid={NonRootWorker.EffectiveUid()}"); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task A_network_off_brokered_run_reaches_its_broker_and_nothing_else(bool durable) + { + if (!NonRootWorker.Require()) return; + + using var arms = new SealedEgressE2ETests(output); + await arms.ReachesItsBrokerAndNothingElseAsync(durable, Lane); + } + + [Fact] + public async Task A_relayed_run_has_no_ipv6_path_to_the_worker_either() + { + if (!NonRootWorker.Require()) return; + + using var arms = new SealedEgressE2ETests(output); + await arms.HasNoIpv6PathToTheWorkerAsync(Lane); + } + + [Fact] + public async Task A_brokered_child_the_worker_cannot_relay_is_refused_and_would_have_reached_nothing() + { + if (!NonRootWorker.Require()) return; + + using var arms = new SealedEgressE2ETests(output); + await arms.IsRefusedAndWouldHaveReachedNothingAsync(Lane); + } + + [Fact] + public async Task The_same_live_agent_reaches_the_next_worker_through_its_socket_after_a_restart() + { + if (!NonRootWorker.Require()) return; + + using var arms = new SealedEgressE2ETests(output); + await arms.ReachesTheNextWorkerAfterARestartAsync(Lane); + } + + [Fact] + public async Task A_severed_child_reaches_its_broker_through_the_lease_socket_and_the_same_child_reaches_the_next_worker() + { + if (!NonRootWorker.Require()) return; + + await new ModelCredentialBrokerNetnsE2ETests(output).SocketChannelAsync(Lane); + } + + [Theory] + [InlineData(ClaudeCodeHarness.HarnessKind)] + [InlineData(CodexHarness.HarnessKind)] + public async Task A_network_off_reviewer_reaches_its_model_through_the_relay(string harnessKind) + { + if (!NonRootWorker.Require()) return; + + using var arms = new ReviewerReadsItsDiffE2ETests(output); + await arms.NetworkOffReviewerAsync(harnessKind, Lane); + } +} diff --git a/backend/tests/CodeSpace.SandboxTests/ReviewerReadsItsDiffE2ETests.cs b/backend/tests/CodeSpace.SandboxTests/ReviewerReadsItsDiffE2ETests.cs index dcba2c27c..2f71a68a3 100644 --- a/backend/tests/CodeSpace.SandboxTests/ReviewerReadsItsDiffE2ETests.cs +++ b/backend/tests/CodeSpace.SandboxTests/ReviewerReadsItsDiffE2ETests.cs @@ -28,9 +28,10 @@ namespace CodeSpace.SandboxTests; /// RUNS the command, not whether a model would choose to. /// /// Every arm runs its tier's production posture, network off included. Under bubblewrap a network-off run whose -/// model is brokered runs in a namespace sealed to that broker (AgentRunExecutor.ApplySealedEgress), so it -/// reaches its model and nothing else; before that seal it was severed from the broker too and reached no model at -/// all, which is why these arms once had to turn the network on to ask anything. +/// model is brokered is severed, and reaches its model through the codespace-mcp relay in front of its CLI and +/// the lease's socket bound read-only into the sandbox (AgentRunExecutor.ApplyModelBrokerChannel), so it +/// reaches its model and nothing else; before that it was severed from the broker too and reached no model at all, +/// which is why these arms once had to turn the network on to ask anything. /// /// What it found on Linux, pinned so that a fix has to flip it deliberately: both CLIs read the diff under our /// bubblewrap, over a workspace the kernel mounts read-only for a Confined run. Codex does so only because the runner @@ -101,12 +102,12 @@ private async Task ReadsTheDiffAsync(string harnessKind, AgentAutonomyLevel tier var repo = NewReviewRepository(); var upstream = new ScriptedModelUpstream([$"git diff {repo.Base} {repo.Head}"], $"REVIEW-DONE-{repo.Nonce}"); using var broker = LoopbackModelCredentialBroker.ForTest(upstream); - var brokered = await OpenLeaseAsync(broker); + var brokered = await OpenLeaseAsync(broker, AgentAutonomyPolicy.Derive(tier)); var production = AgentAutonomyPolicy.Derive(tier); var task = ReviewTask(harnessKind, repo, production, Brokered(harness, brokered)); - var run = await RunAsync(harness, task, brokered.RebindPort); + var run = await RunAsync(harness, task, brokered); run.Spec.ReadOnlyWorkingDirectory.ShouldBe(production.WriteScope == AgentWriteScope.ReadOnly, $"fixture check: a {tier} reviewer must be launched over the workspace mount its write scope gives it, or this arm does not show git reads surviving it"); @@ -150,11 +151,11 @@ public async Task A_confined_reviewer_cannot_write_its_workspace(string harnessK var repo = NewReviewRepository(); var upstream = new ScriptedModelUpstream([$"echo TAMPER-{repo.Nonce} > app.txt"], $"REVIEW-DONE-{repo.Nonce}"); using var broker = LoopbackModelCredentialBroker.ForTest(upstream); - var brokered = await OpenLeaseAsync(broker); + var brokered = await OpenLeaseAsync(broker, AgentAutonomyPolicy.Derive(AgentAutonomyLevel.Confined)); var task = ReviewTask(harnessKind, repo, AgentAutonomyPolicy.Derive(AgentAutonomyLevel.Confined), Brokered(harness, brokered)); - var run = await RunAsync(harness, task, brokered.RebindPort); + var run = await RunAsync(harness, task, brokered); var fedBack = ToolOutputs(upstream.Requests); @@ -196,13 +197,13 @@ public async Task A_standard_codex_writes_its_workspace_under_our_confinement_bu var hookProbe = Path.Combine(repo.Directory, ".git", "hooks", $"cs-probe-{repo.Nonce}"); var upstream = new ScriptedModelUpstream([$"printf CHANGED-{repo.Nonce} > app.txt", $"printf x > {systemProbe}", $"touch {hookProbe}"], $"WORK-DONE-{repo.Nonce}"); using var broker = LoopbackModelCredentialBroker.ForTest(upstream); - var brokered = await OpenLeaseAsync(broker); + var brokered = await OpenLeaseAsync(broker, AgentAutonomyPolicy.Derive(AgentAutonomyLevel.Standard)); var task = ReviewTask(harnessKind, repo, AgentAutonomyPolicy.Derive(AgentAutonomyLevel.Standard), Brokered(harness, brokered)) with { Goal = "Change app.txt." }; ReviewRun run; - try { run = await RunAsync(harness, task, brokered.RebindPort); } + try { run = await RunAsync(harness, task, brokered); } finally { if (File.Exists(systemProbe)) File.Delete(systemProbe); } // a failed refusal must not leave the probe behind for the next run run.Result.Status.ShouldBe(SandboxStatus.Success, customMessage: $"the Standard Codex run did not finish cleanly (exit {run.Result.ExitCode}); stderr: {Tail(run.Result.Stderr)}; last tool output: {Tail(ToolOutputs(upstream.Requests), 600)}"); @@ -217,33 +218,36 @@ public async Task A_standard_codex_writes_its_workspace_under_our_confinement_bu [Theory] [InlineData(ClaudeCodeHarness.HarnessKind)] [InlineData(CodexHarness.HarnessKind)] - public async Task A_network_off_reviewer_reaches_its_model_through_the_sealed_namespace(string harnessKind) + public Task A_network_off_reviewer_reaches_its_model_through_the_relay(string harnessKind) => NetworkOffReviewerAsync(harnessKind, lane: "root"); + + /// + /// The network-off arm, for either lane. The durable launch every agent run takes, network off, under confinement: + /// the run must be launched severed, with no namespace of the worker's (so the shipped non-root worker can launch + /// it too), reach its model through the relay in front of its CLI and the lease's socket, read the diff, and be + /// recorded as sealed to its broker. Before the seal this arm pinned the opposite — a Confined reviewer reached no model. + /// + internal async Task NetworkOffReviewerAsync(string harnessKind, string lane) { - // The durable launch every agent run takes, network off, under confinement: the run must be launched inside a - // namespace sealed to its broker (recorded on its handle), reach its model through it, read the diff, and leave - // no namespace behind. Before the seal this arm pinned the opposite — a Confined reviewer reached no model. var harness = HarnessFor(harnessKind); if (!Armed(harnessKind) || OperatingSystem.IsWindows()) return; if (BubblewrapSandbox.Available is null) { - // Only confinement seals the network: an unconfined host would let this run reach the broker and say nothing. + // Only confinement severs the network: an unconfined host would let this run reach the broker and say nothing. BubblewrapSandbox.IsRequired.ShouldBeFalse("Sandbox:RequireConfinement is set but this host cannot sandbox (bwrap/userns) — the E2E cannot prove what confinement does here"); return; } - FilteredEgressNetns.CanSeal.ShouldBeTrue("this confining host could not build a throwaway namespace, so every network-off brokered run on it is severed from its model"); - await RequirePinnedBinaryAsync(harness, harnessKind); var repo = NewReviewRepository(); var upstream = new ScriptedModelUpstream([$"git diff {repo.Base} {repo.Head}"], $"REVIEW-DONE-{repo.Nonce}"); using var broker = LoopbackModelCredentialBroker.ForTest(upstream); - var brokered = await OpenLeaseAsync(broker); + var brokered = await OpenLeaseAsync(broker, AgentAutonomyPolicy.Derive(AgentAutonomyLevel.Confined)); var task = ReviewTask(harnessKind, repo, AgentAutonomyPolicy.Derive(AgentAutonomyLevel.Confined), Brokered(harness, brokered)); - var spec = ProductionSpec(harness, task, brokered.RebindPort); + var spec = ProductionSpec(harness, task, brokered); var key = Guid.NewGuid().ToString("N"); var runner = new LocalProcessRunner(); var lines = new List(); @@ -255,13 +259,14 @@ public async Task A_network_off_reviewer_reaches_its_model_through_the_sealed_na using var budget = new CancellationTokenSource(TimeSpan.FromSeconds((task.TimeoutSeconds ?? 300) + 60)); var result = await runner.AttachAsync(handle, (frame, _) => { lines.Add(frame.Text); return Task.CompletedTask; }, budget.Token); - handle.EgressNetnsKey.ShouldBe(key, "a network-off brokered run must be launched inside a sealed namespace keyed by the run"); + spec.ModelBrokerSocketPath.ShouldBe(brokered.SocketPath, "fixture check: the spec carries the socket its lease bound, as the executor's own hardening stamps it"); + handle.EgressNetnsKey.ShouldBeNull("a network-off brokered run gets no namespace of the worker's: the relay reaches its broker without one"); handle.Confinement.ShouldNotBeNull().EgressSealedToBroker.ShouldBeTrue("the launch must record that the run was sealed to its broker"); - result.Status.ShouldBe(SandboxStatus.Success, customMessage: $"the {harnessKind} reviewer did not finish cleanly through the sealed namespace (exit {result.ExitCode}); stderr: {Tail(result.Stderr)}; requests the model saw: {Describe(upstream.Requests)}"); - upstream.Requests.ShouldContain(r => r.Body.Contains($"MARKER-NEW-{repo.Nonce}", StringComparison.Ordinal), $"the diff must reach the model through the sealed namespace; last tool output: {Tail(ToolOutputs(upstream.Requests), 600)}"); + result.Status.ShouldBe(SandboxStatus.Success, customMessage: $"the {harnessKind} reviewer did not finish cleanly through the relay (exit {result.ExitCode}); stderr: {Tail(result.Stderr)}; requests the model saw: {Describe(upstream.Requests)}"); + upstream.Requests.ShouldContain(r => r.Body.Contains($"MARKER-NEW-{repo.Nonce}", StringComparison.Ordinal), $"the diff must reach the model through the relay; last tool output: {Tail(ToolOutputs(upstream.Requests), 600)}"); (await GitAsync(repo.Directory, "status --porcelain")).ShouldBeEmpty("a Confined reviewer leaves the workspace exactly as it found it"); - output.WriteLine($"{RanMarker} network-off-sealed {harnessKind} seconds={clock.Elapsed.TotalSeconds:F1}"); + output.WriteLine($"{RanMarker} {(lane == "root" ? "" : lane + " ")}network-off-relayed {harnessKind} uid={NonRootWorker.EffectiveUid()} seconds={clock.Elapsed.TotalSeconds:F1}"); } public void Dispose() @@ -302,14 +307,18 @@ private async Task RequirePinnedBinaryAsync(IAgentHarness harness, string harnes version.ShouldContain(pinned, customMessage: $"'{command} --version' reported '{version.Trim()}', but production pins {pinned} (backend/Dockerfile.worker) — the answer this test gives is only about the pinned binary"); } - private static async Task OpenLeaseAsync(LoopbackModelCredentialBroker broker) + /// The lease the executor opens for a run with these permissions — with the socket it asks for exactly where the run's network will be its own (). + private async Task OpenLeaseAsync(LoopbackModelCredentialBroker broker, AgentPermissions permissions) { + var runId = Guid.NewGuid(); var lease = new ModelCredentialLeaseRequest { - RunId = Guid.NewGuid(), TeamId = Guid.NewGuid(), Epoch = 1, Ttl = TimeSpan.FromMinutes(10), + RunId = runId, TeamId = Guid.NewGuid(), Epoch = 1, Ttl = TimeSpan.FromMinutes(10), SocketPath = AgentRunExecutor.ModelBrokerSocketPathFor(permissions, runId), Upstream = new ResolvedModelCredential { Provider = "Custom", ApiKey = "sk-review-e2e-upstream", BaseUrl = "https://scripted-model.invalid" }, }; + if (lease.SocketPath is { } socketPath) _directories.Add(Path.GetDirectoryName(socketPath)!); + return (await broker.OpenAsync(lease, CancellationToken.None)).ShouldNotBeNull("the broker must be able to listen on this host — a brokered run has no other route to its model"); } @@ -325,13 +334,13 @@ private static async Task OpenLeaseAsync(LoopbackModelC Environment = new Dictionary(brokeredEnvironment) { ["HOME"] = NewDirectory("review-home") }, }; - /// The spec the executor would hand the runner: the harness invocation sealed to the run's broker lease when its network is off, and with its write scope applied, so a read-only run's workspace is mounted read-only wherever the host confines. - private static SandboxSpec ProductionSpec(IAgentHarness harness, AgentTask task, int? brokerPort) => - AgentRunExecutor.ApplyWriteScope(AgentRunExecutor.ApplySealedEgress(harness.BuildInvocation(task), task.Permissions, brokerPort), task.Permissions); + /// The spec the executor would hand the runner: the harness invocation with its broker channel stamped when its network is off, and with its write scope applied, so a read-only run's workspace is mounted read-only wherever the host confines. + private static SandboxSpec ProductionSpec(IAgentHarness harness, AgentTask task, BrokeredModelCredential brokered) => + AgentRunExecutor.ApplyWriteScope(AgentRunExecutor.ApplyModelBrokerChannel(harness.BuildInvocation(task), brokered), task.Permissions); - private static async Task RunAsync(IAgentHarness harness, AgentTask task, int? brokerPort) + private static async Task RunAsync(IAgentHarness harness, AgentTask task, BrokeredModelCredential brokered) { - var spec = ProductionSpec(harness, task, brokerPort); + var spec = ProductionSpec(harness, task, brokered); var lines = new List(); using var budget = new CancellationTokenSource(TimeSpan.FromSeconds((task.TimeoutSeconds ?? 300) + 60)); diff --git a/backend/tests/CodeSpace.SandboxTests/SealedEgressE2ETests.cs b/backend/tests/CodeSpace.SandboxTests/SealedEgressE2ETests.cs index 5723eb75c..77cb1eca3 100644 --- a/backend/tests/CodeSpace.SandboxTests/SealedEgressE2ETests.cs +++ b/backend/tests/CodeSpace.SandboxTests/SealedEgressE2ETests.cs @@ -1,11 +1,12 @@ using System.Diagnostics; using System.Globalization; using System.Net; +using System.Net.NetworkInformation; using System.Net.Sockets; using System.Security.Cryptography; using System.Text.Json; +using CodeSpace.Core.Services.Agents; using CodeSpace.Core.Services.Agents.Credentials.Broker; -using CodeSpace.Core.Services.Agents.Sandbox; using CodeSpace.Core.Services.Agents.Sandbox.Exceptions; using CodeSpace.Core.Services.Agents.Sandbox.Isolation; using CodeSpace.Core.Services.Agents.Sandbox.Runners; @@ -17,16 +18,17 @@ namespace CodeSpace.SandboxTests; /// /// 🟢 Sandbox isolation E2E (high fidelity, Rule 12): a network-off run whose model is brokered, launched by the REAL -/// under bubblewrap, runs in a namespace SEALED to its broker — and a probe from inside -/// it, through the real chain (ip netns exec → prlimit → bwrap → python3), observes exactly one open door: the -/// real broker answers, while the internet, DNS over TCP and UDP, and another listener on the worker's own gateway -/// address all stay shut. The allowlist plan with no IPs would fail three of those (it accepts DNS anywhere, NATs out, -/// and has no input filter), and plain severing fails the first — which is why a network-off brokered run on a -/// confining host reached no model before this. +/// under bubblewrap, reaches its broker and nothing else — and a probe from inside it, +/// through the real chain (prlimit → bwrap → codespace-mcp relay → python3), observes exactly one open door: the +/// real broker answers, through the relay on the sandbox's own loopback and the lease's socket bound read-only into it +/// (the child can neither delete that socket nor plant a file beside it), while the internet, DNS over TCP and UDP, the +/// worker's own address and its loopback listeners all stay shut. No +/// namespace of the worker's is built, so the same arms run as root and, from , as the +/// shipped non-root worker, which may not build one. /// -/// Needs bwrap + ip + nft + the privilege to build a namespace, so it runs for real ONLY in the privileged -/// sandbox-isolation job; elsewhere it returns. Every arm that ran prints , which the lane -/// requires, so a silent return can never pass for coverage. +/// Needs bubblewrap, so it runs for real ONLY in the sandbox-isolation job; elsewhere it returns. The two arms +/// that stage host routing rules need root and run in the root lane alone. Every arm that ran prints +/// , which the lane requires, so a silent return can never pass for coverage. /// [Trait("Category", "Sandbox")] public sealed class SealedEgressE2ETests(ITestOutputHelper output) : IDisposable @@ -34,223 +36,275 @@ public sealed class SealedEgressE2ETests(ITestOutputHelper output) : IDisposable /// Printed by every arm that actually ran; the sandbox lane requires one per arm in the test output. public const string RanMarker = "[sealed-egress-e2e] ran"; - private readonly List _spoolDirs = []; + private static readonly TimeSpan Deadline = TimeSpan.FromSeconds(60); + + private readonly List _dirs = []; [Theory] [InlineData(true)] [InlineData(false)] - public async Task A_network_off_brokered_run_reaches_its_broker_and_nothing_else(bool durable) + public Task A_network_off_brokered_run_reaches_its_broker_and_nothing_else(bool durable) => ReachesItsBrokerAndNothingElseAsync(durable, lane: "root"); + + [Fact] + public Task A_relayed_run_has_no_ipv6_path_to_the_worker_either() => HasNoIpv6PathToTheWorkerAsync(lane: "root"); + + [Fact] + public Task The_same_live_agent_reaches_the_next_worker_through_its_socket_after_a_restart() => ReachesTheNextWorkerAfterARestartAsync(lane: "root"); + + [Fact] + public Task A_brokered_child_the_worker_cannot_relay_is_refused_and_would_have_reached_nothing() => IsRefusedAndWouldHaveReachedNothingAsync(lane: "root"); + + [Fact] + public async Task A_host_policy_rule_that_discards_replies_from_the_broker_port_cannot_reach_a_relayed_run() { - if (!Seals()) return; + // The route check a namespace sealed through a veth once needed — a policy rule keyed on the protocol and the + // broker's port discarded its replies after a clean setup — has nothing left to guard: a relayed run's broker + // traffic never leaves loopback, which the kernel's local table answers before any rule is consulted. The rule + // is staged for the lease's own port only, and the kernel's own lookup shows it would discard a reply to any + // peer that is not local. + if (!Confines() || !OperatingSystem.IsLinux() || NonRootWorker.EffectiveUid() != 0) return; + + using var run = await RelayedRunAsync(); + var table = RandomNumberGenerator.GetInt32(10_000, 1_000_000).ToString(CultureInfo.InvariantCulture); + var port = run.Brokered.RebindPort!.Value.ToString(CultureInfo.InvariantCulture); + string[] rule = ["pref", "100", "ipproto", "6", "sport", port, "lookup", table]; + + string[] replyToAPeer = ["ip", "route", "get", "10.9.9.9", "ipproto", "6", "sport", port]; + + try + { + (await RunHostExitAsync(replyToAPeer)).ShouldBe(0, "control: before the rule, this host routes a reply from the broker's port to a peer that is not local"); + (await RunHostExitAsync(["ip", "route", "add", "unreachable", "default", "table", table])).ShouldBe(0, "setup: the null route must be installable in its own table"); + (await RunHostExitAsync(["ip", "rule", "add", .. rule])).ShouldBe(0, "setup: the rule that consults it before main must be installable"); + (await RunHostExitAsync(replyToAPeer)).ShouldNotBe(0, "control: with the rule, that reply is discarded — which is what it did to a veth-sealed run's broker replies"); - using var broker = LoopbackModelCredentialBroker.ForTest(new AlwaysOkUpstream()); - var brokered = (await broker.OpenAsync(Lease(), CancellationToken.None)).ShouldNotBeNull("the broker must be able to listen on a host that seals — a sealed run has no other route to a model"); + var (result, probe) = await RunAsync(run.Spec); + result.Status.ShouldBe(SandboxStatus.Success, $"the probe must run to its end; stderr: {result.Stderr}"); + probe["broker"].ShouldBe("200", $"the relayed call is answered: loopback is local, and no policy rule is consulted before it; probe: {Describe(probe)}"); + + output.WriteLine($"{RanMarker} relay-policy-route broker={probe["broker"]}"); + } + finally + { + await RunHostExitAsync(["ip", "rule", "del", .. rule]); + await RunHostExitAsync(["ip", "route", "flush", "table", table]); + } + } + + /// The main arm, for either lane: see the class remarks. + internal async Task ReachesItsBrokerAndNothingElseAsync(bool durable, string lane) + { + if (!Confines()) return; + + using var run = await RelayedRunAsync(); using var otherListener = new TcpListener(IPAddress.Any, 0); otherListener.Start(); - var spec = new SandboxSpec + var spec = run.Spec with { - Command = "/usr/bin/python3", - Args = ["-c", ProbeScript], - AllowNetwork = false, - ModelBrokerPort = brokered.RebindPort, - // An egress proxy the worker (or the task) carries, as a proxied deployment would: the sealed namespace - // cannot reach it, so the launch must drop it — which the probe reports directly, since the broker's NO_PROXY - // exemption would let urllib past it even if the drop were gone. - Environment = new Dictionary { ["BROKER_URL"] = brokered.BaseUrl, ["RUN_TOKEN"] = brokered.RunToken, ["OTHER_PORT"] = ((IPEndPoint)otherListener.LocalEndpoint).Port.ToString(), ["HTTP_PROXY"] = "http://10.255.255.1:3128", ["http_proxy"] = "http://10.255.255.1:3128" }, - TimeoutSeconds = 60, + Environment = new Dictionary(run.Spec.Environment) + { + ["OTHER_PORT"] = ((IPEndPoint)otherListener.LocalEndpoint).Port.ToString(CultureInfo.InvariantCulture), + ["WORKER_IP"] = WorkerIpv4(), + ["SOCK_PATH"] = run.SocketPath, + // An egress proxy the worker (or the task) carries, as a proxied deployment would: a network-off child + // cannot reach it, so the launch must drop it — which the probe reports directly, since the broker's + // NO_PROXY exemption would let urllib past it even if the drop were gone. + ["HTTP_PROXY"] = "http://10.255.255.1:3128", ["http_proxy"] = "http://10.255.255.1:3128", + }, }; var (result, probe) = durable ? await RunDurableAsync(spec) : await RunAsync(spec); - result.Status.ShouldBe(SandboxStatus.Success, $"the probe itself must run to its end inside the sealed namespace; stderr: {result.Stderr}"); - probe["broker"].ShouldBe("200", $"the run's own broker is the one destination a sealed run must reach — directly, not through the proxy it was handed; probe: {Describe(probe)}"); - probe["proxies"].ShouldBe("none", $"a sealed launch drops the proxy variables outright — the broker answering is not enough, since the NO_PROXY exemption alone lets urllib past a proxy a stricter reader would still use; probe: {Describe(probe)}"); - probe["internet"].ShouldNotBe("open", $"a sealed run must not reach the internet; probe: {Describe(probe)}"); + result.Status.ShouldBe(SandboxStatus.Success, $"the probe itself must run to its end inside the sandbox; stderr: {result.Stderr}"); + probe["broker"].ShouldBe("200", $"the run's own broker is the one destination a network-off run must reach — through the relay and its socket, not through the proxy it was handed. Check the socket with `ls -la {Path.GetDirectoryName(run.SocketPath)}`; probe: {Describe(probe)}"); + probe["proxies"].ShouldBe("none", $"a sealed launch drops the proxy variables outright; probe: {Describe(probe)}"); + probe["internet"].ShouldNotBe("open", $"a network-off run must not reach the internet; probe: {Describe(probe)}"); probe["dns_tcp"].ShouldNotBe("open", $"no DNS over TCP — a resolver is a tunnel; probe: {Describe(probe)}"); probe["dns_udp"].ShouldNotBe("answered", $"no DNS over UDP either; probe: {Describe(probe)}"); - probe["gateway_other"].ShouldNotBe("open", $"another listener on the worker's own gateway address (its API, another run's broker) must stay shut; probe: {Describe(probe)}"); + probe["worker_eth0"].ShouldNotBe("open", $"the worker's own address (its API, every other run's lease) must stay shut; probe: {Describe(probe)}"); + probe["host_loopback"].ShouldNotBe("open", $"and so must the worker's loopback listeners: the child's 127.0.0.1 is its own, where only the relay listens; probe: {Describe(probe)}"); + probe["links"].Split(',').ShouldNotContain(link => link.StartsWith("eth", StringComparison.Ordinal) || link.StartsWith("cs", StringComparison.Ordinal) || link.StartsWith("veth", StringComparison.Ordinal), $"no interface but loopback and the kernel's own tunnels: {probe["links"]}"); + AssertSocketDirectoryIsReadOnly(probe["sock_unlink"], probe["sock_plant"], run.SocketPath); - output.WriteLine($"{RanMarker} {(durable ? "durable" : "non-durable")} {Describe(probe)}"); + await run.Broker.RevokeAsync(run.RunId, "e2e-revoke", fencedToEpoch: null, CancellationToken.None); + + File.Exists(run.SocketPath).ShouldBeFalse("a revoked lease's socket goes with it"); + Directory.Exists(Path.GetDirectoryName(run.SocketPath)).ShouldBeTrue("its directory stays, since a running sandbox's bind pins it"); + + output.WriteLine($"{RanMarker} {(lane == "root" ? "" : lane + " ")}{(durable ? "durable" : "non-durable")} uid={NonRootWorker.EffectiveUid()} {Describe(probe)}"); } - [Fact] - public async Task A_sealed_namespace_drops_the_gateway_over_ipv6_link_local_too() + /// The IPv6 arm, for either lane: the relayed child has no v6 path to the worker — not over loopback, which is its own, nor to any address the worker holds, which it has no route to. + internal async Task HasNoIpv6PathToTheWorkerAsync(string lane) { - if (!Seals()) return; + if (!Confines()) return; - // Only the host knows its veth's link-local address, so this arm drives the namespace directly rather than - // through a launch. A v4-only table would let this through; the sealed table is inet. Two things keep the arm - // honest: it waits out duplicate-address detection (a tentative address refuses everything, table or not), and - // it then deletes the table and connects again — the probe must get through without it, or its refusal proved - // nothing about the table. - var key = Guid.NewGuid().ToString("N"); - var setup = await FilteredEgressNetns.SetupSealedAsync(key, brokerPort: 9, timeoutSeconds: 20, CancellationToken.None); + using var listener = new TcpListener(IPAddress.IPv6Any, 0); + listener.Server.DualMode = false; - try + try { listener.Start(); } + catch (SocketException) { - setup.SetupOk.ShouldBeTrue($"the sealed namespace must set up on this host: {setup.SetupError}"); - - // The veth names are the run id's alone, so any lease names them the way the setup above did. - var names = FilteredEgressPlan.BuildSealed(key, 9, new EgressSubnetAllocator.Lease { Cidr = "0.0.0.0/30", HostIp = "0.0.0.1", NsIp = "0.0.0.2" }); - - if (await SettledLinkLocalAsync(["ip", "-6", "-o", "addr", "show", "dev", names.VethHost, "scope", "link"]) is not { } linkLocal) - { - output.WriteLine($"[sealed-egress-e2e] skipped ipv6 (IPv6 is disabled on this host, so there is no v6 path to close)"); - return; - } + output.WriteLine($"[sealed-egress-e2e] skipped relay-ipv6 (IPv6 is disabled on this host, so there is no v6 path to close)"); + return; + } - (await SettledLinkLocalAsync(setup.ExecPrefix.Concat(["ip", "-6", "-o", "addr", "show", "dev", names.VethNs, "scope", "link"]).ToList())).ShouldNotBeNull("the namespace side's link-local must settle too, or it cannot send"); + var port = ((IPEndPoint)listener.LocalEndpoint).Port; + var addresses = WorkerIpv6Addresses(); - using var listener = new TcpListener(IPAddress.IPv6Any, 0); - listener.Start(); + (await ConnectsAsync(IPAddress.IPv6Loopback, port)).ShouldBeTrue("control: the worker's own v6 listener answers on [::1], or a refusal inside proves nothing about the sandbox"); - var connect = $"import socket\ntry:\n s=socket.create_connection(('{linkLocal}%{names.VethNs}', {((IPEndPoint)listener.LocalEndpoint).Port}), timeout=3); s.close(); print('open')\nexcept OSError as e:\n print(type(e).__name__)"; - var probe = setup.ExecPrefix.Concat(["/usr/bin/python3", "-c", connect]).ToList(); + using var run = await RelayedRunAsync(); + var spec = run.Spec with + { + Args = ["-c", Ipv6ProbeScript], + Environment = new Dictionary(run.Spec.Environment) { ["V6_PORT"] = port.ToString(CultureInfo.InvariantCulture), ["V6_TARGETS"] = string.Join(',', addresses.Prepend("::1")) }, + }; - var sealedOutcome = (await RunHostAsync(probe)).Trim(); - (await RunHostExitAsync(["nft", "delete", "table", "inet", names.Namespace])).ShouldBe(0, "control setup: the sealed table must be there to delete"); - var controlOutcome = (await RunHostAsync(probe)).Trim(); + var (result, probe) = await RunAsync(spec); - controlOutcome.ShouldBe("open", $"control: with the sealed table gone the same probe must reach the listener at {linkLocal}, or its refusal above proved nothing about the table (got {controlOutcome})"); - sealedOutcome.ShouldNotBe("open", $"the worker's listener must not be reachable over the host veth's IPv6 link-local address {linkLocal} while the sealed table stands"); + result.Status.ShouldBe(SandboxStatus.Success, $"the probe must run to its end; stderr: {result.Stderr}"); + probe["broker"].ShouldBe("200", $"control: the relay itself works in this sandbox; probe: {Describe(probe)}"); + probe.Where(pair => pair.Key.StartsWith("v6 ", StringComparison.Ordinal)).ShouldAllBe(pair => pair.Value != "open", $"no v6 address of the worker is reachable from inside; probe: {Describe(probe)}"); - output.WriteLine($"{RanMarker} ipv6 outcome={sealedOutcome} control={controlOutcome}"); - } - finally { await FilteredEgressNetns.TeardownAsync(key, CancellationToken.None); } + output.WriteLine($"{RanMarker} {(lane == "root" ? "" : lane + " ")}relay-ipv6 targets={addresses.Count + 1} {Describe(probe)}"); } - [Fact] - public async Task A_30_still_held_by_a_run_that_outlived_its_worker_is_not_handed_to_the_next_run() + /// + /// The restart arm, for either lane, on a REAL durable launch: the agent is started once and kept; worker A's broker + /// serves it through the relay, goes away, and worker B re-binds the run's recorded address and re-opens its socket + /// at the same path. The same process — same pid, same relay — reaches worker B, because the socket's directory was + /// never replaced and the relay opens a fresh connection to the socket for every call. + /// + internal async Task ReachesTheNextWorkerAfterARestartAsync(string lane) { - if (!Seals()) return; + if (!Confines()) return; - // A run survives its worker by design, and so do its namespace and veth; the reservation lock does not. A fresh - // worker that trusted the lock alone would hand the survivor's /30 to its next sealed launch, and the kernel - // would split the two runs' broker replies between two veths. Releasing the survivor's reservation without - // tearing its namespace down is exactly what the restart leaves behind. - var survivor = Guid.NewGuid().ToString("N"); - var next = Guid.NewGuid().ToString("N"); - var first = await FilteredEgressNetns.SetupSealedAsync(survivor, brokerPort: 9, timeoutSeconds: 20, CancellationToken.None); + var runId = Guid.NewGuid(); + var teamId = Guid.NewGuid(); + var socketPath = AgentRunExecutor.ModelBrokerSocketPathFor(new AgentPermissions { Network = AgentNetworkAccess.Off }, runId).ShouldNotBeNull("the executor mints a socket for a network-off run on Linux"); + var workspace = NewDirectory("restart-ws"); + var workerA = LoopbackModelCredentialBroker.ForTest(new CountingUpstream()); + var upstreamB = new CountingUpstream(); try { - first.SetupOk.ShouldBeTrue($"the survivor's sealed namespace must set up on this host: {first.SetupError}"); - EgressSubnetAllocator.Host.Release(survivor); - - var second = await FilteredEgressNetns.SetupSealedAsync(next, brokerPort: 9, timeoutSeconds: 20, CancellationToken.None); - - try + var brokered = (await workerA.OpenAsync(Lease(runId, teamId, socketPath), CancellationToken.None)).ShouldNotBeNull(); + var spec = AgentRunExecutor.ApplyModelBrokerChannel(new SandboxSpec { - second.SetupOk.ShouldBeTrue($"the next run's sealed namespace must set up: {second.SetupError}"); - second.HostIp.ShouldNotBe(first.HostIp, "the survivor's /30 is still on its veth; handing it out again routes one run's replies into the other's namespace"); + Command = "/usr/bin/python3", Args = ["-u", "-c", RestartProbeScript], WorkingDirectory = workspace, TimeoutSeconds = 120, + Environment = new Dictionary { ["BROKER_URL"] = brokered.BaseUrl, ["RUN_TOKEN"] = brokered.RunToken, ["SIGNAL_DIR"] = workspace }, + }, brokered); - output.WriteLine($"{RanMarker} restart-reissue survivor={first.HostIp} next={second.HostIp}"); - } - finally { await FilteredEgressNetns.TeardownAsync(next, CancellationToken.None); } - } - finally { await FilteredEgressNetns.TeardownAsync(survivor, CancellationToken.None); } - } + var key = Guid.NewGuid().ToString("N"); + var runner = new LocalProcessRunner(); + var handle = await runner.LaunchAsync(spec, key, CancellationToken.None); + _dirs.Add(handle.SpoolDirectory); - [Theory] - [InlineData(false)] - [InlineData(true)] // a rule that only TCP from the broker port meets — the lookup the broker's replies make, and so the one the check must ask - public async Task A_host_whose_policy_rule_discards_the_run_s_replies_fails_the_setup_and_leaks_nothing(bool keyedOnTheBrokerReply) - { - if (!Seals()) return; - - // The allocator skips what the host's route listing covers, but a null route in a table that a policy rule - // consults before main wins by rule ORDER, not prefix length: every step of the setup succeeds, and then every - // reply from the broker to the namespace is discarded. The /30 here is from TEST-NET-1 (RFC 5737), which the - // allocator never hands out, and the rule covers only that /30, so it cannot reach another run on this host. - var third = RandomNumberGenerator.GetInt32(0, 64) * 4; - var lease = new EgressSubnetAllocator.Lease { Cidr = $"192.0.2.{third}/30", HostIp = $"192.0.2.{third + 1}", NsIp = $"192.0.2.{third + 2}" }; - var table = RandomNumberGenerator.GetInt32(10_000, 1_000_000).ToString(CultureInfo.InvariantCulture); - string[] rule = keyedOnTheBrokerReply ? ["pref", "100", "to", lease.Cidr, "ipproto", "6", "sport", "9", "lookup", table] : ["pref", "100", "to", lease.Cidr, "lookup", table]; - var runId = Guid.NewGuid().ToString("N"); - var plan = FilteredEgressPlan.BuildSealed(runId, brokerPort: 9, lease); + var lines = new System.Collections.Concurrent.ConcurrentQueue(); + var attach = runner.AttachAsync(handle, (frame, _) => { lines.Enqueue(frame.Text); return Task.CompletedTask; }, CancellationToken.None); - // A run of this test killed between its rule add and its cleanup leaves a rule for its /30 in a table this run - // cannot name; left there, it would fail this run's control and blame the check. - for (var stale = 0; stale < 8 && await RunHostExitAsync(["ip", "rule", "del", "pref", "100", "to", lease.Cidr]) == 0; stale++) { } + var first = await CallAsync(workspace, lines, 1); + first.ShouldStartWith("call1=200", customMessage: "worker A answers the relayed agent while it holds the address"); - try - { - var control = await FilteredEgressNetns.ApplyAsync(runId, plan, timeoutSeconds: 20, CancellationToken.None); - control.SetupOk.ShouldBeTrue($"control: the same plan must set up on a host with no such rule, or the check refuses what it should admit: {control.SetupError}"); - await FilteredEgressNetns.TeardownAsync(runId, CancellationToken.None); + workerA.Dispose(); + (await CallAsync(workspace, lines, 2)).ShouldStartWith("call2=error", customMessage: "with worker A gone the agent's call must fail — that is the restart this arm is about"); - (await RunHostExitAsync(["ip", "route", "add", "unreachable", "192.0.2.0/24", "table", table])).ShouldBe(0, "setup: the null route — broader than a /30, which the route listing ignores — must be installable in its own table"); - (await RunHostExitAsync(["ip", "rule", "add", .. rule])).ShouldBe(0, "setup: the rule that consults it before main must be installable"); + using var workerB = LoopbackModelCredentialBroker.ForTest(upstreamB, logger: new TestOutputLogger(output)); + (await workerB.RebindAsync(RebindOf(brokered, runId, teamId, epoch: 2) with { SocketPath = socketPath }, CancellationToken.None)).ShouldBeTrue("worker B must re-open the run's recorded address, socket and all"); - var refused = await FilteredEgressNetns.ApplyAsync(runId, plan, timeoutSeconds: 20, CancellationToken.None); + var third = await CallAsync(workspace, lines, 3); + third.ShouldStartWith("call3=200", customMessage: $"the SAME agent must reach worker B through the socket re-opened at the same path; if not, compare `stat -c %i {Path.GetDirectoryName(socketPath)}` with what its sandbox binds"); + Pid(third).ShouldBe(Pid(first), "it is the same live agent, not a relaunch"); + upstreamB.Calls.ShouldBe(1, "and it was worker B that relayed it"); - refused.SetupOk.ShouldBeFalse("a namespace the host can never answer must fail its setup, not admit a run that spends its timeout on a dead broker"); - refused.SetupError.ShouldNotBeNull().ShouldContain(string.Join(' ', plan.RouteCheckArgv), customMessage: "the refusal names the lookup that found it, so an operator can rerun it"); - (await NetnsExistsAsync(plan.Namespace)).ShouldBeFalse("a setup that failed its route check tears its namespace down"); - (await RunHostExitAsync(["ip", "link", "show", plan.VethHost])).ShouldNotBe(0, "and the host end of its veth, with the address on it"); + await File.WriteAllTextAsync(Path.Combine(workspace, "stop"), ""); + (await attach.WaitAsync(Deadline)).Status.ShouldBe(SandboxStatus.Success, "the agent ends on its own once released"); + handle.EgressNetnsKey.ShouldBeNull("no namespace of the worker's was built for it"); - output.WriteLine($"{RanMarker} policy-route-discard-{(keyedOnTheBrokerReply ? "l4" : "dst")} {refused.SetupError}"); - } - finally - { - await RunHostExitAsync(["ip", "rule", "del", .. rule]); - await RunHostExitAsync(["ip", "route", "flush", "table", table]); - await FilteredEgressNetns.TeardownAsync(runId, CancellationToken.None); + output.WriteLine($"{RanMarker} {(lane == "root" ? "" : lane + " ")}restart uid={NonRootWorker.EffectiveUid()} pid={Pid(first)}"); } + finally { workerA.Dispose(); } } - [Fact] - public async Task A_sealed_setup_that_fails_on_this_host_refuses_the_launch_typed_and_leaks_nothing() + /// + /// The refusal arm, for either lane: the REAL runner, on this confining host, refuses a network-off brokered spec it + /// could not carry to its broker — the lease came back without a socket, or the helper is not there — and the same + /// spec launched anyway, as nothing refused it before, really is cut off: its call to its broker reaches nothing. + /// + internal async Task IsRefusedAndWouldHaveReachedNothingAsync(string lane) { - if (!Seals()) return; + if (!Confines()) return; - // A host that proved it can seal can still fail one run's setup — a name collision, a kernel refusal. The launch - // must then refuse under the same typed wall the executor's pre-spend admission raises, naming the failed step, - // and tear down whatever the partial setup built. Occupying the host veth's name makes the plan's own - // `ip link add` fail exactly as a collision would. - var key = Guid.NewGuid().ToString("N"); - var names = FilteredEgressPlan.BuildSealed(key, 9, new EgressSubnetAllocator.Lease { Cidr = "0.0.0.0/30", HostIp = "0.0.0.1", NsIp = "0.0.0.2" }); + using var broker = LoopbackModelCredentialBroker.ForTest(new CountingUpstream()); + var runId = Guid.NewGuid(); + var brokered = (await broker.OpenAsync(Lease(runId, Guid.NewGuid(), socketPath: null), CancellationToken.None)).ShouldNotBeNull(); + var spec = AgentRunExecutor.ApplyModelBrokerChannel(ProbeSpec(brokered), brokered); - // A veth, not a dummy: the veth driver is what the plan itself needs, so it is loaded wherever sealing works at all. - (await RunHostExitAsync(["ip", "link", "add", names.VethHost, "type", "veth", "peer", "name", "csp-" + names.VethHost[4..]])).ShouldBe(0, $"fixture: could not occupy {names.VethHost}"); + spec.ModelBrokerPort.ShouldNotBeNull("fixture check: a network-off brokered spec, stamped by the executor's own hardening"); - try - { - using var broker = LoopbackModelCredentialBroker.ForTest(new AlwaysOkUpstream()); - var brokered = (await broker.OpenAsync(Lease(), CancellationToken.None)).ShouldNotBeNull(); - var spec = new SandboxSpec { Command = "/bin/true", AllowNetwork = false, ModelBrokerPort = brokered.RebindPort, TimeoutSeconds = 30 }; - _spoolDirs.Add(LocalProcessRunner.SpoolDirectoryFor(key)); + var noSocket = Should.Throw(() => new LocalProcessRunner().EnsureEgressAdmissible(spec)); + noSocket.Cause.ShouldBe(SealedEgressUnavailableException.CauseBrokerSocketUnavailable); + ((CodeSpace.Messages.Failures.IFailure)noSocket).Code.ShouldBe(CodeSpace.Messages.Failures.FailureCodes.SandboxSealedEgressUnavailable); - var thrown = await Should.ThrowAsync(() => new LocalProcessRunner().LaunchAsync(spec, key, CancellationToken.None)); - var refusal = (thrown as SealedEgressUnavailableException ?? thrown.InnerException as SealedEgressUnavailableException).ShouldNotBeNull($"the launch must refuse typed, not as {thrown.GetType().Name}: {thrown.Message}"); + var missing = Path.Combine(NewDirectory("no-helper"), "codespace-mcp"); + Should.Throw(() => LocalProcessRunner.EnsureEgressAdmissible(spec with { ModelBrokerSocketPath = "/spool/k/broker/seg/s" }, BubblewrapSandbox.Available is not null, missing)) + .Cause.ShouldStartWith(SealedEgressUnavailableException.CauseRelayMissing); - ((CodeSpace.Messages.Failures.IFailure)refusal).Code.ShouldBe(CodeSpace.Messages.Failures.FailureCodes.SandboxSealedEgressUnavailable); - refusal.Cause.ShouldContain("ip link add", customMessage: $"the refusal must name the setup step that failed: {refusal.Cause}"); - refusal.Message.ShouldContain("fix what that step names", customMessage: $"a host that can seal is told to fix the failed step, not to grant privileges and wait for a re-probe: {refusal.Message}"); - (await NetnsExistsAsync(names.Namespace)).ShouldBeFalse("a failed setup must tear down the namespace it had already created"); + var (result, probe) = await RunAsync(spec); - output.WriteLine($"{RanMarker} setup-failure cause={refusal.Cause}"); - } - finally - { - await RunHostExitAsync(["ip", "link", "del", names.VethHost]); // best-effort: the failed setup's teardown may already have removed it (and its peer with it) - await FilteredEgressNetns.TeardownAsync(key, CancellationToken.None); - } + result.Status.ShouldBe(SandboxStatus.Success, $"the probe must run to its end; stderr: {result.Stderr}"); + probe["broker"].ShouldStartWith("URLError", customMessage: $"launched without the relay, the child reaches no broker at all — which is what the refusal spares it; probe: {Describe(probe)}"); + + output.WriteLine($"{RanMarker} {(lane == "root" ? "" : lane + " ")}relay-refused uid={NonRootWorker.EffectiveUid()} cause={noSocket.Cause}"); } public void Dispose() { - foreach (var dir in _spoolDirs) + foreach (var dir in _dirs) { - try { Directory.Delete(dir, recursive: true); } catch { /* best-effort cleanup of a spool dir */ } + try { Directory.Delete(dir, recursive: true); } catch { /* best-effort cleanup of a scratch or spool dir */ } } } + /// + /// Python defining sock_writes(sock): from inside the sandbox, try to delete the lease's socket and to plant a + /// file beside it, and return each attempt's errno name (ok if it worked) — the two writes a read-only bind of + /// the socket's directory must refuse. Shared with the allowlist arm, which binds the same directory. + /// + internal const string SocketDirectoryWrites = """ + def sock_writes(sock): + import errno, os + def attempt(write): + try: + write(); return 'ok' + except OSError as e: + return errno.errorcode.get(e.errno, str(e.errno)) + def plant(): + open(os.path.join(os.path.dirname(sock), 'planted'), 'w').close() + return attempt(lambda: os.unlink(sock)), attempt(plant) + """; + + /// + /// The socket's directory was bound READ-ONLY into the sandbox (): the child + /// connected through it, but could neither delete the worker's socket nor plant a file in the worker's directory — + /// read off both sides of the bind, the errno the child saw and what the host still has. + /// + internal static void AssertSocketDirectoryIsReadOnly(string unlink, string plant, string socketPath) + { + var directory = Path.GetDirectoryName(socketPath)!; + + unlink.ShouldBe("EROFS", $"the child must not be able to delete the worker's socket: its directory is bound read-only. Check the launch's bwrap argv binds {directory} with --ro-bind, not --bind"); + plant.ShouldBe("EROFS", "nor plant a file in the worker's socket directory"); + File.Exists(socketPath).ShouldBeTrue("the worker's socket is still there on the host"); + File.Exists(Path.Combine(directory, "planted")).ShouldBeFalse("and nothing was planted beside it on the host"); + } + /// Probes every door from inside the run and prints one JSON object of what each did. It never fails on a shut door — the test decides. - private const string ProbeScript = """ - import json, os, socket, urllib.parse, urllib.request + private const string ProbeScript = SocketDirectoryWrites + "\n" + """ + import json, os, socket, urllib.request res = {} url = os.environ['BROKER_URL'] req = urllib.request.Request(url + '/v1/messages', data=b'{}', method='POST', headers={'Authorization': 'Bearer ' + os.environ['RUN_TOKEN'], 'content-type': 'application/json'}) @@ -258,6 +312,8 @@ public void Dispose() res['broker'] = str(urllib.request.urlopen(req, timeout=10).status) except Exception as e: res['broker'] = type(e).__name__ + ':' + str(e) + if 'SOCK_PATH' in os.environ: + res['sock_unlink'], res['sock_plant'] = sock_writes(os.environ['SOCK_PATH']) def tcp(host, port): try: s = socket.create_connection((host, port), timeout=3); s.close(); return 'open' @@ -272,20 +328,85 @@ def udp(host, port): res['internet'] = tcp('1.1.1.1', 80) res['dns_tcp'] = tcp('8.8.8.8', 53) res['dns_udp'] = udp('8.8.8.8', 53) - res['gateway_other'] = tcp(urllib.parse.urlparse(url).hostname, int(os.environ['OTHER_PORT'])) + if 'OTHER_PORT' in os.environ: + res['worker_eth0'] = tcp(os.environ['WORKER_IP'], int(os.environ['OTHER_PORT'])) + res['host_loopback'] = tcp('127.0.0.1', int(os.environ['OTHER_PORT'])) + res['links'] = ','.join(sorted(name for _, name in socket.if_nameindex())) res['proxies'] = ','.join(sorted(n for n in os.environ if n.lower() in ('http_proxy', 'https_proxy', 'all_proxy'))) or 'none' print(json.dumps(res)) """; - /// A lane that seals is root with bwrap, ip and nft; there a namespace that cannot be built is a failure, not a skip. - private static bool Seals() + /// The broker through the relay, then every IPv6 target the worker holds, from inside the sandbox. + private const string Ipv6ProbeScript = """ + import json, os, socket, urllib.request + res = {} + req = urllib.request.Request(os.environ['BROKER_URL'] + '/v1/messages', data=b'{}', method='POST', headers={'Authorization': 'Bearer ' + os.environ['RUN_TOKEN'], 'content-type': 'application/json'}) + try: + res['broker'] = str(urllib.request.urlopen(req, timeout=10).status) + except Exception as e: + res['broker'] = type(e).__name__ + ':' + str(e) + port = int(os.environ['V6_PORT']) + for target in os.environ['V6_TARGETS'].split(','): + try: + s = socket.create_connection((target, port), timeout=3); s.close(); res['v6 ' + target] = 'open' + except OSError as e: + res['v6 ' + target] = type(e).__name__ + print(json.dumps(res)) + """; + + /// A kept agent: each time go<n> appears in its workspace it makes one broker call and prints the outcome with its pid; stop ends it. + private const string RestartProbeScript = """ + import os, sys, time, urllib.request + url, tok, d = os.environ['BROKER_URL'] + '/v1/messages', os.environ['RUN_TOKEN'], os.environ['SIGNAL_DIR'] + def call(): + req = urllib.request.Request(url, data=b'{}', method='POST', headers={'Authorization': 'Bearer ' + tok, 'content-type': 'application/json'}) + try: + return str(urllib.request.urlopen(req, timeout=10).status) + except Exception as e: + return 'error:' + type(e).__name__ + n = 0 + while True: + n += 1 + while not os.path.exists(os.path.join(d, 'go%d' % n)): + if os.path.exists(os.path.join(d, 'stop')): + sys.exit(0) + time.sleep(0.1) + print('call%d=%s pid=%d' % (n, call(), os.getpid()), flush=True) + """; + + /// A lane that confines has bubblewrap and the helper beside this assembly; there a missing one is a failure, not a skip. + private static bool Confines() { - if (BubblewrapSandbox.Available is null || !FilteredEgressNetns.IsSupported) return false; + if (BubblewrapSandbox.Available is null) + { + BubblewrapSandbox.IsRequired.ShouldBeFalse("Sandbox:RequireConfinement is set but this host cannot sandbox (bwrap/userns) — the E2E cannot prove how a network-off run reaches its broker here"); + return false; + } - FilteredEgressNetns.CanSeal.ShouldBeTrue("bwrap, ip and nft are all here, but this process could not build a throwaway namespace — without that privilege every network-off brokered run is severed from its model"); + File.Exists(LocalProcessRunner.McpProxyBinaryPath()).ShouldBeTrue($"the codespace-mcp helper that runs the relay must be at {LocalProcessRunner.McpProxyBinaryPath()} — the test project's build copies it there"); return true; } + /// A network-off run's lease with a socket at the path the executor mints, and the spec the executor would stamp from it. + private async Task RelayedRunAsync() + { + var runId = Guid.NewGuid(); + var socketPath = AgentRunExecutor.ModelBrokerSocketPathFor(new AgentPermissions { Network = AgentNetworkAccess.Off }, runId).ShouldNotBeNull("the executor mints a socket for a network-off run on Linux"); + var broker = LoopbackModelCredentialBroker.ForTest(new CountingUpstream()); + var brokered = (await broker.OpenAsync(Lease(runId, Guid.NewGuid(), socketPath), CancellationToken.None)).ShouldNotBeNull("the broker must be able to lease on this host"); + + brokered.SocketPath.ShouldBe(socketPath, $"the lease must bind its socket; check `ls -la {Path.GetDirectoryName(socketPath)}`"); + _dirs.Add(Path.GetDirectoryName(socketPath)!); + + return new RelayedRun(runId, socketPath, broker, brokered, AgentRunExecutor.ApplyModelBrokerChannel(ProbeSpec(brokered), brokered)); + } + + private static SandboxSpec ProbeSpec(BrokeredModelCredential brokered) => new() + { + Command = "/usr/bin/python3", Args = ["-c", ProbeScript], AllowNetwork = false, TimeoutSeconds = 60, + Environment = new Dictionary { ["BROKER_URL"] = brokered.BaseUrl, ["RUN_TOKEN"] = brokered.RunToken }, + }; + private async Task<(SandboxResult Result, Dictionary Probe)> RunDurableAsync(SandboxSpec spec) { var key = Guid.NewGuid().ToString("N"); @@ -293,18 +414,15 @@ private static bool Seals() var lines = new List(); var handle = await runner.LaunchAsync(spec, key, CancellationToken.None); - _spoolDirs.Add(handle.SpoolDirectory); + _dirs.Add(handle.SpoolDirectory); - handle.EgressNetnsKey.ShouldBe(key, "a sealed run must launch inside a namespace keyed by the run and recorded on its handle, or no reaper can tear it down"); + handle.EgressNetnsKey.ShouldBeNull("a relayed network-off run gets no namespace of the worker's, so there is nothing to tear down by name"); var confinement = handle.Confinement.ShouldNotBeNull("the launch must record what confinement it applied"); confinement.EgressSealedToBroker.ShouldBeTrue("the record must say the run was sealed to its broker, not merely severed"); confinement.NetworkSevered.ShouldBeTrue("a sealed run is severed from everything else"); var result = await runner.AttachAsync(handle, (frame, _) => { lines.Add(frame.Text); return Task.CompletedTask; }, CancellationToken.None); - (await NetnsExistsAsync(FilteredEgressPlan.NamespaceFor(key))).ShouldBeFalse("the sealed namespace must be torn down on the run's terminal path"); - (await RunHostExitAsync(["nft", "list", "table", "inet", FilteredEgressPlan.NamespaceFor(key)])).ShouldNotBe(0, "and so must its host-side inet table, which deleting the namespace does not remove"); - return (result, ParseProbe(string.Join('\n', lines))); } @@ -319,63 +437,96 @@ private static Dictionary ParseProbe(string stdout) { var line = stdout.Split('\n').Select(l => l.Trim()).LastOrDefault(l => l.StartsWith('{')); - return line is null ? new Dictionary { ["broker"] = $"no probe output: {stdout}", ["internet"] = "?", ["dns_tcp"] = "?", ["dns_udp"] = "?", ["gateway_other"] = "?" } : JsonSerializer.Deserialize>(line)!; + return line is null ? new Dictionary { ["broker"] = $"no probe output: {stdout}", ["internet"] = "?", ["dns_tcp"] = "?", ["dns_udp"] = "?", ["worker_eth0"] = "?", ["host_loopback"] = "?", ["links"] = "?", ["proxies"] = "?", ["sock_unlink"] = "?", ["sock_plant"] = "?" } : JsonSerializer.Deserialize>(line)!; } private static string Describe(Dictionary probe) => string.Join(' ', probe.Select(p => $"{p.Key}={p.Value}")); - private static ModelCredentialLeaseRequest Lease() => - new() { RunId = Guid.NewGuid(), TeamId = Guid.NewGuid(), Epoch = 1, Upstream = new() { Provider = "Anthropic", ApiKey = "sk-sealed-e2e-upstream" }, Ttl = TimeSpan.FromMinutes(5) }; - - /// The link-local address reports once duplicate-address detection has finished with it (no longer "tentative"), or null where IPv6 is disabled and there is none. Fails if it never settles. - private static async Task SettledLinkLocalAsync(IReadOnlyList listArgv) + /// Signal call and wait for the agent's line for it — bounded, naming the call (Rule 12.10). + private static async Task CallAsync(string workspace, System.Collections.Concurrent.ConcurrentQueue lines, int n) { + await File.WriteAllTextAsync(Path.Combine(workspace, $"go{n}"), ""); + var watch = Stopwatch.StartNew(); - while (true) + while (watch.Elapsed < Deadline) { - var line = (await RunHostAsync(listArgv)).Split('\n').FirstOrDefault(l => l.Contains(" fe80:", StringComparison.OrdinalIgnoreCase)); + if (lines.FirstOrDefault(line => line.StartsWith($"call{n}=", StringComparison.Ordinal)) is { } line) return line; + await Task.Delay(100); + } - if (line is null && watch.Elapsed >= TimeSpan.FromSeconds(3)) return null; // no link-local at all: IPv6 is off here + throw new Xunit.Sdk.XunitException($"the kept agent never printed call{n} within {Deadline.TotalSeconds}s; lines so far: {string.Join(" | ", lines)}"); + } - if (line is not null && !line.Contains("tentative", StringComparison.Ordinal)) - return line.Split(' ', StringSplitOptions.RemoveEmptyEntries).First(t => t.StartsWith("fe80:", StringComparison.OrdinalIgnoreCase)).Split('/')[0]; + private static string Pid(string line) => line.Split(" pid=")[1]; - watch.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(15), $"the link-local address never left duplicate-address detection: {line?.Trim()}"); - await Task.Delay(200); - } - } + /// The worker's first non-loopback IPv4 address — its eth0, where the worker's own listeners are reachable from its network. + private static string WorkerIpv4() => + NetworkInterface.GetAllNetworkInterfaces().Where(nic => nic.OperationalStatus == OperationalStatus.Up && nic.NetworkInterfaceType != NetworkInterfaceType.Loopback) + .SelectMany(nic => nic.GetIPProperties().UnicastAddresses).Select(address => address.Address).FirstOrDefault(address => address.AddressFamily == AddressFamily.InterNetwork)?.ToString() + ?? throw new Xunit.Sdk.XunitException("fixture: this host has no non-loopback IPv4 address to probe the worker at"); - private static async Task RunHostExitAsync(IReadOnlyList argv) + /// Every non-loopback IPv6 address the worker holds, link-local ones with their scope — none may be reachable from a relayed sandbox. + private static IReadOnlyList WorkerIpv6Addresses() => + NetworkInterface.GetAllNetworkInterfaces().Where(nic => nic.OperationalStatus == OperationalStatus.Up && nic.NetworkInterfaceType != NetworkInterfaceType.Loopback) + .SelectMany(nic => nic.GetIPProperties().UnicastAddresses.Select(address => (nic.Name, address.Address))) + .Where(pair => pair.Address.AddressFamily == AddressFamily.InterNetworkV6) + .Select(pair => pair.Address.IsIPv6LinkLocal ? $"{pair.Address.ToString().Split('%')[0]}%{pair.Name}" : pair.Address.ToString()).ToList(); + + private static async Task ConnectsAsync(IPAddress address, int port) { - var psi = new ProcessStartInfo { FileName = argv[0], UseShellExecute = false, RedirectStandardOutput = true, RedirectStandardError = true }; - foreach (var arg in argv.Skip(1)) psi.ArgumentList.Add(arg); + using var client = new TcpClient(address.AddressFamily); - using var process = Process.Start(psi)!; - await process.WaitForExitAsync(); + try { await client.ConnectAsync(address, port).WaitAsync(TimeSpan.FromSeconds(3)); return true; } + catch (Exception) { return false; } + } - return process.ExitCode; + private string NewDirectory(string label) + { + var directory = Path.Combine(Path.GetTempPath(), $"cs-sealed-{label}-{Guid.NewGuid():N}"); + Directory.CreateDirectory(directory); + _dirs.Add(directory); + return directory; } - private static async Task NetnsExistsAsync(string ns) => - (await RunHostAsync(["ip", "netns", "list"])).Split('\n').Any(line => line.Trim().Split(' ').FirstOrDefault() == ns); + private static ModelCredentialLeaseRequest Lease(Guid runId, Guid teamId, string? socketPath) => + new() { RunId = runId, TeamId = teamId, Epoch = 1, Upstream = new() { Provider = "Anthropic", ApiKey = "sk-sealed-e2e-upstream" }, Ttl = TimeSpan.FromMinutes(5), SocketPath = socketPath }; - private static async Task RunHostAsync(IReadOnlyList argv) + /// The re-bind the next worker builds from what the run's durable handle carries. + private static ModelCredentialRebindRequest RebindOf(BrokeredModelCredential brokered, Guid runId, Guid teamId, long epoch) => new() + { + RunId = runId, TeamId = teamId, Epoch = epoch, Port = brokered.RebindPort!.Value, PathId = brokered.RebindRoute!, + RunToken = brokered.RunToken, Upstream = new() { Provider = "Anthropic", ApiKey = "sk-sealed-e2e-upstream" }, Ttl = TimeSpan.FromMinutes(5), + }; + + private static async Task RunHostExitAsync(IReadOnlyList argv) { var psi = new ProcessStartInfo { FileName = argv[0], UseShellExecute = false, RedirectStandardOutput = true, RedirectStandardError = true }; foreach (var arg in argv.Skip(1)) psi.ArgumentList.Add(arg); using var process = Process.Start(psi)!; - var stdout = await process.StandardOutput.ReadToEndAsync(); await process.WaitForExitAsync(); - return stdout; + return process.ExitCode; } - /// The provider, answering 200 to anything the broker relays — this lane asserts reachability, never what a model said. - private sealed class AlwaysOkUpstream : HttpMessageHandler + /// A network-off run's lease, the broker holding it, and the spec the executor stamps from it. + private sealed record RelayedRun(Guid RunId, string SocketPath, LoopbackModelCredentialBroker Broker, BrokeredModelCredential Brokered, SandboxSpec Spec) : IDisposable { - protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) => - Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent("{\"ok\":true}") }); + public void Dispose() => Broker.Dispose(); + } + + /// The provider, answering 200 to anything the broker relays and counting it — this lane asserts reachability, never what a model said. + private sealed class CountingUpstream : HttpMessageHandler + { + private int _calls; + + public int Calls => Volatile.Read(ref _calls); + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + Interlocked.Increment(ref _calls); + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent("{\"ok\":true}") }); + } } } diff --git a/backend/tests/CodeSpace.SandboxTests/TestOutputLogger.cs b/backend/tests/CodeSpace.SandboxTests/TestOutputLogger.cs new file mode 100644 index 000000000..62dd85f31 --- /dev/null +++ b/backend/tests/CodeSpace.SandboxTests/TestOutputLogger.cs @@ -0,0 +1,24 @@ +using Microsoft.Extensions.Logging; +using Xunit.Abstractions; + +namespace CodeSpace.SandboxTests; + +/// +/// A logger that writes a component's Warnings (and worse) into the test's own output, so a refusal the component +/// explains only in its log — a re-bind that could not take its port or re-open its socket — is readable next to the +/// assertion it failed (Rule 12.10). Output written after the test has finished is dropped rather than thrown. +/// +internal sealed class TestOutputLogger(ITestOutputHelper output) : ILogger +{ + public IDisposable? BeginScope(TState state) where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => logLevel >= LogLevel.Warning; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) + { + if (!IsEnabled(logLevel)) return; + + try { output.WriteLine($"[{typeof(T).Name} {logLevel}] {formatter(state, exception)}{(exception is null ? "" : $" ({exception.GetType().Name}: {exception.Message})")}"); } + catch (InvalidOperationException) { /* the test is already over */ } + } +} diff --git a/backend/tests/CodeSpace.UnitTests/Agents/AgentAutonomyPolicyTests.cs b/backend/tests/CodeSpace.UnitTests/Agents/AgentAutonomyPolicyTests.cs index abf6c98f8..f99654c2c 100644 --- a/backend/tests/CodeSpace.UnitTests/Agents/AgentAutonomyPolicyTests.cs +++ b/backend/tests/CodeSpace.UnitTests/Agents/AgentAutonomyPolicyTests.cs @@ -238,9 +238,9 @@ public void DescribeNetwork_discloses_a_host_whose_filtered_egress_subnet_reserv AgentAutonomyPolicy.DescribeNetwork(AgentAutonomyLevel.Standard, AgentAutonomyLevel.Trusted, Unbounded) .ShouldNotContain(AgentAutonomyPolicy.ProcessLocalSubnetCaveat, customMessage: "a severed run reserves no /30 at all, so the sentence must not carry a caveat about one"); - // A network-off run SEALED to its broker does hold a /30, so it carries the caveat a severed one must not. + // A network-off run SEALED to its broker reaches it through the relay and its socket, and holds no /30 either. AgentAutonomyPolicy.DescribeNetwork(AgentAutonomyLevel.Standard, AgentAutonomyLevel.Trusted, Unbounded, new SandboxConfinement { Outcome = SandboxConfinementOutcome.Confined, NetworkSevered = true, EgressSealedToBroker = true }) - .ShouldBe("Network: off (Standard)" + AgentAutonomyPolicy.SealedToBrokerQualifier + AgentAutonomyPolicy.ProcessLocalSubnetCaveat, "a sealed run's /30 is only unique inside one worker here too"); + .ShouldBe("Network: off (Standard)" + AgentAutonomyPolicy.SealedToBrokerQualifier, "a sealed run reserves no /30, so the sentence must not carry a caveat about one"); } AgentAutonomyPolicy.DescribeNetwork(AgentAutonomyLevel.Trusted, AgentAutonomyLevel.Trusted, Unbounded) diff --git a/backend/tests/CodeSpace.UnitTests/Agents/AgentRunExecutorEgressTests.cs b/backend/tests/CodeSpace.UnitTests/Agents/AgentRunExecutorEgressTests.cs index fe5cc8601..20c2ecdf2 100644 --- a/backend/tests/CodeSpace.UnitTests/Agents/AgentRunExecutorEgressTests.cs +++ b/backend/tests/CodeSpace.UnitTests/Agents/AgentRunExecutorEgressTests.cs @@ -1,5 +1,6 @@ using CodeSpace.Core.Services.Agents; using CodeSpace.Core.Services.Agents.Sandbox.Isolation; +using CodeSpace.Core.Services.Agents.Sandbox.Runners; using CodeSpace.Messages.Agents; using Shouldly; @@ -28,17 +29,64 @@ public void Full_egress_leaves_the_spec_unchanged() } [Theory] - [InlineData(AgentNetworkAccess.Off, 43121, 43121)] // network off + a brokered model → the port a confining runner seals it to - [InlineData(AgentNetworkAccess.Off, null, null)] // network off, unbrokered → nothing to seal to; severed as always - [InlineData(AgentNetworkAccess.On, 43121, null)] // network on reaches its broker already; its egress is the allowlist's business - public void Only_a_network_off_brokered_run_carries_its_broker_port(AgentNetworkAccess network, int? brokerPort, int? expected) + [InlineData(false, false, 43121, true, 43121, true)] // network off + a brokered model → the port and socket the relay carries it through + [InlineData(false, false, 43121, false, 43121, false)] // network off, a lease with no socket → the port alone (a confining runner refuses it) + [InlineData(false, false, null, false, null, false)] // network off, unbrokered → nothing to reach; severed as always + [InlineData(true, true, 43121, true, 43121, true)] // an allowlist → its network is its own, so it is relayed too + [InlineData(true, false, 43121, true, null, false)] // network on reaches its broker already, on loopback + public void Only_a_brokered_run_whose_network_is_its_own_carries_its_broker_channel(bool allowNetwork, bool allowlist, int? brokerPort, bool socket, int? expectedPort, bool expectSocket) { - var spec = new SandboxSpec { Command = "agent" }; + var spec = new SandboxSpec { Command = "agent", AllowNetwork = allowNetwork, EgressAllowlist = allowlist ? ["gw.example.com"] : null }; + var brokered = brokerPort is null ? null : new BrokeredModelCredential("http://{broker}:43121/r", "token", DateTimeOffset.UtcNow) { RebindPort = brokerPort, SocketPath = socket ? "/spool/k/broker/seg/s" : null }; - var result = AgentRunExecutor.ApplySealedEgress(spec, new AgentPermissions { Network = network }, brokerPort); + var result = AgentRunExecutor.ApplyModelBrokerChannel(spec, brokered); - result.ModelBrokerPort.ShouldBe(expected); - if (expected is null) result.ShouldBeSameAs(spec, "nothing to stamp — the spec is returned untouched"); + result.ModelBrokerPort.ShouldBe(expectedPort); + result.ModelBrokerSocketPath.ShouldBe(expectSocket ? "/spool/k/broker/seg/s" : null); + if (expectedPort is null) result.ShouldBeSameAs(spec, "nothing to stamp — the spec is returned untouched"); + } + + [Theory] + [InlineData(AgentNetworkAccess.Off, AgentEgressPolicy.Full, new string[0])] + [InlineData(AgentNetworkAccess.On, AgentEgressPolicy.Full, new string[0])] + [InlineData(AgentNetworkAccess.On, AgentEgressPolicy.Allowlist, new[] { "registry.npmjs.org" })] + [InlineData(AgentNetworkAccess.On, AgentEgressPolicy.Allowlist, new string[0])] // an allowlist with nothing derivable: severed + [InlineData(AgentNetworkAccess.Off, AgentEgressPolicy.Allowlist, new[] { "registry.npmjs.org" })] + public void The_lease_asks_for_a_socket_exactly_when_the_built_spec_turns_out_to_need_one(AgentNetworkAccess network, AgentEgressPolicy egress, string[] extraHosts) + { + // Drift pin: the lease is opened before the spec exists, so it decides from the PERMISSIONS whether the child's + // network will be its own; the spec's channel is stamped from the SPEC. If the two ever disagree, a child either + // needs a socket its lease never asked for, or a network-sharing run is handed one it never uses. + var permissions = new AgentPermissions { Network = network, Egress = egress, EgressAllowHosts = extraHosts }; + var harnessSpec = new SandboxSpec { Command = "agent", AllowNetwork = network == AgentNetworkAccess.On }; + var built = AgentRunExecutor.ApplyEgressPolicy(harnessSpec, permissions, modelBaseUrl: null, modelProvider: null, workspace: null); + var brokered = new BrokeredModelCredential("http://{broker}:43121/r", "token", DateTimeOffset.UtcNow) { RebindPort = 43121, SocketPath = "/spool/k/broker/seg/s" }; + + var stamped = AgentRunExecutor.ApplyModelBrokerChannel(built, brokered).ModelBrokerPort is not null; + + AgentRunExecutor.ChildNetworkIsPrivate(permissions).ShouldBe(stamped, $"permissions {network}/{egress}/[{string.Join(',', extraHosts)}] built to AllowNetwork={built.AllowNetwork}, allowlist=[{string.Join(',', built.EgressAllowlist ?? [])}]"); + } + + [Theory] + [InlineData(AgentNetworkAccess.Off, AgentEgressPolicy.Full, true)] + [InlineData(AgentNetworkAccess.On, AgentEgressPolicy.Allowlist, true)] + [InlineData(AgentNetworkAccess.On, AgentEgressPolicy.Full, false)] // Trusted: the worker's own network — no socket, nothing changes + public void A_socket_is_minted_only_on_linux_and_only_for_a_run_whose_network_is_its_own(AgentNetworkAccess network, AgentEgressPolicy egress, bool privateNetwork) + { + var runId = Guid.NewGuid(); + + var first = AgentRunExecutor.ModelBrokerSocketPathFor(new AgentPermissions { Network = network, Egress = egress }, runId); + var second = AgentRunExecutor.ModelBrokerSocketPathFor(new AgentPermissions { Network = network, Egress = egress }, runId); + + if (!OperatingSystem.IsLinux() || !privateNetwork) + { + first.ShouldBeNull("a host that never confines, or a run that shares the worker's network, mints nothing — its spec, argv and environment stay exactly as they were"); + return; + } + + first.ShouldNotBeNull().ShouldEndWith("/s"); + Path.GetFileName(Path.GetDirectoryName(Path.GetDirectoryName(first))).ShouldBeOneOf(LocalProcessRunner.ModelBrokerSocketDir, LocalProcessRunner.ModelBrokerShortSocketRoot); + first.ShouldNotBe(second, "a path serves one lease at a time, so every open mints a fresh, unguessable one"); } [Fact] diff --git a/backend/tests/CodeSpace.UnitTests/Agents/ModelCredentialBrokerTests.cs b/backend/tests/CodeSpace.UnitTests/Agents/ModelCredentialBrokerTests.cs index 4eb3fcf61..be2ca9405 100644 --- a/backend/tests/CodeSpace.UnitTests/Agents/ModelCredentialBrokerTests.cs +++ b/backend/tests/CodeSpace.UnitTests/Agents/ModelCredentialBrokerTests.cs @@ -110,18 +110,56 @@ public async Task A_live_lease_relays_the_call_with_the_tenants_key_attached_ser upstream.SeenHeaderValues.ShouldNotContain(brokered.RunToken, "the per-run bearer authenticates to the broker only; forwarding it leaks a capability the provider has no use for"); } - [Fact] - public async Task A_lease_says_whether_a_per_run_namespace_can_reach_it() + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task A_new_lease_listens_on_loopback_only_even_where_a_per_run_namespace_can_exist(bool socket) { - // A sealed network-off run reaches the worker at its namespace gateway, never on loopback, so the lease has to - // say whether it took the wide bind. The wide bind is tried only where namespaces can exist; everywhere else - // it binds loopback and must say it is NOT reachable, which is what refuses a sealed launch before it spends. + // A child in a network of its own reaches its lease through the socket and the relay in its sandbox, so no new + // lease has any reason to hand its port to the host's neighbours — with a socket or without, and on a host + // that builds filtered-egress namespaces (the sandbox lane, a Linux lane with ip and nft) as on one that + // cannot. Mutation: bind wide again where FilteredEgressNetns.IsSupported, and this goes red on such a host. + if (socket && !Socket.OSSupportsUnixDomainSockets) return; + + using var sockets = new BrokerSockets(); using var broker = LoopbackModelCredentialBroker.ForTest(new StubUpstream()); + var runId = Guid.NewGuid(); - var brokered = await broker.OpenAsync(LeaseFor(Guid.NewGuid()), CancellationToken.None); + var brokered = await broker.OpenAsync(LeaseFor(runId) with { SocketPath = socket ? sockets.NewPath() : null }, CancellationToken.None); if (brokered is null) return; // this host cannot bind a listener at all — nothing to assert - brokered.ReachableFromNamespace.ShouldBe(FilteredEgressNetns.IsSupported, "reachable from a namespace exactly when the broker bound every address, which it tries only where a namespace can exist"); + broker.ListenerPrefixForTest(runId).ShouldBe($"http://127.0.0.1:{brokered.RebindPort}/", $"a lease binds loopback alone (namespaces possible here: {FilteredEgressNetns.IsSupported})"); + } + + [Theory] + [InlineData(false, true)] // the legacy re-bind: a handle written before the socket, whose namespaced child calls the gateway + [InlineData(false, false)] // no socket and no namespace: a child on the worker's own network (network on, or an unconfined host) calls loopback + [InlineData(true, true)] // a handle with a socket: its child comes in through it, spliced to loopback + public async Task Only_a_rebind_of_a_namespaced_child_without_a_socket_binds_wide_and_only_where_a_namespace_can_exist(bool socket, bool netns) + { + // Mutation: key the wide bind on the missing socket alone, and the no-namespace row goes red on a host that + // builds filtered-egress namespaces — a Trusted run's loopback lease would come back on every address. + if (socket && !Socket.OSSupportsUnixDomainSockets) return; + + using var sockets = new BrokerSockets(); + using var broker = LoopbackModelCredentialBroker.ForTest(new StubUpstream()); + var port = ReserveLoopbackPort(); + var request = RebindOn(port, epoch: 3) with { SocketPath = socket ? sockets.NewPath() : null, ChildInNetworkNamespace = netns }; + + if (!await broker.RebindAsync(request, CancellationToken.None)) return; // the port was taken in between — nothing to observe + + var wide = !socket && netns && FilteredEgressNetns.IsSupported; + + broker.ListenerPrefixForTest(request.RunId).ShouldBe(wide ? $"http://+:{port}/" : $"http://127.0.0.1:{port}/", $"only a veth-sealed or allowlist run launched before the relay reaches this worker at its namespace gateway, so its re-bind alone keeps the wide bind until they drain; every other child calls loopback (namespaces possible here: {FilteredEgressNetns.IsSupported})"); + } + + /// A loopback port free at the moment of asking — a re-bind names its port, it never picks one. + private static int ReserveLoopbackPort() + { + using var probe = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + probe.Bind(new IPEndPoint(IPAddress.Loopback, 0)); + + return ((IPEndPoint)probe.LocalEndPoint!).Port; } [Fact] @@ -365,12 +403,8 @@ public async Task Rebind_restores_the_recorded_port_route_and_token_so_the_origi [Fact] public async Task Rebind_reports_false_when_the_port_is_taken_rather_than_pretending_it_worked() { - // The fixture holds ONE address — loopback, which is the only candidate host a worker that cannot build - // filtered-egress namespaces ever tries. A host that CAN build them prefers the wide bind, which this fixture - // does not hold, so the refusal would not be falsifiable there. No unit lane is such a host (the privileged - // job runs SandboxTests, not this assembly); the guard is here so the test stays honest if that changes, - // rather than quietly passing because the broker bound a different address than the one under test. - if (FilteredEgressNetns.IsSupported) return; + // The fixture holds ONE address — loopback, which is the only candidate host for a re-bind whose child is not + // in a network namespace of its own, on every host, including one that builds filtered-egress namespaces. // Held for the WHOLE test: released early, the broker could bind the very port this is meant to deny it. using var occupied = new OccupiedPort(); @@ -589,11 +623,12 @@ public async Task A_call_over_the_lease_socket_is_relayed_with_the_tenants_key_a using var broker = LoopbackModelCredentialBroker.ForTest(upstream); var path = sockets.NewPath(); - var brokered = await broker.OpenAsync(LeaseFor(Guid.NewGuid()) with { SocketPath = path }, CancellationToken.None); + var runId = Guid.NewGuid(); + var brokered = await broker.OpenAsync(LeaseFor(runId) with { SocketPath = path }, CancellationToken.None); if (brokered is null) return; // this host cannot bind a loopback listener at all — nothing to assert brokered.SocketPath.ShouldBe(path, "the lease must say which socket it bound — that path is what the durable handle records for a re-attach to re-open"); - brokered.ReachableFromNamespace.ShouldBeFalse( + broker.ListenerPrefixForTest(runId).ShouldBe($"http://127.0.0.1:{brokered.RebindPort}/", "a lease served over a socket binds its TCP listener on LOOPBACK only: its namespaced children come in through the socket, so a wide bind would only hand the port to every neighbour on the host's network"); (await CallOverSocketAsync(path, brokered, brokered.RunToken)).ShouldBe(HttpStatusCode.OK, diff --git a/backend/tests/CodeSpace.UnitTests/Agents/SupervisorDeciderTests.cs b/backend/tests/CodeSpace.UnitTests/Agents/SupervisorDeciderTests.cs index aa4c404e8..cd7bde5d7 100644 --- a/backend/tests/CodeSpace.UnitTests/Agents/SupervisorDeciderTests.cs +++ b/backend/tests/CodeSpace.UnitTests/Agents/SupervisorDeciderTests.cs @@ -542,7 +542,7 @@ public void The_two_deployment_exit_reasons_get_the_remedy_each_one_actually_has // A host that cannot seal is the same kind of deployment answer as one that cannot broker, about the network. LlmSupervisorDecider.EndedByDeploymentSteer(FailureCodes.SandboxSealedEgressUnavailable) - .ShouldBe("RETRY this exact subtask once, in case another worker can seal its network to its model broker; if it ends the same way again, 'ask_human' — that is a deployment setting only an operator can change. Either way do NOT re-plan it and do NOT amend its check — there is nothing wrong with either."); + .ShouldBe("RETRY this exact subtask once, in case another worker can relay its sandbox to its model broker; if it ends the same way again, 'ask_human' — that is a deployment setting only an operator can change. Either way do NOT re-plan it and do NOT amend its check — there is nothing wrong with either."); leaseLost.ShouldNotBe(brokerDown, "one remedy text for two different faults is how a bounded repair becomes an unbounded loop"); leaseLost.ShouldNotContain("ask_human", Case.Sensitive, "a live worker is the whole repair — escalating a rolling restart to a human is noise"); diff --git a/backend/tests/CodeSpace.UnitTests/Workflows/FilteredEgressPlanTests.cs b/backend/tests/CodeSpace.UnitTests/Workflows/FilteredEgressPlanTests.cs index 1c1d4d8f1..b4381f5fa 100644 --- a/backend/tests/CodeSpace.UnitTests/Workflows/FilteredEgressPlanTests.cs +++ b/backend/tests/CodeSpace.UnitTests/Workflows/FilteredEgressPlanTests.cs @@ -26,47 +26,7 @@ public void Names_are_run_unique_and_consistent_across_the_plan() a.ExecPrefix.ShouldBe(new[] { "ip", "netns", "exec", a.Namespace }, "the command runs inside this run's netns"); a.TeardownCommands.ShouldContain(c => c.SequenceEqual(new[] { "ip", "netns", "del", a.Namespace }), "teardown deletes the netns"); a.TeardownCommands.ShouldContain(c => c.SequenceEqual(new[] { "nft", "delete", "table", "ip", a.Namespace }), "teardown deletes the nft table"); - a.TeardownCommands.ShouldContain(c => c.SequenceEqual(new[] { "nft", "delete", "table", "inet", a.Namespace }), "teardown deletes a sealed run's inet table too — every reaper knows only the run id"); - } - - [Fact] - public void A_sealed_plan_has_no_route_no_forwarding_and_no_nat() - { - var plan = FilteredEgressPlan.BuildSealed("run-5ea1ed01", 43121, Subnet); - - plan.SetupCommands.ShouldNotContain(c => c.Contains("route"), "no default route: a packet to anywhere but the /30 fails with ENETUNREACH at once"); - plan.SetupCommands.ShouldNotContain(c => c[0] == "sysctl", "nothing is forwarded, so the host's forwarding switch is left alone"); - plan.SetupCommands.ShouldContain(c => c.SequenceEqual(new[] { "ip", "netns", "exec", plan.Namespace, "ip", "addr", "add", $"{Subnet.NsIp}/30", "dev", plan.VethNs }), "the namespace still holds its /30, so the gateway is on-link"); - plan.HostIp.ShouldBe(Subnet.HostIp, "the gateway the child reaches its broker at"); - plan.ExecPrefix.ShouldBe(new[] { "ip", "netns", "exec", plan.Namespace }); - plan.TeardownCommands.Select(c => string.Join(' ', c)).ShouldBe(FilteredEgressPlan.TeardownCommandsFor("run-5ea1ed01").Select(c => string.Join(' ', c)), "a sealed namespace is torn down by the same run-id-only commands every reaper already runs"); - } - - [Fact] - public void A_sealed_ruleset_admits_only_the_broker_port_on_the_gateway() - { - // Pinned whole, because every line is load-bearing and a membership assertion is satisfied by the wrong chain: - // inet (the veth's IPv6 link-local must be covered by the same drop), replace-not-append (a revise round reuses - // the name), an INPUT filter admitting only the broker's port on the gateway (the worker's own listeners and - // every other run's broker are reached through that hook), and a FORWARD drop. No DNS, no NAT, and keyed on the - // veth — never a subnet a degraded allocator might hand another run too. - var plan = FilteredEgressPlan.BuildSealed("run-5ea1ed02", 43121, Subnet); - - plan.NftRuleset.ShouldBe( - $"table inet {plan.Namespace} {{}}\n" + - $"delete table inet {plan.Namespace}\n" + - $"table inet {plan.Namespace} {{\n" + - " chain input {\n" + - " type filter hook input priority 0;\n" + - $" iifname \"{plan.VethHost}\" ct state established,related accept\n" + - $" iifname \"{plan.VethHost}\" ip daddr {Subnet.HostIp} tcp dport 43121 accept\n" + - $" iifname \"{plan.VethHost}\" drop\n" + - " }\n" + - " chain forward {\n" + - " type filter hook forward priority 0;\n" + - $" iifname \"{plan.VethHost}\" drop\n" + - " }\n" + - "}\n"); + a.TeardownCommands.ShouldContain(c => c.SequenceEqual(new[] { "nft", "delete", "table", "inet", a.Namespace }), "teardown still deletes the inet table a network-off run sealed to its broker before the relay left behind — every reaper knows only the run id"); } [Fact] @@ -129,10 +89,7 @@ public void Setup_creates_the_netns_and_veth_and_default_route() [Fact] public void The_route_check_asks_the_kernel_how_the_host_reaches_the_namespace_end() { - var plan = FilteredEgressPlan.BuildSealed("run-ffff6666", 43121, Subnet); - - plan.RouteCheckArgv.ShouldBe(new[] { "ip", "route", "get", "10.5.7.18", "from", "10.5.7.17", "ipproto", "6", "sport", "43121" }, "what a reply from the broker carries — to the namespace's end, from the gateway, TCP from its port — so a rule keyed on the protocol or the source port is seen too; as text, which every iproute2 prints"); - FilteredEgressPlan.Build("run-ffff6666", new[] { "1.1.1.1" }, Subnet).RouteCheckArgv.ShouldBe(new[] { "ip", "route", "get", "10.5.7.18", "from", "10.5.7.17" }, "an allowlist run has no one port its replies come from"); + FilteredEgressPlan.Build("run-ffff6666", new[] { "1.1.1.1" }, Subnet).RouteCheckArgv.ShouldBe(new[] { "ip", "route", "get", "10.5.7.18", "from", "10.5.7.17" }, "to the namespace's end, from the gateway — an allowlist run has no one port its replies come from; as text, which every iproute2 prints"); } [Theory] @@ -149,10 +106,10 @@ public void The_route_check_asks_the_kernel_how_the_host_reaches_the_namespace_e public void Only_a_route_through_the_run_s_own_veth_passes_the_check(int exit, string output, string? failure) { // The /30 was chosen from the routes the host lists, but a policy rule consults its tables in rule order, not - // by prefix length: a blackhole 10.0.0.0/8 in a table checked before main discards the broker's replies to a + // by prefix length: a blackhole 10.0.0.0/8 in a table checked before main discards the replies to a // cleanly set-up namespace. Only the kernel's own lookup sees that, so anything but a route through the run's // own veth fails the setup instead of admitting a run that can never be answered. - var plan = FilteredEgressPlan.BuildSealed("run-ffff6666", 43121, Subnet); + var plan = FilteredEgressPlan.Build("run-ffff6666", new[] { "1.1.1.1" }, Subnet); var reason = plan.RouteCheckFailure(exit, output); diff --git a/backend/tests/CodeSpace.UnitTests/Workflows/LocalProcessDurableRunnerTests.cs b/backend/tests/CodeSpace.UnitTests/Workflows/LocalProcessDurableRunnerTests.cs index 16ff22ff1..959524308 100644 --- a/backend/tests/CodeSpace.UnitTests/Workflows/LocalProcessDurableRunnerTests.cs +++ b/backend/tests/CodeSpace.UnitTests/Workflows/LocalProcessDurableRunnerTests.cs @@ -715,7 +715,7 @@ public void A_read_only_spec_keeps_its_workspace_out_of_the_writable_binds_and_t { var spec = new SandboxSpec { Command = "/bin/sh", WorkingDirectory = "/work/ws", ReadOnlyWorkingDirectory = readOnly }; - var plan = LocalProcessRunner.PlanFor(spec, spec.Args, "/spool/agent-home", Array.Empty()); + var plan = LocalProcessRunner.PlanFor(spec, spec.Command, spec.Args, "/spool/agent-home", Array.Empty()); plan.WorkingDirectoryReadOnly.ShouldBe(readOnly); plan.WritablePaths.Contains("/work/ws").ShouldBe(!readOnly, "a read-only run's workspace is never offered as a writable path"); @@ -737,7 +737,7 @@ public void An_mcp_launch_binds_its_socket_directory_read_only_and_the_helper_fi // unlinking the socket or planting a file beside it. var spec = new SandboxSpec { Command = "/usr/local/bin/claude", WorkingDirectory = "/work/ws", Mcp = Wiring("/spool/k/mcp/seg/s") }; - var plan = LocalProcessRunner.PlanFor(spec, spec.Args, "/spool/k/agent-home", Array.Empty()); + var plan = LocalProcessRunner.PlanFor(spec, spec.Command, spec.Args, "/spool/k/agent-home", Array.Empty()); var argv = BubblewrapSandbox.BuildArgs(plan).ToList(); plan.WritablePaths.ShouldNotContain("/spool/k/mcp/seg", "the socket's directory is never offered as a writable path"); @@ -857,21 +857,49 @@ public void A_durable_codex_launch_stands_its_sandbox_down_exactly_where_this_ho } [Theory] - [InlineData(false, true, false)] // network off inside a namespace: the sealed launch — its proxy is unreachable, so it goes - [InlineData(false, false, true)] // network off with no namespace: severed, nothing to route — left as it always was - [InlineData(true, true, true)] // network on inside a namespace: an allowlist run, which may well reach its proxy - public void A_sealed_launch_carries_no_proxy_its_namespace_cannot_reach(bool allowNetwork, bool inNamespace, bool keepsProxy) + [InlineData(false, true, false, true, false)] // network off, relayed under bwrap: the sealed launch — its proxy is unreachable, so it goes + [InlineData(false, true, false, false, true)] // network off on a host that does not confine: it shares the worker's network, left as it always was + [InlineData(false, false, false, true, true)] // network off with no socket on its lease: not relayed (the admission refuses it) — nothing is stripped + [InlineData(true, true, true, true, true)] // an allowlist run relayed inside its namespace: its allowlist may well admit its proxy + [InlineData(true, true, false, true, true)] // network on with no allowlist: the worker's own network + public void A_sealed_launch_carries_no_proxy_its_namespace_cannot_reach(bool allowNetwork, bool socket, bool inNamespace, bool confines, bool keepsProxy) { - var spec = new SandboxSpec { Command = "agent", AllowNetwork = allowNetwork, Environment = new Dictionary { ["HTTPS_PROXY"] = "http://proxy.corp:3128", ["http_proxy"] = "http://proxy.corp:3128", ["NO_PROXY"] = "localhost" } }; - var prefix = inNamespace ? new[] { "ip", "netns", "exec", "cs-egr-deadbeef" } : Array.Empty(); + var spec = new SandboxSpec { Command = "agent", AllowNetwork = allowNetwork, EgressAllowlist = inNamespace ? ["api.anthropic.com"] : null, ModelBrokerPort = 43121, ModelBrokerSocketPath = socket ? "/spool/k/broker/seg/s" : null }; - var info = LocalProcessRunner.BuildDurableStartInfo(spec, TempDir(), prefix); + LocalProcessRunner.SealedEgress(spec, inNamespace, confines).ShouldBe(!keepsProxy, "only a network-off run the relay carries to its broker is sealed: its one destination is that broker, and a CLI honouring a proxy would send every call it does not exempt where the child cannot reach"); + } + + [Fact] + public void A_durable_network_off_brokered_launch_drops_its_proxies_exactly_where_this_host_relays_it() + { + // Honest on either host: where bwrap confines, the child is severed and relayed, so its proxies go; where it does + // not, the child shares the worker's network, and its proxies stay exactly as they were. + var spec = new SandboxSpec { Command = "agent", ModelBrokerPort = 43121, ModelBrokerSocketPath = "/spool/k/broker/seg/s", Environment = new Dictionary { ["HTTPS_PROXY"] = "http://proxy.corp:3128", ["http_proxy"] = "http://proxy.corp:3128", ["NO_PROXY"] = "localhost" } }; + var relayed = BubblewrapSandbox.Available is not null; - info.Environment.ContainsKey("HTTPS_PROXY").ShouldBe(keepsProxy, "a sealed child's one destination is its broker on the gateway; a CLI honouring a proxy would send every model call where the namespace cannot reach"); - info.Environment.ContainsKey("http_proxy").ShouldBe(keepsProxy, "both spellings"); + var info = LocalProcessRunner.BuildDurableStartInfo(spec, TempDir()); + + info.Environment.ContainsKey("HTTPS_PROXY").ShouldBe(!relayed, $"the proxy strip reads the same predicate as the relay (confines={relayed})"); + info.Environment.ContainsKey("http_proxy").ShouldBe(!relayed, "both spellings"); info.Environment.ContainsKey("NO_PROXY").ShouldBeTrue("NO_PROXY is harmless and left alone"); } + [Fact] + public void An_allowlist_launch_keeps_its_proxy_and_exempts_the_relay_on_its_loopback() + { + var spec = new SandboxSpec + { + Command = "agent", AllowNetwork = true, EgressAllowlist = ["api.anthropic.com"], ModelBrokerPort = 43121, ModelBrokerSocketPath = "/spool/k/broker/seg/s", + Environment = new Dictionary { ["ANTHROPIC_BASE_URL"] = $"http://{SandboxSpec.ModelBrokerHostToken}:43121/route", ["HTTPS_PROXY"] = "http://proxy.corp:3128", ["NO_PROXY"] = "localhost" }, + }; + + var info = LocalProcessRunner.BuildDurableStartInfo(LocalProcessRunner.ResolveModelBrokerHost(spec), TempDir(), ["ip", "netns", "exec", "cs-egr-deadbeef"]); + + info.Environment["HTTPS_PROXY"].ShouldBe("http://proxy.corp:3128", "an allowlist run's network is filtered, not off: its proxy may be exactly what the allowlist admits"); + info.Environment["NO_PROXY"].ShouldNotBeNull().Split(',').ShouldContain("127.0.0.1", "its broker is the relay on its own loopback, which must never be sent through that proxy"); + info.Environment["ANTHROPIC_BASE_URL"].ShouldBe("http://127.0.0.1:43121/route", "the relay answers the CLI's loopback address inside the namespace"); + } + private static readonly IReadOnlyList CodexArgs = new[] { "exec", "--json", "--model", "gpt-5.4", "--sandbox", "read-only", "-c", "otel.exporter=none", "-" }; [Fact] diff --git a/backend/tests/CodeSpace.UnitTests/Workflows/LocalProcessRunnerEnvScrubTests.cs b/backend/tests/CodeSpace.UnitTests/Workflows/LocalProcessRunnerEnvScrubTests.cs index a51d22989..41e469229 100644 --- a/backend/tests/CodeSpace.UnitTests/Workflows/LocalProcessRunnerEnvScrubTests.cs +++ b/backend/tests/CodeSpace.UnitTests/Workflows/LocalProcessRunnerEnvScrubTests.cs @@ -290,23 +290,23 @@ public async Task The_watchdog_char_pump_delivers_whole_lines_trims_CRLF_and_flu finally { Environment.SetEnvironmentVariable(LocalProcessRunner.StdoutIdleTimeoutEnvVar, prior); } } - [Theory] - [InlineData("10.63.12.1", "http://10.63.12.1:41234/r0uteId")] // inside a filtered-egress netns: its own gateway is the only address the worker has there - [InlineData(null, "http://127.0.0.1:41234/r0uteId")] // sharing the host network: loopback - [InlineData("", "http://127.0.0.1:41234/r0uteId")] // a netns that reported no address is treated as no netns, never left as a token - public void The_model_broker_host_token_is_resolved_to_the_address_this_child_can_reach(string? gatewayIp, string expected) + [Fact] + public void The_model_broker_host_token_is_resolved_to_loopback_in_every_child() { // The ARGV carrier is not decoration. Codex ignores OPENAI_BASE_URL and re-emits the broker's URL as a `-c` // model-provider override, so a pass that substituted only the ENV shipped a literal "{codespace:…}" host on // the command line — the CLI then could not even build an HTTP request ("builder error"), which reads as a - // provider outage. Mutation: resolve Environment only, and the Args assertion below goes red. + // provider outage. Mutation: resolve Environment only, and the Args assertion below goes red. Loopback in every + // child, a namespaced one included: there the relay answers 127.0.0.1 inside the child's own namespace, and the + // resolution takes no gateway to get wrong. + const string expected = "http://127.0.0.1:41234/r0uteId"; var spec = EnvSpec() with { Args = new[] { "-c", $"model_providers.codespace.base_url=http://{SandboxSpec.ModelBrokerHostToken}:41234/r0uteId/v1" }, Environment = new Dictionary { ["ANTHROPIC_BASE_URL"] = $"http://{SandboxSpec.ModelBrokerHostToken}:41234/r0uteId", ["ANTHROPIC_AUTH_TOKEN"] = "run-token" }, }; - var resolved = LocalProcessRunner.ResolveModelBrokerHost(spec, gatewayIp); + var resolved = LocalProcessRunner.ResolveModelBrokerHost(spec); resolved.Environment["ANTHROPIC_BASE_URL"].ShouldBe(expected, customMessage: "the token must never survive into a child — a base URL still carrying it is a broken URL the CLI reports as a network error, not as a refused credential"); @@ -316,18 +316,17 @@ public void The_model_broker_host_token_is_resolved_to_the_address_this_child_ca } [Theory] - [InlineData("10.63.12.1", "localhost,127.0.0.1", null, "localhost,127.0.0.1,10.63.12.1", "localhost,127.0.0.1,10.63.12.1")] // a netns run: its gateway joins what is already exempt - [InlineData(null, "localhost", null, "localhost,127.0.0.1", "localhost,127.0.0.1")] // a loopback broker a NO_PROXY happens to omit - [InlineData(null, "localhost,127.0.0.1", null, "localhost,127.0.0.1", "localhost,127.0.0.1")] // already exempt: left as it was - [InlineData("10.63.12.1", ".corp.internal", null, ".corp.internal,10.63.12.1", ".corp.internal,10.63.12.1")] // only one spelling set: BOTH carry the operator's entries - [InlineData("10.63.12.1", null, "gitlab.corp", "gitlab.corp,10.63.12.1", "gitlab.corp,10.63.12.1")] - [InlineData("10.63.12.1", "a.corp", "b.corp", "a.corp,b.corp,10.63.12.1", "a.corp,b.corp,10.63.12.1")] // both set: their union, in both - [InlineData("10.63.12.1", "*", null, "*,10.63.12.1", "*")] // bypass-everything: the uppercase readers still need the IP named - [InlineData(null, "a.corp", "*", "a.corp,*,127.0.0.1", "*")] - public void A_brokered_child_is_told_not_to_proxy_its_broker(string? gatewayIp, string? upper, string? lower, string expectedUpper, string expectedLower) + [InlineData("localhost", null, "localhost,127.0.0.1", "localhost,127.0.0.1")] // a loopback broker a NO_PROXY happens to omit + [InlineData("localhost,127.0.0.1", null, "localhost,127.0.0.1", "localhost,127.0.0.1")] // already exempt: left as it was + [InlineData(".corp.internal", null, ".corp.internal,127.0.0.1", ".corp.internal,127.0.0.1")] // only one spelling set: BOTH carry the operator's entries + [InlineData(null, "gitlab.corp", "gitlab.corp,127.0.0.1", "gitlab.corp,127.0.0.1")] + [InlineData("a.corp", "b.corp", "a.corp,b.corp,127.0.0.1", "a.corp,b.corp,127.0.0.1")] // both set: their union, in both + [InlineData("*", null, "*,127.0.0.1", "*")] // bypass-everything: the uppercase readers still need the IP named + [InlineData("a.corp", "*", "a.corp,*,127.0.0.1", "*")] + public void A_brokered_child_is_told_not_to_proxy_its_broker(string? upper, string? lower, string expectedUpper, string expectedLower) { - // The broker's address is one no operator's NO_PROXY can name ahead of time — a per-run gateway — so a CLI - // honouring the worker's proxy sent every model call to the proxy instead. Readers prefer different spellings, + // The broker's address is loopback, which an operator's NO_PROXY may simply omit — and an allowlist run keeps its + // proxy while its broker is the relay on that loopback — so a CLI honouring the proxy sent every model call there instead. Readers prefer different spellings, // and a spelling created with the broker alone hid the operator's own entries. A `*` is the one entry the two // families read differently: Codex's reqwest takes it as an entry that matches no IP address (its brokered // calls went to the proxy with NO_PROXY=* alone), while curl, Python and undici honour it only as the whole value. @@ -335,7 +334,7 @@ public void A_brokered_child_is_told_not_to_proxy_its_broker(string? gatewayIp, if (upper is not null) environment["NO_PROXY"] = upper; if (lower is not null) environment["no_proxy"] = lower; - var resolved = WithWorkerProxyEnvironment(new Dictionary(), () => LocalProcessRunner.ResolveModelBrokerHost(EnvSpec() with { Environment = environment }, gatewayIp)); + var resolved = WithWorkerProxyEnvironment(new Dictionary(), () => LocalProcessRunner.ResolveModelBrokerHost(EnvSpec() with { Environment = environment })); resolved.Environment["NO_PROXY"].ShouldBe(expectedUpper); resolved.Environment["no_proxy"].ShouldBe(expectedLower); @@ -352,21 +351,21 @@ public void A_brokered_child_with_no_proxy_keeps_its_environment(string? workerO var environment = new Dictionary { ["ANTHROPIC_BASE_URL"] = $"http://{SandboxSpec.ModelBrokerHostToken}:41234/r0uteId" }; var worker = workerOnly is null ? new Dictionary() : new Dictionary { [workerOnly] = "socks5h://proxy.corp:1080" }; - var resolved = WithWorkerProxyEnvironment(worker, () => LocalProcessRunner.ResolveModelBrokerHost(EnvSpec() with { Environment = environment }, "10.63.12.1")); + var resolved = WithWorkerProxyEnvironment(worker, () => LocalProcessRunner.ResolveModelBrokerHost(EnvSpec() with { Environment = environment })); resolved.Environment.Keys.ShouldBe(new[] { "ANTHROPIC_BASE_URL" }, customMessage: "only the broker host is substituted when the child has no proxy to exempt it from"); } [Theory] - [InlineData(null, "10.63.12.1")] - [InlineData(".corp.internal,gitlab.corp", ".corp.internal,gitlab.corp,10.63.12.1")] // the worker's own exemptions survive into the list, or the agent's git and pip to internal hosts go through the proxy + [InlineData(null, "127.0.0.1")] + [InlineData(".corp.internal,gitlab.corp", ".corp.internal,gitlab.corp,127.0.0.1")] // the worker's own exemptions survive into the list, or the agent's git and pip to internal hosts go through the proxy public void A_proxy_the_worker_passes_through_the_scrub_is_one_the_child_is_exempted_from(string? workerNoProxy, string expected) { var environment = new Dictionary { ["ANTHROPIC_BASE_URL"] = $"http://{SandboxSpec.ModelBrokerHostToken}:41234/r0uteId" }; var worker = new Dictionary { ["https_proxy"] = "http://proxy.corp:3128" }; if (workerNoProxy is not null) worker["NO_PROXY"] = workerNoProxy; - var resolved = WithWorkerProxyEnvironment(worker, () => LocalProcessRunner.ResolveModelBrokerHost(EnvSpec() with { Environment = environment }, "10.63.12.1")); + var resolved = WithWorkerProxyEnvironment(worker, () => LocalProcessRunner.ResolveModelBrokerHost(EnvSpec() with { Environment = environment })); resolved.Environment["NO_PROXY"].ShouldBe(expected, "the worker's https_proxy survives the scrub and reaches the child, so the broker must be exempted from it"); resolved.Environment["no_proxy"].ShouldBe(expected); @@ -379,10 +378,10 @@ public void A_proxy_the_task_hands_its_child_is_one_the_child_is_exempted_from() // gate must count them whether or not the scrub would keep the worker's copy of the name. var environment = new Dictionary { ["ANTHROPIC_BASE_URL"] = $"http://{SandboxSpec.ModelBrokerHostToken}:41234/r0uteId", ["ALL_PROXY"] = "socks5h://proxy.corp:1080" }; - var resolved = WithWorkerProxyEnvironment(new Dictionary(), () => LocalProcessRunner.ResolveModelBrokerHost(EnvSpec() with { Environment = environment }, "10.63.12.1")); + var resolved = WithWorkerProxyEnvironment(new Dictionary(), () => LocalProcessRunner.ResolveModelBrokerHost(EnvSpec() with { Environment = environment })); - resolved.Environment["NO_PROXY"].ShouldBe("10.63.12.1", "a brokered call sent to the task's proxy fails like a provider outage"); - resolved.Environment["no_proxy"].ShouldBe("10.63.12.1"); + resolved.Environment["NO_PROXY"].ShouldBe("127.0.0.1", "a brokered call sent to the task's proxy fails like a provider outage"); + resolved.Environment["no_proxy"].ShouldBe("127.0.0.1"); } /// Run with every proxy variable of this process cleared but : the worker's own values are the fallback, and this host's must not leak in. @@ -403,7 +402,7 @@ public void A_command_carrying_the_broker_host_token_is_resolved_too() // design: a carrier this pass skips is a token that reaches a child, and each skipped carrier cost a run. var spec = EnvSpec() with { Command = $"/opt/reach-{SandboxSpec.ModelBrokerHostToken}" }; - LocalProcessRunner.ResolveModelBrokerHost(spec, "10.63.12.1").Command.ShouldBe("/opt/reach-10.63.12.1"); + LocalProcessRunner.ResolveModelBrokerHost(spec).Command.ShouldBe("/opt/reach-127.0.0.1"); } [Fact] @@ -411,7 +410,7 @@ public void A_spec_with_no_broker_token_is_returned_untouched() { var spec = EnvSpec() with { Args = new[] { "-c", "echo ok" } }; - LocalProcessRunner.ResolveModelBrokerHost(spec, "10.63.12.1").ShouldBeSameAs(spec, + LocalProcessRunner.ResolveModelBrokerHost(spec).ShouldBeSameAs(spec, "every run whose credential was not brokered must keep a byte-identical command, argv and environment — and pay nothing for a feature it isn't using"); } diff --git a/backend/tests/CodeSpace.UnitTests/Workflows/ModelBrokerRelayRunnerTests.cs b/backend/tests/CodeSpace.UnitTests/Workflows/ModelBrokerRelayRunnerTests.cs new file mode 100644 index 000000000..8c6830bd4 --- /dev/null +++ b/backend/tests/CodeSpace.UnitTests/Workflows/ModelBrokerRelayRunnerTests.cs @@ -0,0 +1,233 @@ +using CodeSpace.Core.Services.Agents.Harnesses.Codex; +using CodeSpace.Core.Services.Agents.Sandbox.Isolation; +using CodeSpace.Core.Services.Agents.Sandbox.Runners; +using CodeSpace.Mcp.Relay; +using CodeSpace.Messages.Agents; +using Shouldly; + +namespace CodeSpace.UnitTests.Workflows; + +/// +/// Pins how the local runner puts a brokered CLI whose network is its own behind the codespace-mcp relay: the +/// one predicate that decides it (), the order of the chain it builds +/// (cgroup → netns → prlimit → bwrap → relay → cli), what bubblewrap binds for it, the confinement record, and +/// that every launch the relay is not for keeps exactly the argv it had. Pure over the host's two tool paths +/// (), so every posture is pinned on any host; the real kernel is the +/// sandbox lane's (SealedEgressE2ETests, NonRootWorkerE2ETests). +/// +[Trait("Category", "Unit")] +public sealed class ModelBrokerRelayRunnerTests +{ + private const string Bwrap = "/usr/bin/bwrap"; + private const string Prlimit = "/usr/bin/prlimit"; + private const int Port = 43121; + private const string Socket = "/spool/k/broker/seg/s"; + + // ── The relay's argv, and the helper that reads it ────────────────────────────────────────────────────────── + + [Fact] + public void The_relay_wrap_is_the_helper_s_own_verb_and_the_helper_parses_exactly_what_it_builds() + { + // A cross-process seam with no compiler across it: the runner writes this argv, the helper parses it. So the + // assertion goes through the helper's REAL parser, not through the writer's output alone. + var (command, args) = ModelBrokerRelay.Wrap("/app/codespace-mcp", Port, Socket, "claude", ["-p", "--model", "m"]); + + command.ShouldBe("/app/codespace-mcp"); + args.ShouldBe(new[] { "relay", "43121", Socket, "--", "claude", "-p", "--model", "m" }); + ModelBrokerRelay.Verb.ShouldBe(BrokerRelay.Verb, "the runner's verb is the one the helper dispatches on"); + + var parsed = BrokerRelayCommand.Parse(args.Skip(1).ToList()); + + parsed.Port.ShouldBe(Port); + parsed.Broker.ToString().ShouldBe(Socket); + parsed.Cli.ShouldBe("claude"); + parsed.CliArgs.ShouldBe(new[] { "-p", "--model", "m" }); + } + + [Fact] + public void The_admission_reads_back_the_helper_s_own_listen_failed_status() => + // The admission asks the helper to listen on a port this process holds and takes this status as "it runs the + // relay" (ModelBrokerRelay.HelperRunsRelay). The helper's constant owns the value; the two ends must agree. + ModelBrokerRelay.ListenFailedExitCode.ShouldBe(BrokerRelay.ListenFailedExitCode); + + // ── The one predicate ────────────────────────────────────────────────────────────────────────────────────── + + [Theory] + [InlineData(true, true, false, false, false, true, true)] // network off, confined: bwrap severs it — relayed + [InlineData(true, true, false, false, false, false, false)] // network off on a host that does not confine: it shares the worker's network and calls loopback itself + [InlineData(true, false, false, false, false, true, false)] // no socket on the lease: nothing to relay to (the admission refuses it) + [InlineData(false, true, false, false, false, true, false)] // no broker port: nothing brokered + [InlineData(true, true, true, false, false, true, false)] // network on, no allowlist: the worker's own network + [InlineData(true, true, true, true, true, true, true)] // an allowlist in its filtered namespace, under bwrap + [InlineData(true, true, true, true, true, false, true)] // an allowlist in its namespace on a host with no bwrap: relayed without it + [InlineData(true, true, true, true, false, true, true)] // an allowlist this host cannot enforce: bwrap severs it, and it is relayed + [InlineData(true, true, true, true, false, false, false)] // …and unconfined, it shares the worker's network + public void A_child_is_relayed_exactly_when_its_network_is_its_own_and_its_lease_has_a_socket(bool port, bool socket, bool allowNetwork, bool allowlist, bool inNamespace, bool confines, bool relays) + { + var spec = new SandboxSpec { Command = "agent", AllowNetwork = allowNetwork, EgressAllowlist = allowlist ? ["api.anthropic.com"] : null, ModelBrokerPort = port ? Port : null, ModelBrokerSocketPath = socket ? Socket : null }; + + LocalProcessRunner.RelaysModelBroker(spec, inNamespace, confines).ShouldBe(relays); + } + + // ── The chain, in order ─────────────────────────────────────────────────────────────────────────────────── + + [Fact] + public void A_network_off_relayed_chain_is_cgroup_then_prlimit_then_bwrap_then_the_relay_then_the_cli() + { + var spec = RelayedSpec(allowNetwork: false) with { Command = "/usr/bin/claude", Args = ["-p", "go"], MaxProcesses = 64, MaxFileSizeMb = 1 }; + + var argv = Chain(spec, egressPrefix: [], bwrap: Bwrap); + + argv.Take(2).ShouldBe(new[] { "CGSELF", "x" }, "the cgroup self-add is outermost"); + argv[2].ShouldBe(Prlimit, "no namespace of the worker's for a network-off run: prlimit comes next"); + At(argv, Bwrap).ShouldBeGreaterThan(At(argv, Prlimit), "bubblewrap inside prlimit"); + argv.ShouldContain("--unshare-net", "the network is off: bubblewrap severs it, and the relay is the one way out"); + + var relay = At(argv, LocalProcessRunner.McpProxyBinaryPath()); + + relay.ShouldBeGreaterThan(At(argv, Bwrap), "the relay runs INSIDE bubblewrap, so it binds the sandbox's loopback and not the worker's"); + argv[relay - 1].ShouldBe("--", "the relay is bubblewrap's command"); + argv.Skip(relay).ShouldBe(new[] { LocalProcessRunner.McpProxyBinaryPath(), "relay", Port.ToString(), Socket, "--", "/usr/bin/claude", "-p", "go" }, "and the CLI, with its own argv, is the relay's"); + } + + [Fact] + public void An_allowlist_relayed_chain_enters_its_namespace_before_prlimit_and_shares_it_inside_bwrap() + { + var spec = RelayedSpec(allowNetwork: true) with { EgressAllowlist = ["api.anthropic.com"], Command = "/usr/bin/codex", Args = ["exec"] }; + + var argv = Chain(spec, egressPrefix: ["EGRESS", "y"], bwrap: Bwrap); + + argv.Take(4).ShouldBe(new[] { "CGSELF", "x", "EGRESS", "y" }, "cgroup, then the filtered namespace, outermost"); + argv[4].ShouldBe(Prlimit); + argv.ShouldNotContain("--unshare-net", "inside its filtered namespace bubblewrap shares it, so the allowlist still holds"); + argv.TakeLast(7).ShouldBe(new[] { LocalProcessRunner.McpProxyBinaryPath(), "relay", Port.ToString(), Socket, "--", "/usr/bin/codex", "exec" }, "the relay is innermost, in front of the CLI"); + } + + [Fact] + public void In_a_namespace_on_a_host_with_no_bwrap_the_relay_is_applied_directly() + { + var spec = RelayedSpec(allowNetwork: true) with { EgressAllowlist = ["api.anthropic.com"], Command = "codex", Args = ["exec"] }; + + var argv = Chain(spec, egressPrefix: ["EGRESS", "y"], bwrap: null, prlimit: null); + + argv.ShouldBe(new[] { "CGSELF", "x", "EGRESS", "y", LocalProcessRunner.McpProxyBinaryPath(), "relay", Port.ToString(), Socket, "--", "codex", "exec" }, "the P6 posture: a namespace with no bubblewrap still needs the relay, and nothing stands its CLI's own sandbox down"); + } + + [Fact] + public void A_cli_s_own_sandbox_is_stood_down_on_its_own_argv_inside_the_relay() + { + // The stand-down swaps the CLI's argv, so it must land on the CLI's, not the relay's: applied after the relay + // wrapped the argv, the fragment would not be found, and the launch would refuse. + var substitution = new ArgsSubstitution { Replace = ["--sandbox", "read-only"], With = ["--sandbox", CodexHarness.ConfinedSandboxMode] }; + var spec = RelayedSpec(allowNetwork: false) with { Command = "codex", Args = ["exec", "--sandbox", "read-only", "-"], WhenRunnerConfines = substitution }; + + var argv = Chain(spec, egressPrefix: [], bwrap: Bwrap, prlimit: null); + + argv.TakeLast(5).ShouldBe(new[] { "codex", "exec", "--sandbox", CodexHarness.ConfinedSandboxMode, "-" }, "the CLI runs stood down behind the relay"); + argv.ShouldNotContain("read-only"); + } + + [Fact] + public void A_durable_launch_on_this_host_relays_exactly_where_bwrap_confines_it() + { + // Honest on either host: the durable builder is the same chain, fed this host's own tool paths. + var spec = RelayedSpec(allowNetwork: false) with { Command = "agent", Args = ["go"] }; + + var argv = LocalProcessRunner.BuildDurableStartInfo(spec, Path.Combine(Path.GetTempPath(), "cs-relay-" + Guid.NewGuid().ToString("N"))).ArgumentList; + + argv.Contains("relay").ShouldBe(BubblewrapSandbox.Available is not null, "a network-off brokered child is relayed wherever bubblewrap severs it, and nowhere else"); + argv.TakeLast(2).ShouldBe(new[] { "agent", "go" }, "the CLI and its argv still trail the chain"); + } + + // ── Every other launch keeps its argv ──────────────────────────────────────────────────────────────────────── + + [Theory] + [InlineData(true, Bwrap)] // network on, no allowlist: the child shares the worker's network and calls its lease directly + [InlineData(false, null)] // network off on a host that does not confine (macOS dev, a pod without userns): unconfined, nothing to relay + public void A_lease_with_a_socket_changes_nothing_for_a_child_that_shares_the_worker_s_network(bool allowNetwork, string? bwrap) + { + var relayedLease = RelayedSpec(allowNetwork) with { Command = "/usr/bin/claude", Args = ["-p", "go"] }; + var noLease = relayedLease with { ModelBrokerPort = null, ModelBrokerSocketPath = null }; + + Chain(relayedLease, egressPrefix: [], bwrap).ShouldBe(Chain(noLease, egressPrefix: [], bwrap), "byte for byte: the broker's socket is for children that cannot reach loopback, and this one can"); + } + + // ── What bubblewrap binds for it ──────────────────────────────────────────────────────────────────────────── + + [Fact] + public void A_relayed_plan_binds_the_socket_directory_and_the_helper_read_only_and_the_cli_s_own_directory() + { + var spec = RelayedSpec(allowNetwork: false) with { Command = "/srv/cs-cli/bin/claude", WorkingDirectory = "/work/ws" }; + var (relay, relayArgs) = ModelBrokerRelay.Wrap(LocalProcessRunner.McpProxyBinaryPath(), Port, Socket, spec.Command, spec.Args); + + var plan = LocalProcessRunner.PlanFor(spec, relay, relayArgs, "/spool/k/agent-home", Array.Empty()); + var argv = BubblewrapSandbox.BuildArgs(plan).ToList(); + + plan.Command.ShouldBe(relay, "the relay is what bubblewrap runs"); + BoundAs(argv, "/spool/k/broker/seg").ShouldBe("--ro-bind-try", "the socket's own directory, read-only: connect() needs no write, and a read-only bind stops the agent unlinking or planting a socket"); + plan.WritablePaths.ShouldNotContain("/spool/k/broker/seg"); + foreach (var file in LocalProcessRunner.McpProxyFiles(relay)) BoundAs(argv, file).ShouldBe("--ro-bind-try", $"the helper that runs the relay is bound file by file ({file})"); + BoundAs(argv, Path.GetDirectoryName(relay)!).ShouldBeEmpty("never the helper's directory, the worker's own app dir"); + BoundAs(argv, "/srv/cs-cli/bin").ShouldBe("--ro-bind-try", "the CLI is no longer bubblewrap's command, so its own directory is bound for it, as it was when it was"); + } + + [Fact] + public void A_relayed_cli_under_the_read_only_roots_needs_no_bind_of_its_own() + { + var spec = RelayedSpec(allowNetwork: false) with { Command = "/usr/bin/claude" }; + var (relay, relayArgs) = ModelBrokerRelay.Wrap(LocalProcessRunner.McpProxyBinaryPath(), Port, Socket, spec.Command, spec.Args); + + var plan = LocalProcessRunner.PlanFor(spec, relay, relayArgs, null, Array.Empty()); + + plan.ReadOnlyExtraPaths.ShouldNotContain("/usr/bin", "/usr is already a read-only root"); + } + + [Fact] + public void A_plan_that_is_not_relayed_binds_nothing_new() + { + var spec = new SandboxSpec { Command = "/srv/cs-cli/bin/claude", AllowNetwork = true, ModelBrokerPort = Port, ModelBrokerSocketPath = Socket }; + + var plan = LocalProcessRunner.PlanFor(spec, spec.Command, spec.Args, null, Array.Empty()); + + plan.ReadOnlyExtraPaths.ShouldBeEmpty("a child on the worker's own network reaches its lease on loopback: no socket, no helper, no extra directory"); + plan.Command.ShouldBe(spec.Command); + } + + // ── The record ───────────────────────────────────────────────────────────────────────────────────────────── + + [Theory] + [InlineData(false, true, false, true, true)] // network off, relayed: severed, and sealed to its broker + [InlineData(false, false, false, true, false)] // network off, no socket: severed, and nothing reached (the admission refuses it) + [InlineData(true, true, true, false, false)] // an allowlist relayed in its namespace: filtered, not severed, not sealed + public void A_confined_launch_records_the_route_its_relay_kept(bool allowNetwork, bool socket, bool inNamespace, bool expectedSevered, bool expectedSealed) + { + var spec = new SandboxSpec { Command = "agent", AllowNetwork = allowNetwork, EgressAllowlist = inNamespace ? ["api.anthropic.com"] : null, ModelBrokerPort = Port, ModelBrokerSocketPath = socket ? Socket : null }; + + var record = LocalProcessRunner.LaunchConfinement(spec, inNamespace ? ["ip", "netns", "exec", "cs-egr-deadbeef"] : Array.Empty(), Bwrap, unavailableReason: null); + + record.Outcome.ShouldBe(SandboxConfinementOutcome.Confined); + record.NetworkSevered.ShouldBe(expectedSevered); + record.EgressSealedToBroker.ShouldBe(expectedSealed, "sealed means severed from everything but the broker, which is what the relay leaves a network-off run"); + } + + [Fact] + public void An_unconfined_launch_records_that_it_was_unconfined_whatever_its_lease() + { + var record = LocalProcessRunner.LaunchConfinement(RelayedSpec(allowNetwork: false), Array.Empty(), bwrap: null, SandboxConfinement.ReasonNotLinux); + + record.Outcome.ShouldBe(SandboxConfinementOutcome.Unconfined); + record.EgressSealedToBroker.ShouldBeFalse("nothing sealed a run nothing confined"); + } + + private static SandboxSpec RelayedSpec(bool allowNetwork) => new() { Command = "agent", AllowNetwork = allowNetwork, ModelBrokerPort = Port, ModelBrokerSocketPath = Socket }; + + /// The chain this spec launches behind, on a host whose bwrap and prlimit are the given paths (null where absent), under a stand-in cgroup prefix. + private static IReadOnlyList Chain(SandboxSpec spec, IReadOnlyList egressPrefix, string? bwrap, string? prlimit = Prlimit) => + LocalProcessRunner.ChildCommand(new LocalProcessRunner.CommandIsolationContext(spec, null, null, egressPrefix, ["CGSELF", "x"]), bwrap, prlimit); + + private static int At(IReadOnlyList argv, string token) => argv.ToList().LastIndexOf(token); + + /// The flags that mount onto itself, joined; empty when nothing does. + private static string BoundAs(IReadOnlyList argv, string path) => + string.Join(" ", Enumerable.Range(0, argv.Count - 2).Where(i => argv[i + 1] == path && argv[i + 2] == path).Select(i => argv[i])); +} diff --git a/backend/tests/CodeSpace.UnitTests/Workflows/NativeLaunchRegistryTests.cs b/backend/tests/CodeSpace.UnitTests/Workflows/NativeLaunchRegistryTests.cs index 2b67d4108..876435633 100644 --- a/backend/tests/CodeSpace.UnitTests/Workflows/NativeLaunchRegistryTests.cs +++ b/backend/tests/CodeSpace.UnitTests/Workflows/NativeLaunchRegistryTests.cs @@ -152,7 +152,9 @@ public void A_broker_port_binds_the_launch_and_a_spec_without_one_serializes_as_ var spec = NativeLaunchProtocol.Freeze(new SandboxSpec { Command = "/bin/sh" }); NativeLaunchProtocol.SpecHash(spec with { ModelBrokerPort = 43121 }).ShouldNotBe(NativeLaunchProtocol.SpecHash(spec), "a launch sealed to a broker is a different execution from one severed from everything"); + NativeLaunchProtocol.SpecHash(spec with { ModelBrokerPort = 43121, ModelBrokerSocketPath = "/spool/k/broker/seg/s" }).ShouldNotBe(NativeLaunchProtocol.SpecHash(spec with { ModelBrokerPort = 43121 }), "a relayed launch is a different execution from one with no socket to relay to"); JsonSerializer.Serialize(spec, NativeLaunchProtocol.Json).ShouldNotContain("modelBrokerPort", customMessage: "an unbrokered spec must serialize, and hash, exactly as it did before the field existed"); + JsonSerializer.Serialize(spec with { ModelBrokerPort = 43121 }, NativeLaunchProtocol.Json).ShouldNotContain("modelBrokerSocketPath", customMessage: "nor does a spec whose lease has no socket carry the socket field"); } [Fact] diff --git a/backend/tests/CodeSpace.UnitTests/Workflows/SandboxEgressPolicyTests.cs b/backend/tests/CodeSpace.UnitTests/Workflows/SandboxEgressPolicyTests.cs index 1b1b8863e..a79be5644 100644 --- a/backend/tests/CodeSpace.UnitTests/Workflows/SandboxEgressPolicyTests.cs +++ b/backend/tests/CodeSpace.UnitTests/Workflows/SandboxEgressPolicyTests.cs @@ -19,16 +19,14 @@ public void No_network_is_None_regardless_of_an_allowlist() } [Theory] - [InlineData(false, 43121, SandboxEgressMode.Sealed)] // network off + a broker it can be sealed to → sealed to that port - [InlineData(false, null, SandboxEgressMode.None)] // network off, nothing to seal to → severed, as always - [InlineData(true, 43121, SandboxEgressMode.Full)] // network on already reaches its broker — a port never narrows or widens it - public void A_network_off_run_is_sealed_only_to_a_broker_it_was_given(bool allowNetwork, int? sealablePort, SandboxEgressMode expected) + [InlineData(false, SandboxEgressMode.None)] // network off, a brokered run included: it reaches its broker through the relay and its socket, never through a network + [InlineData(true, SandboxEgressMode.Full)] // network on already reaches its broker — having one never narrows or widens it + public void A_network_off_run_is_severed_whatever_reaches_its_broker(bool allowNetwork, SandboxEgressMode expected) { - var policy = SandboxEgressPolicy.Derive(allowNetwork, allowlist: null, canEnforceAllowlist: true, sealablePort); + var policy = SandboxEgressPolicy.Derive(allowNetwork, allowlist: null, canEnforceAllowlist: true); - policy.Mode.ShouldBe(expected); - policy.BrokerPort.ShouldBe(expected == SandboxEgressMode.Sealed ? sealablePort : null, "the sealed port is carried only by the sealed mode"); - policy.AllowedHosts.ShouldBeEmpty("a sealed run resolves no hosts — its one destination is an address, not a name"); + policy.Mode.ShouldBe(expected, "there is no mode between severed and full for a run with no allowlist — the broker is not a network destination"); + policy.AllowedHosts.ShouldBeEmpty("no allowlist, so no hosts to resolve"); } [Fact] diff --git a/backend/tests/CodeSpace.UnitTests/Workflows/SealedEgressAdmissionTests.cs b/backend/tests/CodeSpace.UnitTests/Workflows/SealedEgressAdmissionTests.cs index e5739ca1c..55a8156ca 100644 --- a/backend/tests/CodeSpace.UnitTests/Workflows/SealedEgressAdmissionTests.cs +++ b/backend/tests/CodeSpace.UnitTests/Workflows/SealedEgressAdmissionTests.cs @@ -8,64 +8,175 @@ namespace CodeSpace.UnitTests.Workflows; /// -/// Pins the local runner's refusal of a network-off brokered run it would confine but cannot seal — the admission the -/// executor asks for before anything is spent. The executor asking at all, and landing the refusal typed with no spend -/// and no process, is pinned one tier up in AgentRunExecutorTests; the kernel refusing a real setup is pinned in -/// the sandbox lane. +/// Pins the local runner's refusal of a brokered run whose child would have a network of its own on this host but no +/// way to its broker — the admission the executor asks for before anything is spent. The executor asking at all, and +/// landing the refusal typed with no spend and no process, is pinned one tier up in AgentRunExecutorTests; the +/// worker that cannot build a namespace admitting such a run is pinned on the real kernel in the non-root lane. +/// +/// The helper is ASKED whether it runs the relay (), so the stand-ins +/// here are real executables: one answers the way the relay does, one the way the MCP proxy from before the relay +/// does. The shipped helper's own answer is pinned below against the real apphost beside this assembly. POSIX only. /// [Trait("Category", "Unit")] -public class SealedEgressAdmissionTests +public sealed class SealedEgressAdmissionTests : IDisposable { + private readonly string _dir = Directory.CreateTempSubdirectory("cs-relay-helper-").FullName; + + [Theory] + [InlineData(true, true, Helper.Relay, true, null)] // everything the relay needs: admitted + [InlineData(true, false, Helper.Relay, true, SealedEgressUnavailableException.CauseBrokerSocketUnavailable)] // the broker could not bind the socket the relay connects to + [InlineData(true, true, Helper.Missing, true, SealedEgressUnavailableException.CauseRelayMissing)] // no helper to run the relay + [InlineData(true, true, Helper.PreRelay, true, SealedEgressUnavailableException.CauseRelayMissing)] // a helper built before the relay: its MCP proxy reads any argv as its own + [InlineData(true, true, Helper.SelfContained, true, SealedEgressUnavailableException.CauseRelayMissing)] // a self-contained publish, which cannot start from the files the sandbox binds + [InlineData(true, true, Helper.NotExecutable, true, SealedEgressUnavailableException.CauseRelayMissing)] // a file that cannot start at all + [InlineData(true, true, Helper.BindsAnyway, true, SealedEgressUnavailableException.CauseRelayMissing)] // one that starts its CLI on a port it cannot have bound is not the relay + [InlineData(true, false, Helper.Missing, true, SealedEgressUnavailableException.CauseBrokerSocketUnavailable)] // both missing: the socket is named first, since it is per run + [InlineData(true, false, Helper.Missing, false, null)] // the child shares the worker's network: it calls loopback itself + [InlineData(false, false, Helper.Missing, true, null)] // nothing brokered: nothing to reach + public void Each_wall_a_private_network_child_can_hit_is_named(bool port, bool socket, Helper helper, bool privateNetwork, string? expected) + { + if (OperatingSystem.IsWindows()) return; + + var spec = new SandboxSpec { Command = "agent", ModelBrokerPort = port ? 43121 : null, ModelBrokerSocketPath = socket ? "/spool/k/broker/seg/s" : null }; + + var refusal = LocalProcessRunner.RelayRefusal(spec, privateNetwork, HelperAt(helper)); + + if (expected is null) refusal.ShouldBeNull(); + else refusal.ShouldNotBeNull().ShouldStartWith(expected, customMessage: "the cause leads with the wall"); + } + + [Fact] + public void A_missing_helper_is_named_by_the_path_that_was_looked_for() + { + var spec = new SandboxSpec { Command = "agent", ModelBrokerPort = 43121, ModelBrokerSocketPath = "/spool/k/broker/seg/s" }; + + LocalProcessRunner.RelayRefusal(spec, childNetworkIsPrivate: true, "/nowhere/codespace-mcp").ShouldBe($"{SealedEgressUnavailableException.CauseRelayMissing} (looked for /nowhere/codespace-mcp)", "an operator is told where the worker looked, which is where CODESPACE_MCP_PROXY_PATH points or next to its assembly"); + } + + [Theory] + [InlineData(Helper.PreRelay, "{path} did not answer as the relay: it predates it, or cannot start")] + [InlineData(Helper.SelfContained, "{path} is a self-contained publish of several files, which cannot start from the files the sandbox binds")] + public void A_helper_that_is_there_but_cannot_relay_is_named_with_why(Helper helper, string why) + { + if (OperatingSystem.IsWindows()) return; + + var spec = new SandboxSpec { Command = "agent", ModelBrokerPort = 43121, ModelBrokerSocketPath = "/spool/k/broker/seg/s" }; + var path = HelperAt(helper); + + LocalProcessRunner.RelayRefusal(spec, childNetworkIsPrivate: true, path).ShouldBe($"{SealedEgressUnavailableException.CauseRelayMissing} ({why.Replace("{path}", path)})", "the operator is told which file and what is wrong with it, since the file is there"); + } + + [Fact] + public void The_shipped_helper_answers_as_the_relay() + { + if (OperatingSystem.IsWindows()) return; + + // The real apphost the unit build lands beside this assembly, where LocalProcessRunner.McpProxyBinaryPath() looks. + var shipped = Path.Combine(AppContext.BaseDirectory, "codespace-mcp"); + File.Exists(shipped).ShouldBeTrue($"fixture: the unit build lands the codespace-mcp apphost at {shipped}"); + + ModelBrokerRelay.HelperRunsRelay(shipped).ShouldBeTrue("the helper this release ships must answer the admission's question as the relay — refused on a port it cannot bind, having started nothing — or every brokered run whose network is its own is refused"); + } + + [Fact] + public void Only_a_yes_is_remembered_and_only_for_the_file_that_gave_it() + { + if (OperatingSystem.IsWindows()) return; + + var path = Path.Combine(_dir, "replaced-codespace-mcp"); + + WriteExecutable(path, PreRelayScript); + ModelBrokerRelay.HelperRunsRelay(path).ShouldBeFalse("control: a helper from before the relay does not answer as it"); + + WriteExecutable(path, RelayScript + "\n# replaced by this release's build"); + ModelBrokerRelay.HelperRunsRelay(path).ShouldBeTrue("a no is not remembered: the operator who replaces the helper is admitted on the next run, without a restart"); + + WriteExecutable(path, PreRelayScript + "\n# rolled back to a build from before the relay"); + ModelBrokerRelay.HelperRunsRelay(path).ShouldBeFalse("a yes belongs to the file that gave it: a replaced helper is asked again"); + } + [Theory] - [InlineData(false, false, false, SealedEgressUnavailableException.CauseMissingTools)] - [InlineData(true, false, true, SealedEgressUnavailableException.CauseNoPrivilege)] - [InlineData(true, true, false, SealedEgressUnavailableException.CauseBrokerLoopbackOnly)] - [InlineData(true, true, true, null)] - public void Each_wall_a_confining_host_can_hit_is_named(bool haveTools, bool canSeal, bool brokerReachable, string? expected) => - LocalProcessRunner.SealRefusal(haveTools, canSeal, brokerReachable).ShouldBe(expected); + [InlineData(false, null, false, true, true)] // network off: severed wherever bwrap confines + [InlineData(false, null, false, false, false)] // but unconfined, a network-off child shares the worker's network + [InlineData(true, null, false, true, false)] // network on, no allowlist: the worker's network + [InlineData(true, "api.anthropic.com", false, true, true)] // an allowlist bwrap cannot enforce: severed + [InlineData(true, "api.anthropic.com", true, false, true)] // an allowlist in its filtered namespace, confined or not + public void The_admission_reads_the_same_private_network_rule_as_the_launch(bool allowNetwork, string? allowlistHost, bool inNamespace, bool confines, bool expected) + { + var spec = new SandboxSpec { Command = "agent", AllowNetwork = allowNetwork, EgressAllowlist = allowlistHost is null ? null : [allowlistHost] }; + + LocalProcessRunner.ChildNetworkIsPrivate(spec, inNamespace, confines).ShouldBe(expected); + } [Fact] public void A_spec_with_no_broker_port_is_admitted_on_any_host() => - // Nothing to seal: every unbrokered or network-on run launches exactly as it always did, even on a host that could not seal. - Should.NotThrow(() => new LocalProcessRunner().EnsureEgressAdmissible(new SandboxSpec { Command = "agent" }, modelBrokerReachableFromNamespace: false)); + // Nothing to reach: every unbrokered run launches exactly as it always did. + Should.NotThrow(() => new LocalProcessRunner().EnsureEgressAdmissible(new SandboxSpec { Command = "agent" })); [Fact] - public void A_brokered_network_off_spec_is_refused_exactly_where_this_host_confines_but_cannot_seal() + public void A_brokered_network_off_spec_without_its_socket_is_refused_exactly_where_this_host_would_sever_it() { // Honest on either host: unconfined hosts (macOS dev, a pod without userns) admit it untouched, because nothing - // there would have severed it; a confining host refuses it unless it can seal and the broker is reachable. + // there gives the child a network of its own; a confining host refuses it, because nothing could carry it to its broker. var spec = new SandboxSpec { Command = "agent", ModelBrokerPort = 43121 }; - var refusal = BubblewrapSandbox.Available is null ? null : LocalProcessRunner.SealRefusal(FilteredEgressNetns.IsSupported, FilteredEgressNetns.CanSeal, brokerReachableFromNamespace: false); - var thrown = Record.Exception(() => new LocalProcessRunner().EnsureEgressAdmissible(spec, modelBrokerReachableFromNamespace: false)); + var thrown = Record.Exception(() => new LocalProcessRunner().EnsureEgressAdmissible(spec)); - if (refusal is null) thrown.ShouldBeNull(); - else thrown.ShouldBeOfType().Cause.ShouldStartWith(refusal, customMessage: "the cause leads with the wall, then the probe's own account of it"); + if (BubblewrapSandbox.Available is null) thrown.ShouldBeNull(); + else thrown.ShouldBeOfType().Cause.ShouldBe(SealedEgressUnavailableException.CauseBrokerSocketUnavailable); } [Fact] public void The_refusal_is_an_unavailable_failure_that_names_its_cause_and_its_remedy() { - IFailure failure = new SealedEgressUnavailableException(SealedEgressUnavailableException.CauseNoPrivilege); + IFailure failure = new SealedEgressUnavailableException(SealedEgressUnavailableException.CauseRelayMissing); failure.Kind.ShouldBe(FailureKind.Unavailable, "nothing about the launch can change to make it work — only the host can"); - failure.Code.ShouldBe(FailureCodes.SandboxSealedEgressUnavailable); - failure.ClientMessage.ShouldBe("This host cannot give a network-off run a sealed route to its model."); - ((Exception)failure).Message.ShouldContain(SealedEgressUnavailableException.CauseNoPrivilege, customMessage: "the operator must be told which wall it was"); - ((Exception)failure).Message.ShouldContain("a retry on this host helps only once it can build one", customMessage: "a probe failure can be transient, so the remedy must not promise a retry is hopeless"); + failure.Code.ShouldBe(FailureCodes.SandboxSealedEgressUnavailable, "the wire name the supervisor and stored results key on is kept"); + failure.ClientMessage.ShouldBe("This host cannot give this run's sandbox a route to its model."); + ((Exception)failure).Message.ShouldContain(SealedEgressUnavailableException.CauseRelayMissing, customMessage: "the operator must be told which wall it was"); + ((Exception)failure).Message.ShouldContain("CODESPACE_MCP_PROXY_PATH", customMessage: "and where to put the helper"); + ((Exception)failure).Message.ShouldContain("this release", customMessage: "and that an override must name a helper that has the relay"); + ((Exception)failure).Message.ShouldNotContain("CAP_NET_ADMIN", customMessage: "a namespaced run needs no privilege of the worker's any more; the remedy must not send an operator to grant one"); } - [Fact] - public void A_setup_step_that_failed_on_a_host_that_can_seal_is_told_to_fix_that_step() + public void Dispose() + { + try { Directory.Delete(_dir, recursive: true); } catch { /* best-effort */ } + } + + /// The helper each row stands the worker's up with. + public enum Helper { Relay, Missing, PreRelay, SelfContained, NotExecutable, BindsAnyway } + + /// Answers the admission's question the way the relay does: asked to listen on a port something already holds, it exits with the relay's listen-failed status and starts nothing. + private static readonly string RelayScript = $"#!/bin/sh\necho 'codespace-mcp relay: cannot listen on 127.0.0.1:1 (AddressAlreadyInUse); the CLI was not started.' >&2\nexit {ModelBrokerRelay.ListenFailedExitCode}"; + + /// Answers the way a codespace-mcp from before the relay does: its MCP proxy reads any argv as its own and exits with its usage error. + private const string PreRelayScript = "#!/bin/sh\necho 'The MCP proxy requires a socket path in CODESPACE_MCP_SOCKET.' >&2\nexit 2"; + + private string HelperAt(Helper helper) + { + var directory = Directory.CreateDirectory(Path.Combine(_dir, helper.ToString())).FullName; + var path = Path.Combine(directory, "codespace-mcp"); + + switch (helper) + { + case Helper.Relay: WriteExecutable(path, RelayScript); break; + case Helper.PreRelay: WriteExecutable(path, PreRelayScript); break; + case Helper.BindsAnyway: WriteExecutable(path, "#!/bin/sh\nexit 0"); break; + case Helper.NotExecutable: File.WriteAllText(path, "not a program"); break; + case Helper.SelfContained: + WriteExecutable(path, RelayScript); + File.WriteAllText(Path.Combine(directory, "codespace-mcp.runtimeconfig.json"), """{"runtimeOptions":{"tfm":"net10.0","includedFrameworks":[{"name":"Microsoft.NETCore.App","version":"10.0.8"}]}}"""); + break; + } + + return path; + } + + private static void WriteExecutable(string path, string script) { - // The probe proved this worker can build a namespace, so granting it more and waiting for a re-probe fixes - // nothing: the failed step (a route that discards the run's /30, say) fails every launch until it is fixed. - var refusal = SealedEgressUnavailableException.SetupFailed("ip route get 10.1.1.2 from 10.1.1.1 → exit 2: RTNETLINK answers: No route to host"); - - ((IFailure)refusal).Code.ShouldBe(FailureCodes.SandboxSealedEgressUnavailable, "the same wall as every other refusal, so the supervisor steers it the same way"); - refusal.Cause.ShouldBe("the sealed namespace's setup failed: ip route get 10.1.1.2 from 10.1.1.1 → exit 2: RTNETLINK answers: No route to host"); - refusal.Message.ShouldContain("fix what that step names", customMessage: "the remedy points at the failed step"); - refusal.Message.ShouldNotContain("Dockerfile.worker", customMessage: "the worker already has what a sealed namespace needs"); - refusal.Message.ShouldNotContain("once a minute", customMessage: "the setup runs afresh on every launch; no probe is waited for"); + File.WriteAllText(path, script + "\n"); + File.SetUnixFileMode(path, UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute); } } diff --git a/frontend/src/lib/networkPosture.fixture.json b/frontend/src/lib/networkPosture.fixture.json index 36e07489b..9d68f5df5 100644 --- a/frontend/src/lib/networkPosture.fixture.json +++ b/frontend/src/lib/networkPosture.fixture.json @@ -14,7 +14,7 @@ { "effective": "Standard", "ceiling": "Standard", "deployment": "Standard", "line": "Network: clamped off by deployment ceiling (Standard) — severed only where the sandbox confines" }, { "effective": "Trusted", "ceiling": "Trusted", "deployment": "Standard", "line": "Network: on (Trusted)" } ], - "_sealedDoc": "egressSealedToBroker marks a network-off run whose brokered model was reached through a namespace sealed to that broker (no route, no NAT, no DNS, one gateway port). It is severed from everything else, so networkSevered is true too, and the sentence names the one route it kept.", + "_sealedDoc": "egressSealedToBroker marks a network-off run whose brokered model was still reached: through the codespace-mcp relay in its sandbox and the lease's socket bound read-only into it (runs launched before the relay reached it through a namespace sealed to that broker instead). It is severed from everything else, so networkSevered is true too, and the sentence names the one route it kept.", "_credentialDoc": "A confinement record also carries what the launch did about the MODEL CREDENTIAL (modelCredentialBrokered). Absent/null means the run injected none and the sentence says nothing; false means the tenant's own provider key went into the sandbox and the line discloses it (AgentAutonomyPolicy.DirectModelCredentialCaveat), on EVERY posture branch including 'on' — that is the run whose agent can spend the key. true says nothing, because for a brokered run the sentence would be false.", "confinementCases": [ {