Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 55 additions & 12 deletions .github/workflows/sandbox-isolation.yml
Original file line number Diff line number Diff line change
Expand Up @@ -224,8 +224,8 @@ jobs:
executed=$(grep -oE 'executed="[0-9]+"' "$trx" | head -1 | grep -oE '[0-9]+')
passed=$(grep -oE 'passed="[0-9]+"' "$trx" | head -1 | grep -oE '[0-9]+')
echo "executed=${executed:-0} passed=${passed:-0}"
if [ "${executed:-0}" -lt 82 ]; then
echo "::error::Expected >=82 sandbox isolation tests to run (bwrap/prlimit confinement + cap-drop + cgroup-namespace re-root + egress-allowlist filter + cgroup resource cap + durable-launch cgroup wiring + argv/envp per-string kernel ceiling + a prompt past it riding stdin + a read-only workspace mount + a network-off run reaching its broker through the relay and nothing else + an allowlist run relayed to its broker + a read-only reviewer reading its diff with the real CLIs + a bwrap probe that runs the launch argv + the MCP helper bound file by file behind a read-only socket dir + a CLI reaching its broker socket through the relay + a severed child reaching its broker over the lease socket across a worker restart + a pre-relay namespaced run re-bound at its gateway and torn down with its seal + an allowlist run's veth guarded both ways, a flow the worker opened before the run included), but only ${executed:-0} did — the Category=Sandbox filter matched too few (trait regression?). If a case was deliberately removed, lower this number in the same PR."
if [ "${executed:-0}" -lt 83 ]; then
echo "::error::Expected >=83 sandbox isolation tests to run (bwrap/prlimit confinement + cap-drop + cgroup-namespace re-root + egress-allowlist filter + cgroup resource cap + durable-launch cgroup wiring + argv/envp per-string kernel ceiling + a prompt past it riding stdin + a read-only workspace mount + a network-off run reaching its broker through the relay and nothing else + an allowlist run relayed to its broker + a read-only reviewer reading its diff with the real CLIs + a bwrap probe that runs the launch argv + the MCP helper bound file by file behind a read-only socket dir + a CLI reaching its broker socket through the relay + a severed child reaching its broker over the lease socket across a worker restart + a pre-relay namespaced run re-bound at its gateway and torn down with its seal + an allowlist run's veth guarded both ways, a flow the worker opened before the run included + forwarding a root worker may not write named before an allowlist is planned), but only ${executed:-0} did — the Category=Sandbox filter matched too few (trait regression?). If a case was deliberately removed, lower this number in the same PR."
exit 1
fi

Expand Down Expand Up @@ -265,14 +265,15 @@ jobs:
assert len(cases) == rows and all(r.get('outcome') == 'Passed' for r in cases), f'{method}: all {rows} case(s) must pass'
print('All 6 sealed-egress arms ran and passed.')

# The allowlist plan's host-routing arms, the guard on its veth, and the relayed allowlist launch return early
# without ip and nft; require their markers. The IPv6 arm prints its marker only where the veth has a link-local.
for marker in ('[filtered-egress-e2e] ran restart-reissue', '[filtered-egress-e2e] ran policy-route-discard-dst', '[filtered-egress-e2e] ran worker-shut', '[filtered-egress-e2e] ran ipv6-link-local', '[filtered-egress-e2e] ran shadowed-peer-refused', '[filtered-egress-e2e] ran established-flow-refused', '[filtered-egress-e2e] ran pmtu-upload', '[filtered-egress-e2e] ran stale-guard-replaced', '[durable-egress-e2e] ran allowlist-relay'):
# The allowlist plan's host-routing arms, the guard on its veth, the relayed allowlist launch, and the forwarding
# probe's read-only arm return early without ip and nft, or off root; require their markers. The IPv6 arm prints
# its marker only where the veth has a link-local.
for marker in ('[filtered-egress-e2e] ran restart-reissue', '[filtered-egress-e2e] ran policy-route-discard-dst', '[filtered-egress-e2e] ran worker-shut', '[filtered-egress-e2e] ran ipv6-link-local', '[filtered-egress-e2e] ran shadowed-peer-refused', '[filtered-egress-e2e] ran established-flow-refused', '[filtered-egress-e2e] ran pmtu-upload', '[filtered-egress-e2e] ran stale-guard-replaced', '[filtered-egress-e2e] ran forwarding-read-only', '[durable-egress-e2e] ran allowlist-relay'):
assert marker in text, f'"{marker}" is missing — this lane is root with ip and nft, so the allowlist plan must run'
for method in ('FilteredEgressNetnsE2ETests.A_30_still_held_by_a_run_that_outlived_its_worker_is_not_handed_to_the_next_run', 'FilteredEgressNetnsE2ETests.A_host_whose_policy_rule_discards_the_run_s_replies_fails_the_setup_and_leaks_nothing', 'FilteredEgressNetnsE2ETests.An_allowlist_run_reaches_neither_the_worker_s_gateway_nor_its_address_while_dns_on_the_worker_and_the_allowlist_still_answer', 'FilteredEgressNetnsE2ETests.An_allowlist_run_has_no_path_to_the_worker_over_the_veth_s_ipv6_link_local_either', 'FilteredEgressNetnsE2ETests.A_peer_the_worker_reaches_at_the_run_s_address_is_refused_and_the_sandbox_receives_nothing', 'FilteredEgressNetnsE2ETests.A_flow_the_worker_opened_to_the_shadowed_peer_before_the_run_is_neither_handed_to_the_sandbox_nor_answered_from_it', 'FilteredEgressNetnsE2ETests.An_upload_across_a_narrower_uplink_completes_because_the_worker_s_frag_needed_reaches_the_run', 'FilteredEgressNetnsE2ETests.A_guard_an_earlier_teardown_left_behind_is_replaced_not_added_to', 'DurableLaunchEgressE2ETests.An_allowlist_run_reaches_its_broker_through_the_relay_and_its_allowlist_still_holds'):
for method in ('FilteredEgressNetnsE2ETests.A_30_still_held_by_a_run_that_outlived_its_worker_is_not_handed_to_the_next_run', 'FilteredEgressNetnsE2ETests.A_host_whose_policy_rule_discards_the_run_s_replies_fails_the_setup_and_leaks_nothing', 'FilteredEgressNetnsE2ETests.An_allowlist_run_reaches_neither_the_worker_s_gateway_nor_its_address_while_dns_on_the_worker_and_the_allowlist_still_answer', 'FilteredEgressNetnsE2ETests.An_allowlist_run_has_no_path_to_the_worker_over_the_veth_s_ipv6_link_local_either', 'FilteredEgressNetnsE2ETests.A_peer_the_worker_reaches_at_the_run_s_address_is_refused_and_the_sandbox_receives_nothing', 'FilteredEgressNetnsE2ETests.A_flow_the_worker_opened_to_the_shadowed_peer_before_the_run_is_neither_handed_to_the_sandbox_nor_answered_from_it', 'FilteredEgressNetnsE2ETests.An_upload_across_a_narrower_uplink_completes_because_the_worker_s_frag_needed_reaches_the_run', 'FilteredEgressNetnsE2ETests.A_guard_an_earlier_teardown_left_behind_is_replaced_not_added_to', 'FilteredEgressNetnsE2ETests.Forwarding_a_root_worker_may_not_write_is_named_before_an_allowlist_is_planned_on_it', 'DurableLaunchEgressE2ETests.An_allowlist_run_reaches_its_broker_through_the_relay_and_its_allowlist_still_holds'):
cases = [r for r in results if method in r.get('testName', '')]
assert len(cases) == 1 and cases[0].get('outcome') == 'Passed', f'{method}: must pass'
print('All 9 allowlist-plan arms ran and passed.')
print('All 10 allowlist-plan arms ran and passed.')

# The broker's socket, relay-revoke and legacy-gateway arms return early without bwrap or ip/nft; require each
# marker, the legacy survivor's teardown of the seal it carried included.
Expand Down Expand Up @@ -327,9 +328,10 @@ jobs:

- name: Test the non-root worker posture (uid 1654, no capabilities)
# The shipped worker runs as uid 1654 with no capabilities and may not build a network namespace of its own,
# and every step above runs as root. Category=SandboxNonRoot runs the relay's arms again as that uid; each one
# first asserts geteuid() != 0, that bubblewrap confines, and that no namespace can be built, so the lane cannot
# pass as root or on a host that could fall back to a veth namespace. RequireConfinement stays set. The runner's
# and every step above runs as root. Category=SandboxNonRoot runs the relay's arms again as that uid, and an
# allowlist run it cannot filter, which is severed and still relayed; each one first asserts geteuid() != 0,
# that bubblewrap confines, and that no namespace can be built, so the lane cannot pass as root or on a host
# that could fall back to a veth namespace. RequireConfinement stays set. The runner's
# kernel restricts unprivileged user namespaces through AppArmor where it has that switch; lifting it is the
# same node setting operators give the worker. The root arms above leave their short-path socket roots behind
# owner-only, which uid 1654 could not enter, so those go first.
Expand Down Expand Up @@ -366,13 +368,53 @@ jobs:
root = ET.parse(path).getroot()
counters = root.find('.//{*}Counters')
executed, passed = int(counters.get('executed')), int(counters.get('passed'))
assert executed >= 9 and passed == executed, f'expected all 9 non-root arms to run and pass, got executed={executed} passed={passed}'
assert executed >= 10 and passed == executed, f'expected all 10 non-root arms to run and pass, got executed={executed} passed={passed}'
text = open(path, encoding='utf-8').read()
for marker in ('[non-root-e2e] ran admission uid=1654', '[sealed-egress-e2e] ran non-root durable uid=1654', '[sealed-egress-e2e] ran non-root non-durable uid=1654', '[sealed-egress-e2e] ran non-root restart uid=1654', '[sealed-egress-e2e] ran non-root relay-refused uid=1654', '[sealed-egress-e2e] ran non-root relay-ipv6', '[broker-socket-e2e] ran non-root socket-channel uid=1654', '[review-diff-e2e] ran non-root network-off-relayed claude-code uid=1654', '[review-diff-e2e] ran non-root network-off-relayed codex-cli uid=1654'):
for marker in ('[non-root-e2e] ran admission uid=1654', '[sealed-egress-e2e] ran non-root durable uid=1654', '[sealed-egress-e2e] ran non-root non-durable uid=1654', '[sealed-egress-e2e] ran non-root restart uid=1654', '[sealed-egress-e2e] ran non-root relay-refused uid=1654', '[sealed-egress-e2e] ran non-root relay-ipv6', '[broker-socket-e2e] ran non-root socket-channel uid=1654', '[review-diff-e2e] ran non-root network-off-relayed claude-code uid=1654', '[review-diff-e2e] ran non-root network-off-relayed codex-cli uid=1654', '[durable-egress-e2e] ran non-root allowlist-severed uid=1654'):
assert marker in text, f'non-root arm marker "{marker}" is missing — the arm returned early or did not run as the worker uid'
print(f'All {executed} non-root arms ran as uid 1654 and passed.')
PY

- name: Test the unconfined worker posture (uid 1654, nothing confining)
# docker-compose.yml ships the worker image with none of the grants bubblewrap needs: uid 1654 with ip and nft
# installed, nothing confining, RequireConfinement off. A confining worker severs an allowlist it cannot filter;
# here nothing would enforce that, so Category=SandboxUnconfined pins that such a run is still planned into its
# namespace and aborted at the setup, never launched on the worker's network. It runs as that uid with
# CODESPACE_BWRAP_PATH naming no binary and RequireConfinement unset — the one lane that must not confine —
# and each arm first asserts exactly that posture. Reuses the home, packages and build output the non-root step
# made readable to that uid.
shell: bash
run: |
set -euo pipefail
results=backend/TestResults/unconfined
mkdir -p "$results"
chown 1654:1654 "$results"
packages=$(dotnet nuget locals global-packages --list | sed -E 's/^global-packages: *//; s:/+$::')
test -d "$packages"
setpriv --reuid 1654 --regid 1654 --clear-groups --inh-caps=-all --bounding-set=-all --no-new-privs \
env -u Sandbox__RequireConfinement HOME=/tmp/nonroot-home DOTNET_CLI_HOME=/tmp/nonroot-home NUGET_PACKAGES="$packages" CODESPACE_BWRAP_PATH=/nonexistent/bwrap \
dotnet test backend/tests/CodeSpace.SandboxTests/CodeSpace.SandboxTests.csproj \
--no-build \
--filter "Category=SandboxUnconfined" \
--logger "console;verbosity=detailed" \
--logger "trx;LogFileName=sandbox-unconfined.trx" \
--results-directory "$results"

- name: Assert the unconfined arms actually ran as uid 1654
run: |
python3 - <<'PY'
import xml.etree.ElementTree as ET
path = 'backend/TestResults/unconfined/sandbox-unconfined.trx'
root = ET.parse(path).getroot()
counters = root.find('.//{*}Counters')
executed, passed = int(counters.get('executed')), int(counters.get('passed'))
assert executed >= 3 and passed == executed, f'expected all 3 unconfined arms to run and pass, got executed={executed} passed={passed}'
text = open(path, encoding='utf-8').read()
for marker in ('[unconfined-e2e] ran allowlist-never-unfiltered durable uid=1654', '[unconfined-e2e] ran allowlist-never-unfiltered one-shot uid=1654', '[unconfined-e2e] ran admission uid=1654'):
assert marker in text, f'unconfined arm marker "{marker}" is missing — the arm returned early or did not run as the worker uid'
print(f'All {executed} unconfined arms ran as uid 1654 and passed.')
PY

- name: Upload test results
if: always()
uses: actions/upload-artifact@v4
Expand All @@ -381,4 +423,5 @@ jobs:
path: |
backend/TestResults/*.trx
backend/TestResults/nonroot/*.trx
backend/TestResults/unconfined/*.trx
if-no-files-found: ignore
26 changes: 16 additions & 10 deletions backend/Dockerfile.worker
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,8 @@
# can still repoint CODESPACE_CODEX_CLI_PATH / CODESPACE_CLAUDE_CODE_PATH at a different binary.
#
# CONFINEMENT POSTURE. The worker logs one "Sandbox posture:" line at boot — whether bubblewrap confines and why not,
# whether the codespace-mcp helper is present, whether the namespace probe (FilteredEgressNetns.CanSeal) holds and why
# not. Read it before the first run; what each tier needs from the deployment is below.
# whether the codespace-mcp helper is present, whether the allowlist filter probe (FilteredEgressNetns.CanFilter) holds
# and why not. Read it before the first run; what each tier needs from the deployment is below.
#
# BUBBLEWRAP (every tier) needs no root and no capability: the non-root user below confines through UNPRIVILEGED user
# namespaces, which a container runtime's defaults deny. Grant them, without --privileged or --cap-add:
Expand Down Expand Up @@ -74,14 +74,20 @@
# (sandbox_sealed_egress_unavailable): so does one whose CODESPACE_MCP_PROXY_PATH names a build from before the relay
# or a self-contained publish. An allowlist run reaches its broker through the same relay.
#
# EGRESS FILTERING (an allowlist run) needs root + CAP_NET_ADMIN + CAP_SYS_ADMIN + a writable net.ipv4.ip_forward on
# top of these packages: FilteredEgressPlan runs `ip netns add` (CAP_SYS_ADMIN: it unshares a network namespace and
# bind-mounts it), builds a veth and an nftables ruleset, and `sysctl -w net.ipv4.ip_forward=1`. With `ip` or `nft`
# missing, FilteredEgressNetns.IsSupported is false and SandboxEgressPolicy.Derive turns the allowlist into Denied (NO
# network at all), never into Full. With them installed but the privilege missing — this image as shipped, since the
# user below is non-root — that probe still passes, so the run is planned Filtered and its launch aborts when setup is
# refused ("Filtered-egress netns setup failed"): it is never launched unfiltered, but it is not quietly Denied either.
# Grant the privilege on any deployment that relies on allowlisted egress.
# EGRESS FILTERING (an allowlist run) needs root + CAP_NET_ADMIN + CAP_SYS_ADMIN, and net.ipv4.ip_forward already 1 or
# writable, on top of these packages: FilteredEgressPlan runs `ip netns add` (CAP_SYS_ADMIN: it unshares a network
# namespace and bind-mounts it), builds a veth and an nftables ruleset, and `sysctl -w net.ipv4.ip_forward=1`, which
# only warns where /proc/sys is read-only or the write is denied, so forwarding that reads 0 there stays off. Where
# bubblewrap confines, the worker proves all of it before it plans a run Filtered (FilteredEgressNetns.CanFilter: the
# binaries, one throwaway namespace, and that sysctl), and the boot line names the step that failed. Where any of it is
# missing — this image as shipped with the grants above, since the user below is non-root —
# SandboxEgressPolicy.Derive turns the allowlist into Denied (NO network at all), never into Full: bubblewrap severs
# the run, its record says NetworkSevered, and a brokered one still reaches its model through the relay above. Where
# bubblewrap does not confine (none of the grants above, as docker-compose.yml ships it), nothing would enforce
# Denied, so the run is still planned Filtered on the binaries alone: its setup filters it, or is refused and aborts
# the launch ("Filtered-egress netns setup failed") — it is never launched on the worker's network, and
# Sandbox__RequireConfinement refuses such a host outright. Grant the privilege on any deployment that relies on
# allowlisted egress.
#
# global.json FLOORS the SDK feature band (a floor, not a full pin, against the floating sdk:10.0 tag). Multi-arch
# base-image digest pinning is a deferred follow-up — it needs the manifest-LIST digest + a Renovate bump (a
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,15 @@ namespace CodeSpace.Core.Services.Agents.Sandbox.Isolation;

/// <summary>
/// Keeps the answer to a question this process can only settle by trying — are <c>ip</c> and <c>nft</c> runnable
/// (<see cref="FilteredEgressNetns.IsSupported"/>), can it build a sealed namespace (<see cref="FilteredEgressNetns.CanSeal"/>).
/// (<see cref="FilteredEgressNetns.IsSupported"/>), can it filter an allowlist run (<see cref="FilteredEgressNetns.CanFilter"/>).
/// A proof holds for the process. A failure may be transient — a fork that failed once, a slow first mount of
/// <c>/run/netns</c>, rtnl held by a burst of teardowns — so it stands for one retry interval and is then probed
/// again, with its reason kept for whoever reports what could not be done.
///
/// <para>The FIRST probe is waited for by every caller: a fresh worker that may well seal must not refuse its first
/// launches while it finds out. A RE-probe is made by one caller outside the lock while the others take the standing
/// failure — which they would have got anyway — instead of queueing behind a probe that can take tens of seconds on a
/// host that keeps failing slowly. Timed on a monotonic clock, so a wall-clock step cannot hold a failure forever or
/// <para>The FIRST probe is waited for by every caller: a fresh worker that may well filter must not sever its first
/// allowlist runs while it finds out. A RE-probe is made by one caller outside the lock while the others take the
/// standing failure — which they would have got anyway — instead of queueing behind a probe that can take tens of
/// seconds on a host that keeps failing slowly. Timed on a monotonic clock, so a wall-clock step cannot hold a failure forever or
/// re-probe on every call.</para>
/// </summary>
internal sealed class CapabilityProbe(Func<string?> probe, Func<TimeSpan> monotonicNow, TimeSpan retryInterval)
Expand Down
Loading
Loading