Repository navigation
Log the cause when a capture recovery settlement fails #3516
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Backend · E2E | |
| # Three E2E gates, all against a throw-away GUID Postgres. ALL E2E tests live in CodeSpace.E2ETests | |
| # (Category=E2E); the gates split it by the Surface trait because they need DIFFERENT runtimes: | |
| # | |
| # 1. e2e-http (Surface=Http) — the REAL ASP.NET pipeline (routing, [AllowAnonymous], model binding, | |
| # the GlobalExceptionFilter) in-memory via WebApplicationFactory: the actual HTTP status code a | |
| # webhook provider receives. Runs directly on the host runner, so it reaches Postgres at | |
| # localhost:5432 (the committed appsettings.json admin connection). | |
| # | |
| # 2. e2e-engine (Surface=Engine) — THE headline intelligent flow driven top-to-bottom as one engine run: | |
| # trigger.manual → llm.complete(responseSchema) planner → flow.map fan-out whose body is a REAL | |
| # agent.run that ACTUALLY EXECUTES (real AgentRunExecutor → real LocalProcessRunner → a fake | |
| # codex CLI process under REAL bubblewrap confinement → real completion → resume) → a synthesizer | |
| # that reduces the per-branch results. These engine-tier tests reuse the shared Postgres fixture + | |
| # seed infra from CodeSpace.IntegrationTests (referenced as a project). bubblewrap is installed + | |
| # Sandbox__RequireConfinement=true so the agent runs CONFINED (the production posture) and a degraded | |
| # sandbox fails HARD rather than silently running unconfined. It runs in a PRIVILEGED container | |
| # (mirroring sandbox-isolation.yml) so unprivileged user namespaces work despite ubuntu-latest's | |
| # AppArmor restriction — and so reaches Postgres by the service ALIAS (postgres:5432), overridden | |
| # via the TestPostgres__AdminConnectionString env var. | |
| # | |
| # 3. e2e-worker (Surface=Worker) — the PROCESSING role, which no other lane boots: a host with | |
| # HangfireHosting=Worker, its two real Hangfire servers draining a real Postgres queue, every | |
| # IRecurringJob registered by the real WorkerHangfireRegistrar and fired once. It needs no bubblewrap, | |
| # so it could have run inside e2e-http; it gets its own job because it is the only lane that EXECUTES | |
| # jobs. A Worker host holds ControlWorkerCount + ProcessorCount * 2 worker connections for its lifetime | |
| # (38 measured on a 12-core box — see FactoryHangfireRoleTests) and its ticks run the real sweeps, | |
| # filesystem reapers included. Its own Postgres service keeps that load and that blast radius off the | |
| # HTTP lane's database server, and its own check line keeps "do the recurring jobs still fire" legible | |
| # in the checks list rather than buried in a 159-test lane. In-process interference is handled | |
| # separately, by the assembly-wide DisableTestParallelization in AssemblyParallelization.cs. | |
| on: | |
| push: | |
| branches: [main] | |
| paths: | |
| - 'backend/**' | |
| - '.github/workflows/backend-e2e.yml' | |
| pull_request: | |
| paths: | |
| - 'backend/**' | |
| - '.github/workflows/backend-e2e.yml' | |
| workflow_dispatch: | |
| concurrency: | |
| group: backend-e2e-${{ github.ref }} | |
| cancel-in-progress: true | |
| permissions: | |
| contents: read | |
| env: | |
| DOTNET_NOLOGO: 'true' | |
| # The DI container fail-fasts without a Variables master key outside Development (the settings arc's | |
| # guard). CI hosts get the SAME committed local-dev test key docker-compose.yml already carries — | |
| # a committed value, not a toggle; production deployments still supply their own real key. | |
| CODESPACE_VARIABLE_MASTER_KEY: 'bG9jYWwtZGV2LW9ubHkta2V5LW5vdC1mb3ItcHJvZCE=' | |
| DOTNET_CLI_TELEMETRY_OPTOUT: 'true' | |
| DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 'true' | |
| jobs: | |
| e2e-http: | |
| name: dotnet test (E2ETests · HTTP · Postgres) | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| services: | |
| postgres: | |
| image: postgres:18-alpine | |
| env: | |
| POSTGRES_USER: postgres | |
| POSTGRES_PASSWORD: '123456' | |
| POSTGRES_DB: postgres | |
| ports: | |
| - 5432:5432 | |
| options: >- | |
| --health-cmd "pg_isready -U postgres -d postgres" | |
| --health-interval 5s | |
| --health-timeout 3s | |
| --health-retries 20 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up .NET 10 | |
| uses: actions/setup-dotnet@v4 | |
| with: | |
| dotnet-version: '10.0.x' | |
| - name: Cache NuGet packages | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.nuget/packages | |
| key: nuget-e2e-${{ runner.os }}-${{ hashFiles('backend/**/*.csproj') }} | |
| restore-keys: nuget-e2e-${{ runner.os }}- | |
| - name: Restore | |
| run: dotnet restore backend/tests/CodeSpace.E2ETests/CodeSpace.E2ETests.csproj | |
| - name: Test (Surface=Http — the REAL ASP.NET pipeline via WebApplicationFactory) | |
| run: > | |
| dotnet test backend/tests/CodeSpace.E2ETests/CodeSpace.E2ETests.csproj | |
| --no-restore | |
| --filter "Category=E2E&Surface=Http" | |
| --logger "console;verbosity=normal" | |
| --logger "trx;LogFileName=e2e-http.trx" | |
| --results-directory backend/TestResults | |
| --blame-hang-timeout 5m | |
| - name: Assert the HTTP E2E actually ran (no silent skip) | |
| # `dotnet test --filter` EXITS 0 even when the filter matches ZERO tests, so a Surface-trait regression | |
| # could green this gate while exercising nothing. Require at least one HTTP E2E to have executed. | |
| run: | | |
| trx=backend/TestResults/e2e-http.trx | |
| test -f "$trx" || { echo "::error::trx not found at $trx — the test step produced no results"; exit 1; } | |
| executed=$(grep -oE 'executed="[0-9]+"' "$trx" | head -1 | grep -oE '[0-9]+') | |
| echo "executed=${executed:-0}" | |
| if [ "${executed:-0}" -lt 1 ]; then | |
| echo "::error::Expected Category=E2E&Surface=Http tests to run, but ${executed:-0} executed — the filter matched nothing (trait regression?)." | |
| exit 1 | |
| fi | |
| - name: Upload test results | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: backend-e2e-http-trx | |
| path: backend/TestResults/*.trx | |
| if-no-files-found: ignore | |
| e2e-engine: | |
| name: headline flow (planner → map → real agents → synthesizer) | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| # Runs in a PRIVILEGED container so unprivileged user namespaces work regardless of the host runner's | |
| # AppArmor policy — current ubuntu-latest (Ubuntu 24.04) ships kernel.apparmor_restrict_unprivileged_userns=1, | |
| # which denies unprivileged userns for an apt-installed bwrap. The bwrap-confined agent in this E2E needs | |
| # working userns, so we mirror the already-green sibling sandbox-isolation.yml (`docker run --privileged | |
| # dotnet/sdk:10.0 + apt install bubblewrap`) — the exact environment these tests were validated against. | |
| container: | |
| image: mcr.microsoft.com/dotnet/sdk:10.0 | |
| options: --privileged | |
| env: | |
| # Fail-closed: the headline E2E runs a REAL agent.run branch through the real LocalProcessRunner, | |
| # which confines it under bubblewrap. With REQUIRE_SANDBOX=1 a host that can't sandbox (no bwrap / | |
| # no unprivileged userns) makes the run fail HARD instead of silently executing the agent unconfined | |
| # — so green here means the agent really ran AND was really confined. | |
| Sandbox__RequireConfinement: 'true' | |
| # Inside a job container the Postgres SERVICE container is reachable by its network ALIAS (`postgres`), | |
| # NOT via the host port mapping (`localhost:5432` is the job container itself). Override the test | |
| # admin connection string (TestPostgres:AdminConnectionString → env double-underscore form) to target | |
| # the service alias. The e2e-http job stays on `localhost` because it runs directly on the host runner. | |
| TestPostgres__AdminConnectionString: 'Host=postgres;Port=5432;Username=postgres;Password=123456;Database=postgres' | |
| services: | |
| postgres: | |
| image: postgres:18-alpine | |
| env: | |
| POSTGRES_USER: postgres | |
| POSTGRES_PASSWORD: '123456' | |
| POSTGRES_DB: postgres | |
| # No `ports:` mapping — inside a container job the service is reached by its alias on the Docker | |
| # network, so the host-port publish is unnecessary. | |
| options: >- | |
| --health-cmd "pg_isready -U postgres -d postgres" | |
| --health-interval 5s | |
| --health-timeout 3s | |
| --health-retries 20 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install bubblewrap + util-linux (prlimit) + iproute2/nftables | |
| # The headline E2E spawns a REAL agent process through the production runner, which wraps it in | |
| # bubblewrap + prlimit (the production confinement posture). Running as root in the SDK container, so | |
| # no sudo. ca-certificates so the container can fetch packages; util-linux for prlimit; iproute2 and | |
| # nftables because the worker image ships them for allowlist runs. A network-off brokered agent needs | |
| # neither: it reaches its broker through the codespace-mcp relay, which the test project's build places | |
| # beside its assembly, and without which it is refused as sandbox_sealed_egress_unavailable. | |
| run: | | |
| apt-get update | |
| apt-get install -y --no-install-recommends bubblewrap util-linux ca-certificates iproute2 nftables | |
| echo "bwrap: $(bwrap --version)" | |
| echo "prlimit: $(prlimit --version)" | |
| - name: Probe — unprivileged user namespaces (informational) | |
| # Informational only: the authoritative gate is the test step below (REQUIRE_SANDBOX=1 → the run fails | |
| # HARD if confinement is unavailable, surfacing as the headline assertion failing). This just surfaces | |
| # a clear "userns works/doesn't" signal early without ever failing the job on a bwrap-flag quirk — | |
| # turning an opaque "run was Failed" into a diagnosable cause. | |
| run: | | |
| if bwrap --ro-bind / / --dev /dev --proc /proc --unshare-all --uid 0 true; then | |
| echo "✓ unprivileged user namespaces work" | |
| else | |
| echo "⚠ userns probe failed — the REQUIRE_SANDBOX=1 test step is authoritative and will fail hard if confinement is truly unavailable" | |
| fi | |
| - name: Cache NuGet packages | |
| uses: actions/cache@v4 | |
| with: | |
| path: /root/.nuget/packages | |
| key: nuget-e2e-engine-${{ hashFiles('backend/**/*.csproj') }} | |
| restore-keys: nuget-e2e-engine- | |
| - name: Restore | |
| run: dotnet restore backend/tests/CodeSpace.E2ETests/CodeSpace.E2ETests.csproj | |
| - name: Test (Surface=Engine — the headline planner→map→real-agents→synthesizer flow) | |
| run: > | |
| dotnet test backend/tests/CodeSpace.E2ETests/CodeSpace.E2ETests.csproj | |
| --no-restore | |
| --filter "Category=E2E&Surface=Engine" | |
| --logger "console;verbosity=normal" | |
| --logger "trx;LogFileName=e2e-engine.trx" | |
| --results-directory backend/TestResults | |
| --blame-hang-timeout 5m | |
| - name: Assert the headline E2E actually ran (no silent skip) | |
| # `dotnet test --filter` EXITS 0 even when the filter matches ZERO tests, so a trait regression | |
| # could make this gate pass green while exercising nothing. Read the trx counters and require at | |
| # least the one headline-flow test to have executed (NOT skipped — the OS guard would skip it on a | |
| # non-Linux host, but this gate IS Linux, so a skip here means the real run was silently dodged). | |
| run: | | |
| trx=backend/TestResults/e2e-engine.trx | |
| test -f "$trx" || { echo "::error::trx not found at $trx — the test step produced no results"; exit 1; } | |
| executed=$(grep -oE 'executed="[0-9]+"' "$trx" | head -1 | grep -oE '[0-9]+') | |
| passed=$(grep -oE 'passed="[0-9]+"' "$trx" | head -1 | grep -oE '[0-9]+') | |
| echo "executed=${executed:-0} passed=${passed:-0}" | |
| if [ "${executed:-0}" -lt 1 ]; then | |
| echo "::error::Expected the Category=E2E&Surface=Engine headline-flow tests to run, but ${executed:-0} executed — the filter matched nothing (trait regression?) or it was silently skipped." | |
| exit 1 | |
| fi | |
| - name: Upload test results | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: backend-e2e-engine-trx | |
| path: backend/TestResults/*.trx | |
| if-no-files-found: ignore | |
| e2e-worker: | |
| name: recurring jobs fire (worker host · Postgres) | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| services: | |
| postgres: | |
| image: postgres:18-alpine | |
| env: | |
| POSTGRES_USER: postgres | |
| POSTGRES_PASSWORD: '123456' | |
| POSTGRES_DB: postgres | |
| ports: | |
| - 5432:5432 | |
| options: >- | |
| --health-cmd "pg_isready -U postgres -d postgres" | |
| --health-interval 5s | |
| --health-timeout 3s | |
| --health-retries 20 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up .NET 10 | |
| uses: actions/setup-dotnet@v4 | |
| with: | |
| dotnet-version: '10.0.x' | |
| - name: Cache NuGet packages | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.nuget/packages | |
| key: nuget-e2e-${{ runner.os }}-${{ hashFiles('backend/**/*.csproj') }} | |
| restore-keys: nuget-e2e-${{ runner.os }}- | |
| - name: Restore | |
| run: dotnet restore backend/tests/CodeSpace.E2ETests/CodeSpace.E2ETests.csproj | |
| - name: Test (Surface=Worker — every recurring job registered and fired by the real worker host) | |
| run: > | |
| dotnet test backend/tests/CodeSpace.E2ETests/CodeSpace.E2ETests.csproj | |
| --no-restore | |
| --filter "Category=E2E&Surface=Worker" | |
| --logger "console;verbosity=normal" | |
| --logger "trx;LogFileName=e2e-worker.trx" | |
| --results-directory backend/TestResults | |
| --blame-hang-timeout 10m | |
| - name: Assert the worker E2E actually ran (no silent skip) | |
| # `dotnet test --filter` EXITS 0 even when the filter matches ZERO tests, so a Surface-trait regression could | |
| # green this gate while firing nothing — which is the exact condition this gate was added to end. | |
| run: | | |
| trx=backend/TestResults/e2e-worker.trx | |
| test -f "$trx" || { echo "::error::trx not found at $trx — the test step produced no results"; exit 1; } | |
| executed=$(grep -oE 'executed="[0-9]+"' "$trx" | head -1 | grep -oE '[0-9]+') | |
| echo "executed=${executed:-0}" | |
| if [ "${executed:-0}" -lt 1 ]; then | |
| echo "::error::Expected Category=E2E&Surface=Worker tests to run, but ${executed:-0} executed — the filter matched nothing (trait regression?)." | |
| exit 1 | |
| fi | |
| - name: Upload test results | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: backend-e2e-worker-trx | |
| path: backend/TestResults/*.trx | |
| if-no-files-found: ignore |