Add a deploy compose E2E that runs an agent through the real images

.github/workflows/deploy-e2e.yml

@@ -0,0 +1,60 @@
+name: Deploy · Compose E2E
+
+# Proves the DEPLOYABLE ARTIFACTS, not just their logic: builds the REAL api + worker images and boots the real
+# production topology (Postgres + lean API with processing OFF + worker with processing ON), then drives a launch
+# over HTTP and asserts the WORKER runs an agent through its own image to a terminal Success — with the harness
+# pointed at a fake codex, so NO model credential or network is needed. Also asserts both pods' anonymous health
+# probes and that the API image carries zero agent-execution machinery. This is the one lane that exercises the
+# images + the API↔worker split end to end; everything else tests the logic in-process on the runner.
+#
+# Confinement is intentionally NOT exercised here (the container won't grant unprivileged userns; CODESPACE_REQUIRE_SANDBOX
+# is left unset → the run is unconfined) — real bubblewrap confinement is the Sandbox · Isolation lane's job.
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'backend/Dockerfile.api'
+      - 'backend/Dockerfile.worker'
+      - 'backend/global.json'
+      - 'backend/.dockerignore'
+      - 'backend/deploy/**'
+      - 'backend/src/CodeSpace.Api/Program.cs'
+      - 'backend/src/CodeSpace.Api/Startup.cs'
+      - 'backend/src/CodeSpace.Api/Extensions/Hangfire/**'
+      - '.github/workflows/deploy-e2e.yml'
+  pull_request:
+    paths:
+      - 'backend/Dockerfile.api'
+      - 'backend/Dockerfile.worker'
+      - 'backend/global.json'
+      - 'backend/.dockerignore'
+      - 'backend/deploy/**'
+      - 'backend/src/CodeSpace.Api/Program.cs'
+      - 'backend/src/CodeSpace.Api/Startup.cs'
+      - 'backend/src/CodeSpace.Api/Extensions/Hangfire/**'
+      - '.github/workflows/deploy-e2e.yml'
+  workflow_dispatch:
+
+concurrency:
+  group: deploy-e2e-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  compose-e2e:
+    name: build images → launch → worker runs an agent → Success
+    runs-on: ubuntu-latest
+    timeout-minutes: 25
+    steps:
+      - uses: actions/checkout@v4
+
+      # node mints the test JWT (mint-jwt.js); docker + docker compose v2 ship on ubuntu-latest.
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+
+      - name: Compose E2E (real images, fake CLI, no creds)
+        run: backend/deploy/e2e/run.sh

backend/deploy/e2e/docker-compose.e2e.yml

@@ -0,0 +1,61 @@
+# Deploy E2E stack — the REAL production topology, built from the REAL images. Postgres + the lean API
+# (processing OFF, enqueues only) + the worker (processing ON, executes agents). run.sh drives a launch over HTTP
+# and asserts the WORKER runs the agent through its image to a terminal state. Build context is backend/ (../..).
+name: codespace-deploy-e2e
+
+services:
+  postgres:
+    image: postgres:18-alpine
+    environment:
+      POSTGRES_USER: codespace
+      POSTGRES_PASSWORD: codespace
+      POSTGRES_DB: codespace
+      POSTGRES_HOST_AUTH_METHOD: scram-sha-256
+      PGDATA: /var/lib/postgresql/data/pgdata
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U codespace -d codespace"]
+      interval: 3s
+      timeout: 3s
+      retries: 30
+
+  # PUBLIC API — serves HTTP + enqueues, processes NO jobs. Built from the lean Dockerfile.api.
+  api:
+    build:
+      context: ../..
+      dockerfile: Dockerfile.api
+    depends_on:
+      postgres:
+        condition: service_healthy
+    environment:
+      ASPNETCORE_ENVIRONMENT: Production
+      ASPNETCORE_URLS: http://+:8080
+      CodeSpaceStore__ConnectionString: "Host=postgres;Port=5432;Database=codespace;Username=codespace;Password=codespace"
+      Authentication__Jwt__SymmetricKey: "${E2E_JWT_KEY}"
+      CODESPACE_VARIABLE_MASTER_KEY: "${E2E_VARIABLE_MASTER_KEY}"
+      CODESPACE_HANGFIRE_PROCESSING_ENABLED: "false"
+    ports:
+      - "18080:8080"
+
+  # WORKER — the local sandbox runner. Built from Dockerfile.worker. The harness CLI is pointed at the mounted
+  # fake-codex so the agent run completes with NO model credential / network. REQUIRE_SANDBOX is left unset, so on a
+  # host without unprivileged userns the run is unconfined (confinement is proven by the privileged sandbox lane).
+  worker:
+    build:
+      context: ../..
+      dockerfile: Dockerfile.worker
+    depends_on:
+      postgres:
+        condition: service_healthy
+    environment:
+      ASPNETCORE_ENVIRONMENT: Production
+      ASPNETCORE_URLS: http://+:8080
+      CodeSpaceStore__ConnectionString: "Host=postgres;Port=5432;Database=codespace;Username=codespace;Password=codespace"
+      Authentication__Jwt__SymmetricKey: "${E2E_JWT_KEY}"
+      CODESPACE_VARIABLE_MASTER_KEY: "${E2E_VARIABLE_MASTER_KEY}"
+      CODESPACE_HANGFIRE_PROCESSING_ENABLED: "true"
+      CODESPACE_AGENT_MCP_ENDPOINT_ENABLED: "true"
+      CODESPACE_CODEX_CLI_PATH: "/opt/fake-codex"
+    volumes:
+      - ./fake-codex:/opt/fake-codex:ro
+    ports:
+      - "18081:8080"

backend/deploy/e2e/fake-codex

@@ -0,0 +1,13 @@
+#!/bin/sh
+# Fake codex CLI for the deploy compose E2E — the worker's CodexHarness is pointed here via
+# CODESPACE_CODEX_CLI_PATH, so NO model credential or network is needed to prove the agent-run path through the
+# real worker image. It is the static twin of tests/.../FakeCodexCli.cs: take the LAST positional arg as the goal
+# (codex puts the prompt last), JSON-escape it, and print a three-line `codex exec --json`-shaped stream whose
+# final agent_message is "DONE: <goal>" so the real executor's ParseEvent/BuildResult fold a Succeeded run.
+goal=""
+for goal in "$@"; do :; done
+esc=$(printf '%s' "$goal" | sed 's/\\/\\\\/g; s/"/\\"/g')
+printf '{"type":"agent_reasoning","message":"Planning work for: %s"}\n' "$esc"
+printf '{"type":"agent_message","message":"DONE: %s"}\n' "$esc"
+printf '{"type":"task_complete","message":"completed"}\n'
+exit 0

backend/deploy/e2e/mint-jwt.js

@@ -0,0 +1,12 @@
+// Mint an HS256 bearer token for the deploy compose E2E — the same shape the app's JwtBearer accepts
+// (ValidateIssuer/Audience off, ValidateLifetime on, signing key = Authentication:Jwt:SymmetricKey). The `nameid`
+// claim is what ICurrentUser reads (ClaimTypes.NameIdentifier maps to "nameid"). Usage: node mint-jwt.js <key> <userId>
+const crypto = require("crypto");
+const [, , key, userId] = process.argv;
+if (!key || !userId) { console.error("usage: node mint-jwt.js <symmetric-key> <user-guid>"); process.exit(1); }
+
+const enc = (o) => Buffer.from(JSON.stringify(o)).toString("base64url");
+const now = Math.floor(Date.now() / 1000);
+const data = enc({ alg: "HS256", typ: "JWT" }) + "." + enc({ nameid: userId, nbf: now - 30, iat: now, exp: now + 3600 });
+const sig = crypto.createHmac("sha256", key).update(data).digest("base64url");
+process.stdout.write(data + "." + sig);

backend/deploy/e2e/run.sh

@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+# Deploy E2E: build + boot the REAL api/worker/Postgres images, then prove — through HTTP, not in-process — that
+# the API enqueues and the WORKER runs an agent through its own image to a terminal Success. No model credential or
+# network: the worker's harness is pointed at a fake codex. Also asserts both pods' health probes and API leanness.
+#
+#   backend/deploy/e2e/run.sh            # from anywhere; needs docker compose + node + curl
+set -euo pipefail
+cd "$(dirname "$0")"
+
+export E2E_JWT_KEY="deploy-e2e-only-symmetric-jwt-key-at-least-32-bytes"
+# Test-only AES-256 master key (base64 32 bytes) for variable encryption — required in non-Development.
+export E2E_VARIABLE_MASTER_KEY="ECxvpc1Qm6DkzLwTyHnv0OkVj3XaVLVegdoh1NbHziU="
+USER_ID="11111111-1111-1111-1111-111111111111"
+TEAM_ID="22222222-2222-2222-2222-222222222222"
+API="http://localhost:18080"
+WORKER="http://localhost:18081"
+COMPOSE="docker compose -f docker-compose.e2e.yml"
+
+fail() { echo "❌ $1"; echo "--- worker logs (tail) ---"; $COMPOSE logs --tail=120 worker || true; exit 1; }
+cleanup() { $COMPOSE down -v >/dev/null 2>&1 || true; }
+trap cleanup EXIT
+
+wait_ready() { # $1=base-url $2=name — readiness only flips 200 after DbUp + the host are up
+  for _ in $(seq 1 90); do
+    [ "$(curl -fsS -o /dev/null -w '%{http_code}' "$1/health/ready" 2>/dev/null || echo 000)" = "200" ] && { echo "    $2 ready"; return 0; }
+    sleep 2
+  done
+  fail "$2 never became ready at $1/health/ready"
+}
+
+# Boot Postgres + the API FIRST so the API runs DbUp once; only then start the worker (against the migrated schema)
+# — both pods run DbUp at startup, so serialising avoids a concurrent fresh-DB migration race.
+echo "==> build + boot postgres + API (API migrates the schema)"
+$COMPOSE up -d --build postgres api
+echo "==> health probe: API (anonymous; 200 only after DbUp + host are up)"
+wait_ready "$API" "api"
+
+echo "==> boot the worker against the migrated schema"
+$COMPOSE up -d --build worker
+echo "==> health probe: worker"
+wait_ready "$WORKER" "worker"
+[ "$(curl -fsS -o /dev/null -w '%{http_code}' "$API/health/live" 2>/dev/null)" = "200" ] || fail "/health/live not 200 (anonymous liveness)"
+
+echo "==> the API image is LEAN (no agent CLIs / git / proxy)"
+$COMPOSE exec -T api sh -c '
+  for b in codex claude node git bwrap; do command -v "$b" >/dev/null 2>&1 && { echo "LEAK: $b present in API"; exit 1; }; done
+  ls codespace-mcp* >/dev/null 2>&1 && { echo "LEAK: codespace-mcp present in API"; exit 1; }
+  echo "    API carries zero agent-execution machinery"' || fail "API image is not clean"
+
+echo "==> seed a team + user + membership"
+$COMPOSE exec -T postgres psql -U codespace -d codespace -v ON_ERROR_STOP=1 -q < seed.sql || fail "seed failed"
+echo "    seeded"
+
+echo "==> launch a quick chat task via the API (enqueue only — the API processes nothing)"
+JWT="$(node mint-jwt.js "$E2E_JWT_KEY" "$USER_ID")"
+AUTH=(-H "Authorization: Bearer $JWT" -H "X-Team-Id: $TEAM_ID")
+RESP="$(curl -fsS -X POST "$API/api/workflows/runs" "${AUTH[@]}" -H "Content-Type: application/json" \
+  -d '{"taskText":"Deploy E2E smoke task","effort":"quick","harness":"codex-cli","runnerKind":"local","autonomy":"Confined","surfaceKind":"chat"}')" || fail "launch HTTP call failed"
+RUN_ID="$(printf '%s' "$RESP" | sed -n 's/.*"runId":"\([0-9a-f-]*\)".*/\1/p')"
+[ -n "$RUN_ID" ] || fail "launch returned no runId: $RESP"
+echo "    launched run $RUN_ID"
+
+echo "==> poll until the WORKER drives the agent to a terminal state"
+for _ in $(seq 1 80); do
+  # The run's own WorkflowRunStatus is the FIRST "status" in the detail JSON (nested node/agent statuses follow).
+  STATUS="$(curl -fsS "$API/api/workflows/runs/$RUN_ID" "${AUTH[@]}" | grep -o '"status":"[A-Za-z]*"' | head -1 | sed 's/.*:"\([A-Za-z]*\)"/\1/')"
+  echo "    status=$STATUS"
+  case "$STATUS" in
+    Success) echo "✅ the API enqueued and the WORKER ran the agent through its real image to Success"; exit 0 ;;
+    Failure|Cancelled) fail "run reached terminal $STATUS (expected Success)" ;;
+  esac
+  sleep 3
+done
+fail "run never reached a terminal state within the timeout"

backend/deploy/e2e/seed.sql

@@ -0,0 +1,20 @@
+-- Deploy compose E2E seed: a non-bootstrap user + team + Owner membership, so a minted JWT can launch a task
+-- (the default 0006 admin carries password_must_change, which the launch command path gates — this clean user
+-- sidesteps that). Later-added columns (team.kind, app_user.is_bot, password_must_change) all have DEFAULTs, so the
+-- minimal column set below is sufficient. Idempotent (ON CONFLICT DO NOTHING) so a re-run is a no-op.
+INSERT INTO app_user (id, email, name, created_date, created_by, last_modified_date, last_modified_by)
+VALUES ('11111111-1111-1111-1111-111111111111', 'deploy-e2e@codespace.local', 'Deploy E2E',
+        now(), '00000000-0000-0000-0000-000000000001', now(), '00000000-0000-0000-0000-000000000001')
+ON CONFLICT (id) DO NOTHING;
+
+INSERT INTO team (id, slug, name, owner_user_id, created_date, created_by, last_modified_date, last_modified_by)
+VALUES ('22222222-2222-2222-2222-222222222222', 'deploy-e2e', 'Deploy E2E',
+        '11111111-1111-1111-1111-111111111111',
+        now(), '00000000-0000-0000-0000-000000000001', now(), '00000000-0000-0000-0000-000000000001')
+ON CONFLICT (id) DO NOTHING;
+
+INSERT INTO team_membership (id, team_id, user_id, role, created_date, created_by, last_modified_date, last_modified_by)
+VALUES ('33333333-3333-3333-3333-333333333333',
+        '22222222-2222-2222-2222-222222222222', '11111111-1111-1111-1111-111111111111', 'Owner',
+        now(), '00000000-0000-0000-0000-000000000001', now(), '00000000-0000-0000-0000-000000000001')
+ON CONFLICT (team_id, user_id) DO NOTHING;