diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3bf385a891..ed63818486 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,7 +34,7 @@ jobs: scope: name: Change scope if: ${{ !github.event.pull_request.draft }} - runs-on: ubuntu-latest + runs-on: ${{ (github.event.pull_request.head.repo.fork || !vars.TN_RUNNER_LIGHT) && 'ubuntu-24.04' || vars.TN_RUNNER_LIGHT }} timeout-minutes: 5 outputs: scope: ${{ steps.classify.outputs.scope }} @@ -92,7 +92,7 @@ jobs: typecheck: needs: [scope, build-artifacts] if: ${{ needs.scope.outputs.selection == 'full' }} - runs-on: ubuntu-latest + runs-on: ${{ (github.event.pull_request.head.repo.fork || !vars.TN_RUNNER) && 'ubuntu-24.04' || vars.TN_RUNNER }} timeout-minutes: 15 env: TN_CI_SHA: ${{ needs.scope.outputs.candidate_sha }} @@ -122,7 +122,7 @@ jobs: # tree skips it too — every gate it would run here already passed in the run that tested this # exact tree, and `!= 'prose'` would otherwise have bought a second full lint. if: needs.scope.outputs.selection == 'full' || needs.scope.outputs.selection == 'instructions' - runs-on: ubuntu-latest + runs-on: ${{ (github.event.pull_request.head.repo.fork || !vars.TN_RUNNER) && 'ubuntu-24.04' || vars.TN_RUNNER }} timeout-minutes: 10 env: TN_CI_SHA: ${{ needs.scope.outputs.candidate_sha }} @@ -178,7 +178,7 @@ jobs: test: needs: scope if: ${{ needs.scope.outputs.selection == 'full' }} - runs-on: ubuntu-24.04 + runs-on: ${{ (github.event.pull_request.head.repo.fork || !vars.TN_RUNNER) && 'ubuntu-24.04' || vars.TN_RUNNER }} timeout-minutes: 45 env: TN_CI_SHA: ${{ needs.scope.outputs.candidate_sha }} @@ -225,7 +225,7 @@ jobs: test-unit: needs: [scope, build-artifacts] if: ${{ needs.scope.outputs.selection == 'full' }} - runs-on: ubuntu-24.04 + runs-on: ${{ (github.event.pull_request.head.repo.fork || !vars.TN_RUNNER) && 'ubuntu-24.04' || vars.TN_RUNNER }} timeout-minutes: 30 strategy: fail-fast: false @@ -263,7 +263,7 @@ jobs: test-native: needs: [scope, build-artifacts] if: ${{ needs.scope.outputs.selection == 'full' }} - runs-on: ubuntu-24.04 + runs-on: ${{ (github.event.pull_request.head.repo.fork || !vars.TN_RUNNER) && 'ubuntu-24.04' || vars.TN_RUNNER }} timeout-minutes: 75 env: TN_CI_SHA: ${{ needs.scope.outputs.candidate_sha }} @@ -538,7 +538,7 @@ jobs: test-browser: needs: [scope, build-artifacts] if: ${{ needs.scope.outputs.selection == 'full' }} - runs-on: ubuntu-latest + runs-on: ${{ (github.event.pull_request.head.repo.fork || !vars.TN_RUNNER) && 'ubuntu-24.04' || vars.TN_RUNNER }} timeout-minutes: 30 env: TN_CI_SHA: ${{ needs.scope.outputs.candidate_sha }} @@ -592,7 +592,7 @@ jobs: test-playtest: needs: [scope, build-artifacts] if: ${{ needs.scope.outputs.selection == 'full' }} - runs-on: ubuntu-latest + runs-on: ${{ (github.event.pull_request.head.repo.fork || !vars.TN_RUNNER) && 'ubuntu-24.04' || vars.TN_RUNNER }} timeout-minutes: 30 env: TN_CI_SHA: ${{ needs.scope.outputs.candidate_sha }} @@ -633,7 +633,7 @@ jobs: template: - starter - platformer - runs-on: ubuntu-latest + runs-on: ${{ (github.event.pull_request.head.repo.fork || !vars.TN_RUNNER) && 'ubuntu-24.04' || vars.TN_RUNNER }} # The scenario arithmetic this number came from moved out to `template-nonvisual`, which now # owns the non-visual sweep for all eight templates. What is left is `verify:golden-path` — # pack, scaffold, install, dev, test, build, assert, once per matrix leg — which measured 309s @@ -820,7 +820,7 @@ jobs: - { template: starter, shard: "2/3" } - { template: starter, shard: "3/3" } - { template: tower-defense, shard: "1/1" } - runs-on: ubuntu-latest + runs-on: ${{ (github.event.pull_request.head.repo.fork || !vars.TN_RUNNER) && 'ubuntu-24.04' || vars.TN_RUNNER }} timeout-minutes: 30 env: TN_CI_SHA: ${{ needs.scope.outputs.candidate_sha }} @@ -919,7 +919,7 @@ jobs: # as cancelled, so the skip is correct there; it still cannot be mistaken for a pass, because a # cancelled run has no passing checks to merge on. if: ${{ !cancelled() && needs.scope.outputs.selection == 'full' }} - runs-on: ubuntu-latest + runs-on: ${{ (github.event.pull_request.head.repo.fork || !vars.TN_RUNNER_LIGHT) && 'ubuntu-24.04' || vars.TN_RUNNER_LIGHT }} timeout-minutes: 5 env: TN_CI_SHA: ${{ needs.scope.outputs.candidate_sha }} @@ -938,7 +938,7 @@ jobs: benchmark: needs: [scope, build-artifacts] if: ${{ needs.scope.outputs.selection == 'full' }} - runs-on: ubuntu-latest + runs-on: ${{ (github.event.pull_request.head.repo.fork || !vars.TN_RUNNER) && 'ubuntu-24.04' || vars.TN_RUNNER }} timeout-minutes: 25 env: TN_CI_SHA: ${{ needs.scope.outputs.candidate_sha }} @@ -992,7 +992,7 @@ jobs: build: needs: [scope, build-artifacts] if: ${{ !cancelled() && needs.scope.outputs.selection == 'full' }} - runs-on: ubuntu-latest + runs-on: ${{ (github.event.pull_request.head.repo.fork || !vars.TN_RUNNER_LIGHT) && 'ubuntu-24.04' || vars.TN_RUNNER_LIGHT }} timeout-minutes: 2 env: TN_CI_SHA: ${{ needs.scope.outputs.candidate_sha }} @@ -1014,7 +1014,7 @@ jobs: build-artifacts: needs: scope if: ${{ needs.scope.outputs.selection == 'full' }} - runs-on: ubuntu-latest + runs-on: ${{ (github.event.pull_request.head.repo.fork || !vars.TN_RUNNER) && 'ubuntu-24.04' || vars.TN_RUNNER }} timeout-minutes: 20 env: TN_CI_SHA: ${{ needs.scope.outputs.candidate_sha }} @@ -1058,7 +1058,7 @@ jobs: performance-contracts: needs: scope if: ${{ needs.scope.outputs.selection == 'full' && !cancelled() }} - runs-on: ubuntu-latest + runs-on: ${{ (github.event.pull_request.head.repo.fork || !vars.TN_RUNNER) && 'ubuntu-24.04' || vars.TN_RUNNER }} timeout-minutes: 2 env: TN_CI_SHA: ${{ needs.scope.outputs.candidate_sha }} @@ -1124,7 +1124,7 @@ jobs: budgets: needs: [scope, build-artifacts] if: ${{ needs.scope.outputs.selection == 'full' }} - runs-on: ubuntu-latest + runs-on: ${{ (github.event.pull_request.head.repo.fork || !vars.TN_RUNNER) && 'ubuntu-24.04' || vars.TN_RUNNER }} timeout-minutes: 15 env: TN_CI_SHA: ${{ needs.scope.outputs.candidate_sha }} @@ -1250,7 +1250,7 @@ jobs: ci-required: needs: [scope, native-platforms, typecheck, lint, test, test-unit, test-native, test-browser, test-playtest, golden-path-template, template-nonvisual, golden-path, benchmark, build, build-artifacts, budgets, performance-contracts, supply-chain] if: ${{ always() && !github.event.pull_request.draft }} - runs-on: ubuntu-latest + runs-on: ${{ (github.event.pull_request.head.repo.fork || !vars.TN_RUNNER_LIGHT) && 'ubuntu-24.04' || vars.TN_RUNNER_LIGHT }} timeout-minutes: 5 # A reused verdict is only a pass if the run it cites passed its own ci-required job, and that # is an Actions API read rather than anything this repository can compute. @@ -1310,7 +1310,7 @@ jobs: # `!cancelled()` and not `always()`, for the reason golden-path gives above: a cancelled run # has nothing to report and its summary would only describe the cancellation. if: ${{ !cancelled() && needs.scope.outputs.selection == 'full' }} - runs-on: ubuntu-latest + runs-on: ${{ (github.event.pull_request.head.repo.fork || !vars.TN_RUNNER_LIGHT) && 'ubuntu-24.04' || vars.TN_RUNNER_LIGHT }} timeout-minutes: 5 permissions: contents: read diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 2552db7862..51e898d1e7 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -33,7 +33,7 @@ jobs: paths: name: lanes this pull request touches if: ${{ !github.event.pull_request.draft }} - runs-on: ubuntu-latest + runs-on: ${{ (github.event.pull_request.head.repo.fork || !vars.TN_RUNNER_LIGHT) && 'ubuntu-24.04' || vars.TN_RUNNER_LIGHT }} timeout-minutes: 5 permissions: contents: read @@ -64,7 +64,7 @@ jobs: name: csg if: ${{ needs.paths.outputs.csg == 'true' && !github.event.pull_request.draft }} needs: paths - runs-on: ubuntu-latest + runs-on: ${{ (github.event.pull_request.head.repo.fork || !vars.TN_RUNNER) && 'ubuntu-24.04' || vars.TN_RUNNER }} timeout-minutes: 10 defaults: run: @@ -87,7 +87,7 @@ jobs: name: ik if: ${{ needs.paths.outputs.ik == 'true' && !github.event.pull_request.draft }} needs: paths - runs-on: ubuntu-latest + runs-on: ${{ (github.event.pull_request.head.repo.fork || !vars.TN_RUNNER) && 'ubuntu-24.04' || vars.TN_RUNNER }} timeout-minutes: 10 defaults: run: @@ -106,7 +106,7 @@ jobs: name: vegetation if: ${{ needs.paths.outputs.vegetation == 'true' && !github.event.pull_request.draft }} needs: paths - runs-on: ubuntu-latest + runs-on: ${{ (github.event.pull_request.head.repo.fork || !vars.TN_RUNNER) && 'ubuntu-24.04' || vars.TN_RUNNER }} timeout-minutes: 10 defaults: run: @@ -125,7 +125,7 @@ jobs: name: decals if: ${{ needs.paths.outputs.decals == 'true' && !github.event.pull_request.draft }} needs: paths - runs-on: ubuntu-24.04 + runs-on: ${{ (github.event.pull_request.head.repo.fork || !vars.TN_RUNNER) && 'ubuntu-24.04' || vars.TN_RUNNER }} timeout-minutes: 30 steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 @@ -217,7 +217,7 @@ jobs: name: exposure if: ${{ needs.paths.outputs.tone == 'true' && !github.event.pull_request.draft }} needs: paths - runs-on: ubuntu-24.04 + runs-on: ${{ (github.event.pull_request.head.repo.fork || !vars.TN_RUNNER) && 'ubuntu-24.04' || vars.TN_RUNNER }} timeout-minutes: 15 steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 diff --git a/.github/workflows/native-platforms.yml b/.github/workflows/native-platforms.yml index a9ad933fe1..982022f728 100644 --- a/.github/workflows/native-platforms.yml +++ b/.github/workflows/native-platforms.yml @@ -53,7 +53,10 @@ jobs: # The caller is the single selection authority. Dispatch has no exemption inputs and is full. scope: name: Validate caller selection and candidate - runs-on: ubuntu-latest + # PRD-480. `github.event` inside a `workflow_call` is the caller's event payload, so this + # expression reads the same `pull_request` fork flag ci.yml did when it called this file: a fork + # PR falls through to hosted runners and never reaches the owner's machine. + runs-on: ${{ (github.event.pull_request.head.repo.fork || !vars.TN_RUNNER_LIGHT) && 'ubuntu-24.04' || vars.TN_RUNNER_LIGHT }} timeout-minutes: 5 outputs: scope: ${{ steps.classify.outputs.scope }} @@ -89,7 +92,7 @@ jobs: if: >- needs.scope.outputs.selection == 'full' && inputs.ios_only == false - runs-on: ubuntu-24.04 + runs-on: ${{ (github.event.pull_request.head.repo.fork || !vars.TN_RUNNER) && 'ubuntu-24.04' || vars.TN_RUNNER }} timeout-minutes: 30 env: TN_CI_SHA: ${{ needs.scope.outputs.candidate_sha }} @@ -136,7 +139,7 @@ jobs: name: Android V8 source payload needs: scope if: needs.scope.outputs.selection == 'full' && inputs.ios_only != true - runs-on: ubuntu-24.04 + runs-on: ${{ (github.event.pull_request.head.repo.fork || !vars.TN_RUNNER) && 'ubuntu-24.04' || vars.TN_RUNNER }} # 180-minute build ceiling plus 30 minutes for actions/cache/save, per the action's note. timeout-minutes: 210 outputs: @@ -186,6 +189,9 @@ jobs: needs.android-v8-source.outputs.published == 'false' && github.event_name != 'pull_request' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/develop') + # PRD-480 leaves this one on GitHub's machines: it is the release write itself — `contents: + # write` and a `gh release upload` below — and no pool container is handed a token that can + # publish. It is named on the hosted allow-list in ci-structure.spec.ts. runs-on: ubuntu-latest timeout-minutes: 10 permissions: @@ -213,7 +219,7 @@ jobs: name: Native collector evidence coverage needs: [scope, android-emulator-parity, desktop-parity, desktop, ios-simulator] if: ${{ !cancelled() && needs.scope.outputs.selection == 'full' }} - runs-on: ubuntu-24.04 + runs-on: ${{ (github.event.pull_request.head.repo.fork || !vars.TN_RUNNER) && 'ubuntu-24.04' || vars.TN_RUNNER }} timeout-minutes: 5 env: TN_CI_SHA: ${{ needs.scope.outputs.candidate_sha }} @@ -248,7 +254,7 @@ jobs: name: Networking qualification matrix needs: [scope, android-emulator-parity, desktop-parity, desktop, ios-simulator] if: ${{ always() && needs.scope.outputs.selection == 'full' }} - runs-on: ubuntu-24.04 + runs-on: ${{ (github.event.pull_request.head.repo.fork || !vars.TN_RUNNER_LIGHT) && 'ubuntu-24.04' || vars.TN_RUNNER_LIGHT }} timeout-minutes: 5 env: TN_CI_SHA: ${{ needs.scope.outputs.candidate_sha }} @@ -316,6 +322,8 @@ jobs: needs.scope.outputs.selection == 'full' && inputs.ios_only != true && needs.android-v8-source.result == 'success' + # Hosted, measured: SwiftShader renders the emulator's GPU on the CPU, and on a 4-thread tn-local + # slot run 37082733117 spent 41 min of this job's 45 still running APKs (hosted finishes in ~30). runs-on: ubuntu-24.04 timeout-minutes: 45 env: @@ -447,8 +455,14 @@ jobs: run: | echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \ | sudo tee /etc/udev/rules.d/99-kvm4all.rules - sudo udevadm control --reload-rules - sudo udevadm trigger --name-match=kvm + # `|| true` on both udev calls, because the routed `tn-local` container cannot do what + # they do: no udev is running in it and its /sys is read-only, so `udevadm trigger` + # fails there for a reason that has nothing to do with acceleration. The container does + # not need them — `scripts/ci-runners.sh` passes `--device /dev/kvm` and the image's + # entrypoint sets its mode before the job starts — and a step that fails on the way to + # the report below costs this lane the whole emulator run. + sudo udevadm control --reload-rules || true + sudo udevadm trigger --name-match=kvm || true # Report the mode, never assert it. `test -w /dev/kvm` as the last line of this step — # copied from native-release.yml, where nothing has exercised it — turned "this runner # has no KVM" into a failed job, which is worse than the slow boot it was meant to fix. @@ -740,7 +754,7 @@ jobs: if: >- needs.scope.outputs.selection != 'prose' && inputs.ios_only == false - runs-on: ubuntu-24.04 + runs-on: ${{ (github.event.pull_request.head.repo.fork || !vars.TN_RUNNER) && 'ubuntu-24.04' || vars.TN_RUNNER }} timeout-minutes: 20 env: TN_CI_SHA: ${{ needs.scope.outputs.candidate_sha }} @@ -812,7 +826,7 @@ jobs: if: >- needs.scope.outputs.selection == 'full' && inputs.ios_only != true - runs-on: ubuntu-24.04 + runs-on: ${{ (github.event.pull_request.head.repo.fork || !vars.TN_RUNNER) && 'ubuntu-24.04' || vars.TN_RUNNER }} timeout-minutes: 75 env: TN_CI_SHA: ${{ needs.scope.outputs.candidate_sha }} @@ -1242,8 +1256,12 @@ jobs: # The release container is a ZIP on macOS and Windows and desktop-distribution.mjs shells # out to `zip -r -q`. A missing archiver must be a red job, never a silent pass. if [ "${{ matrix.platform }}" = "Windows" ]; then - choco install zip -y --no-progress - choco install nsis --version=3.11.0 -y --no-progress + # The community feed drops requests now and then ("package was not found" for a listed + # version, run 37079046851). Retry twice; makensis -VERSION below still fails closed. + for attempt in 1 2 3; do + choco install zip -y --no-progress && choco install nsis --version=3.11.0 -y --no-progress && break + [ "$attempt" = 3 ] || sleep 30 + done nsis='/c/Program Files (x86)/NSIS' "$nsis/makensis.exe" -VERSION cygpath -w "$nsis" >> "$GITHUB_PATH" @@ -1766,7 +1784,7 @@ jobs: # the one WebGPU backend that publishes an arm64 Linux archive. scripts/download-deps.mjs # and scripts/native-build.mjs select that route from `process.arch`. - platform: linux-x64 - runner: ubuntu-24.04 + runner: ${{ (github.event.pull_request.head.repo.fork || !vars.TN_RUNNER) && 'ubuntu-24.04' || vars.TN_RUNNER }} - platform: linux-arm64 runner: ubuntu-24.04-arm runs-on: ${{ matrix.runner }} diff --git a/AGENTS.md b/AGENTS.md index 619de60fdf..40b36c363a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -87,7 +87,7 @@ TypeScript 5.9 `strict`, **ESM only**; relative imports end in `.js`. Versions c Results go in the existing PRD, PR or response; a separate evidence file only when the user asks or a workflow consumes it. Prose edits need only the doc checks and mirror regeneration — no implementation checkpoints or artificial negative controls. **Fail closed**: malformed input throws, a missing observation fails, an empty assertion set fails. `pnpm test` proves units; **a playtest scenario proves the game** — every runtime-behaviour change gets one, rerun on each later change to that behaviour. -**Nearest lane first, CI last.** A CI round trip costs over an hour; prove it locally (unit test, playtest, emulator, attached device) and push once green. **Prefer an emulator over hardware** when it can hold the claim; a stopped emulator means start it, not "blocked". Never call `xvfb-run` (its exit status is its own cleanup kill; `sh scripts/xvfb.sh ` is the wrapper). An unnamed WebGPU adapter may be SwiftShader: pass `--browser-recipe webgpu` and check `adapter.info`. Free a port with `lsof -ti tcp: | xargs -r kill`; `pkill -f vite` kills your own shell. +**Nearest lane first, CI last.** A CI round trip costs over an hour; prove it locally (unit test, playtest, emulator, attached device) and push once green. While `TN_RUNNER` is set, run only the focused checks for what you changed and push: the full board runs once, on `tn-local`. **Prefer an emulator over hardware** when it can hold the claim; a stopped emulator means start it, not "blocked". Never call `xvfb-run` (its exit status is its own cleanup kill; `sh scripts/xvfb.sh ` is the wrapper). An unnamed WebGPU adapter may be SwiftShader: pass `--browser-recipe webgpu` and check `adapter.info`. Free a port with `lsof -ti tcp: | xargs -r kill`; `pkill -f vite` kills your own shell. Long gates write `artifacts/gates/status.json`: read it with `pnpm gate:status`, probe a stale phase with `pnpm gate:doctor`; `pnpm gate:resume` continues only while worktree, branch, HEAD, lease and artifact still match. None of them repair or remove a worktree. ## Working outside the repo diff --git a/CLAUDE.md b/CLAUDE.md index 37cbe2675f..834c4076c3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -89,7 +89,7 @@ TypeScript 5.9 `strict`, **ESM only**; relative imports end in `.js`. Versions c Results go in the existing PRD, PR or response; a separate evidence file only when the user asks or a workflow consumes it. Prose edits need only the doc checks and mirror regeneration — no implementation checkpoints or artificial negative controls. **Fail closed**: malformed input throws, a missing observation fails, an empty assertion set fails. `pnpm test` proves units; **a playtest scenario proves the game** — every runtime-behaviour change gets one, rerun on each later change to that behaviour. -**Nearest lane first, CI last.** A CI round trip costs over an hour; prove it locally (unit test, playtest, emulator, attached device) and push once green. **Prefer an emulator over hardware** when it can hold the claim; a stopped emulator means start it, not "blocked". Never call `xvfb-run` (its exit status is its own cleanup kill; `sh scripts/xvfb.sh ` is the wrapper). An unnamed WebGPU adapter may be SwiftShader: pass `--browser-recipe webgpu` and check `adapter.info`. Free a port with `lsof -ti tcp: | xargs -r kill`; `pkill -f vite` kills your own shell. +**Nearest lane first, CI last.** A CI round trip costs over an hour; prove it locally (unit test, playtest, emulator, attached device) and push once green. While `TN_RUNNER` is set, run only the focused checks for what you changed and push: the full board runs once, on `tn-local`. **Prefer an emulator over hardware** when it can hold the claim; a stopped emulator means start it, not "blocked". Never call `xvfb-run` (its exit status is its own cleanup kill; `sh scripts/xvfb.sh ` is the wrapper). An unnamed WebGPU adapter may be SwiftShader: pass `--browser-recipe webgpu` and check `adapter.info`. Free a port with `lsof -ti tcp: | xargs -r kill`; `pkill -f vite` kills your own shell. Long gates write `artifacts/gates/status.json`: read it with `pnpm gate:status`, probe a stale phase with `pnpm gate:doctor`; `pnpm gate:resume` continues only while worktree, branch, HEAD, lease and artifact still match. None of them repair or remove a worktree. ## Working outside the repo diff --git a/docs/PRDs/CI/PRD-480-linux-ci-runs-on-the-owner-machine.md b/docs/PRDs/CI/PRD-480-linux-ci-runs-on-the-owner-machine.md index 11faae312d..35a58a727f 100644 --- a/docs/PRDs/CI/PRD-480-linux-ci-runs-on-the-owner-machine.md +++ b/docs/PRDs/CI/PRD-480-linux-ci-runs-on-the-owner-machine.md @@ -56,10 +56,18 @@ runs-on: ${{ (github.event.pull_request.head.repo.fork || !vars.TN_RUNNER) && 'u `tn-local`. Today the agent runs `pnpm test` or `pnpm ci:local --full` here, then CI repeats the same board, so this removes the duplicate. With `TN_RUNNER` unset, CI is hosted and slow again, and the local full run before pushing comes back. -- **Comparable timing:** each container is capped at 4 CPUs and 16 GB, the shape of a hosted - `ubuntu-24.04` runner, so operation budgets and timeouts keep meaning what they meant. -- **Ephemeral:** each container takes one job, exits and is recreated (`--ephemeral`, compose - `restart: always`). No `/tmp`, port, Xvfb display or workspace state crosses jobs. +- **Comparable timing:** each container is pinned (`--cpuset-cpus`) to two whole cores, four threads, + the CPU shape of a hosted `ubuntu-24.04` runner, so `nproc` reads 4 and every worker pool sizes itself + to that. A `--cpus 4` quota is not enough: `nproc` still reads every host CPU and vitest ran 8 workers + in a 4-core budget. This is a controlled starting point, not a promise of identical timing. +- **Host stays stable:** five slots by default, and two cores are never given to a slot. Each slot gets + `--memory 12g --memory-swap 12g --oom-score-adj 800`, so under memory pressure the kernel kills a CI + job before the owner's processes. +- **Light lane:** `scope`, `ci-required`, `run-summary` and other small joins run on one more, + unpinned `tn-local-light` runner (`--cpus 1`, 2 GB) that heavy jobs never select. Otherwise a 20-minute + build on every heavy slot holds a 10-second join in the queue. +- **Ephemeral:** each container takes one job, exits and is recreated by a host-side `docker run --rm` + loop. No `/tmp`, port, Xvfb display or workspace state crosses jobs. **Stays hosted:** `native-platforms` macOS, Windows, iOS and `linux-arm64` legs; the `supply-chain` job (its gitleaks step needs `docker run`, and mounting the Docker socket would give jobs root on the @@ -90,12 +98,16 @@ while `TN_RUNNER` is set: jobs queue until the switch is cleared. ## Acceptance Criteria -- [ ] AC-1 [shared]: proof: CI run id plus `runner_name` per job from `gh api …/runs//jobs`. A non-draft +- [x] AC-1 [shared]: proof: CI run id plus `runner_name` per job from `gh api …/runs//jobs`. A non-draft PR's `CI` run executes every Linux job except `supply-chain` on a `tn-local` runner, and `ci-required` passes. Evidence: pending. + Evidence: CI run 37070815769 (PR #404, head 94732de65, 2026-10-02): conclusion success, `ci-required` success. 39 jobs succeeded: 33 on `tn-local`, 5 on `tn-local-light`, 1 hosted (`supply-chain`); `native-platforms` skipped by scope. - [ ] AC-2 [shared]: proof: per-job `started_at − created_at` summed with the baseline's script. Total time jobs spent waiting for a runner on a full-board PR drops below 60 min (baseline 239 min, run - 37043413533). Evidence: pending. + 37043413533). Measured, not met: run 37070815769 waited 317 min in total across 40 jobs on 5 heavy + + 1 light slots, with 42 min wall against the baseline's 39 min wall and zero hosted minutes. A 40-job board on + six slots queues by construction, so this criterion needs either more slots or overflow to hosted + runners. Owner decision pending. ## Blocked on @@ -111,6 +123,10 @@ while `TN_RUNNER` is set: jobs queue until the switch is cleared. ## Decisions +- 2026-10-02 (João, during the first live runs): use more of the machine, but never at the cost of + desktop stability. Five pinned slots of two cores each, two cores reserved for the host, 12 GB caps + with CI as the OOM victim, and a reserved light lane (review by Astra the same day). + - 2026-10-02 (João): the runner registration token exists. It is a fine-grained token on `ThreeNativeHQ/threenative` only, with Administration read & write, stored in the operator's untracked runner env file. Verified by listing the repository's runners (0). @@ -128,44 +144,71 @@ while `TN_RUNNER` is set: jobs queue until the switch is cleared. ## Execution Phases #### Phase 1: The runner stack comes up and takes jobs - -**Status:** NOT STARTED -**Files:** NEW `tools/ci-runners/Dockerfile`, NEW `tools/ci-runners/compose.yml`, NEW +**Status:** IN PROGRESS — stack files built, runners not yet brought up +**Files:** NEW `tools/ci-runners/Dockerfile`, NEW `tools/ci-runners/entrypoint.sh`, NEW `scripts/ci-runners.sh` (`up [N]`, `down`, `status`; default N = 4, read from the env file). -**Implementation:** Ubuntu 24.04 runner image with the tool set above. Compose service with -`replicas: N`, `cpus: 4`, `mem_limit: 16g`, `restart: always`, and ephemeral repo-scoped registration -with labels `tn-local`. `up` waits for N online runners via `gh api repos/{owner}/{repo}/actions/runners` -before `gh variable set TN_RUNNER --body tn-local`. `down` deletes the variable first, then stops the -stack. The script fails closed when the env file or token is missing. - -- [ ] `scripts/ci-runners.sh up` brings 4 runners online with label `tn-local`. proof: - `scripts/ci-runners.sh status` lists 4 online runners. -- [ ] A runner container recreates itself after its job and keeps no state from it. proof: two + +Deviation from the plan below, owner-approved 2026-10-02: **no compose.yml.** A host-side loop of +`docker run --rm` replaces the compose service, because `restart: always` restarts the *same* +container — same filesystem, same hostname — and the ephemeral claim here is a fresh one of both +per job. + +**Implementation:** Ubuntu 24.04 runner image with the tool set above. One loop per slot +(`setsid nohup`), `cpus 4`, `memory 16g`, and ephemeral repo-scoped registration with labels +`tn-local`. `up` waits for N online runners via `gh api repos/{owner}/{repo}/actions/runners` +before `gh variable set TN_RUNNER --body tn-local`. `down` deletes the variable first, then stops +the stack. The script fails closed when the env file or token is missing. + + +- [x] `scripts/ci-runners.sh up` brings 4 runners online with label `tn-local`. proof: + `scripts/ci-runners.sh status` lists 4 online runners. Evidence: 2026-10-02, `up` printed "4 online; + TN_RUNNER=tn-local" and `status` listed tn-23e70acfd553, tn-365ab3636a2e, tn-4d83d32a4c75 and + tn-6071a30e949e, all online. The first `up` timed out with 0 online (wrong WORKDIR), cleared the + variable and stopped the pool, which is the fail-closed path working. Not started: `status` reports + `TN_RUNNER=unset`, 0 containers, 0 online runners on 2026-10-02. +- [x] A runner container recreates itself after its job and keeps no state from it. proof: two consecutive dispatched jobs report different container hostnames and an empty `/tmp`. + Not started: the image's `/tmp` is empty in a fresh container (`ls -A /tmp | wc -l` → 0), which + is one container's worth of the claim. + Evidence: CI run 37070815769 (PR #404, head 94732de65, 2026-10-02): 38 `tn-local*` jobs ran on 38 distinct runner names (`tn-`) from 6 slots, so every job got a fresh container; each starts from the image, whose `/tmp` is empty (`ls -A /tmp | wc -l` = 0 in the smoke). #### Phase 2: `ci.yml` and the integration workflows route through the switch -**Status:** NOT STARTED +**Status:** IN PROGRESS **Files:** EDIT `.github/workflows/ci.yml`, `.github/workflows/integration-*.yml`, `scripts/__tests__/ci-structure.spec.ts`; the `integration-*.yml` template wherever agents copy it from; `AGENTS.md` ("Nearest lane first, CI last") plus its regenerated `CLAUDE.md` mirror. **Implementation:** Replace each movable Linux `runs-on` with the routing expression. `supply-chain` keeps `ubuntu-latest`. Extend `ci-structure.spec.ts` so every Linux job in these workflows either uses the exact expression or is on a named hosted allow-list (`supply-chain`), and no macOS, Windows or arm64 -job ever does. - -- [ ] Structure spec enforces the routing. proof: `pnpm exec vitest run scripts/__tests__/ci-structure.spec.ts` - passes, and it fails on a job left at a bare `ubuntu-latest`. -- [ ] With `TN_RUNNER` set, a PR run lands its Linux jobs on `tn-local`. proof: AC-1 run. -- [ ] `AGENTS.md` says focused checks then push, not the full board, while `TN_RUNNER` is set. proof: +job ever does. 23 lines routed (18 in `ci.yml`, one each in five `integration-*.yml`); `supply-chain` +and `integration-decals.yml`'s `ubuntu-24.04-arm` job are the only Linux `runs-on` left hosted, and +`native-platforms.yml` is untouched until Phase 3. + +- [x] Structure spec enforces the routing. proof: `pnpm exec vitest run scripts/__tests__/ci-structure.spec.ts scripts/__tests__/ci-needs.spec.ts` + — 142 passed, 0 failed. Red first: with no workflow edited the new test failed on + `.github/workflows/ci.yml scope runs on 'runs-on: ubuntu-latest'`; after routing, setting `budgets` + (ci.yml:1051) back to `ubuntu-latest` failed it again on that job and it was restored. Also green: + `pnpm lint` (exit 0), `pnpm check:docs` (2496 links), `pnpm sync:agents --check` (22 mirrors). + One pre-existing assertion had to be narrowed: `template-nonvisual` forbade the substring + `pull_request` anywhere in the job, which the routing expression mentions without gating on it. + +- [x] With `TN_RUNNER` set, a PR run lands its Linux jobs on `tn-local`. proof: AC-1 run. + Evidence: CI run 37070815769 (PR #404, head 94732de65, 2026-10-02), as AC-1. +- [x] `AGENTS.md` says focused checks then push, not the full board, while `TN_RUNNER` is set. proof: `pnpm sync:agents --check` and `pnpm exec vitest run scripts/__tests__/sync-agent-docs.spec.ts` pass. Land it only after the AC-1 run is green. + Evidence: landed after AC-1; `pnpm sync:agents` + sync-agent-docs, primary-docs and instruction-budget specs, 24 passed (2026-10-02). +- [x] Light jobs never wait behind heavy ones (proof: `ci-structure.spec.ts` case, plus a full-board run + whose `ci-required` starts within 60 s of its last `needs` finishing): `scope`, `golden-path`, `build`, + `ci-required` and `run-summary` route to `tn-local-light`, and no heavy job can. + Evidence: ci-structure.spec.ts routing case (red with `typecheck` on the light lane), and CI run 37070815769 (PR #404, head 94732de65, 2026-10-02): `ci-required` started 22:48:50Z, 3 s after the last job it needs finished at 22:48:47Z. - [ ] With `TN_RUNNER` unset, the same workflow runs fully hosted. proof: `workflow_dispatch` run id with every `runner_name` hosted. #### Phase 3: The Linux `native-platforms` legs move too -**Status:** NOT STARTED +**Status:** IN PROGRESS — routing and image landed, live run pending **Files:** EDIT `.github/workflows/native-platforms.yml`, `tools/ci-runners/Dockerfile` (JDK 17, Android SDK, emulator, `/dev/kvm`), `scripts/__tests__/ci-structure.spec.ts`. **Implementation:** Route `web-reference`, `desktop-parity`, `android-emulator-parity`, `release-reports`, @@ -173,6 +216,32 @@ the remaining `ubuntu-24.04` jobs and the `linux-x64` rows of the `desktop` and Keep `macos-15`, `windows-2025`, `ubuntu-24.04-arm` and `ios-simulator` hosted. The `android-emulator-runner` action needs `/dev/kvm` access inside the container. +Decisions this phase had to make, 2026-10-02: + +- **`publish-android-v8` stays hosted.** It is the release write itself — `permissions: contents: write` + and a `gh release upload` — and no pool container is handed a token that can publish. + `android-v8-source` does move: it needs no secret, it is a published-asset download on every run but + a cold one, and its own action records a two-core cold build finishing inside the 210-minute ceiling, + which is this pool's slot shape. +- **Light lane: `scope` and `networking-matrix` only.** Both are a script with no `pnpm install` and no + toolchain. `performance-coverage` and `release-reports` are joins too, but each installs the workspace + and `performance-coverage` has a five-minute ceiling, so they take a heavy slot. +- **`desktop` has no Linux row.** Its matrix is `macOS` and `Windows` only; the movable Linux matrix row + is `starter-linux/linux-x64`. +- **The KVM step had to stop asserting udev.** In the container there is no udev and `/sys` is read-only, + so `udevadm control --reload-rules` and `udevadm trigger` both exit 1 and took the whole emulator lane + with them. They are now `|| true`, which is what the step's own "report the mode, never assert it" + rule already says; `scripts/ci-runners.sh` passes `--device /dev/kvm` and the image's entrypoint sets + the node's mode before the job can read it. +- **The image carries the pinned SDK, not the emulator.** `cmdline-tools` 16.0 with licenses accepted, + `platform-tools`, `platforms;android-35`, `platforms;android-36` (`compileSdk` in + `android/app/build.gradle.kts`), `build-tools;35.0.0` and `ndk;28.2.13676358` (`ndkVersion` there, + and what the workflows' own `sdkmanager` line installs), owned by `runner` because sdkmanager writes + into `ANDROID_HOME` at job time. `android-emulator-runner` installs `emulator` and the system image + itself, which is the only job that wants them. Image: 9.28 GB against the pool's 5.25 GB. + - [ ] A `native-platforms` run passes with its Linux legs on `tn-local`. proof: run id plus per-job `runner_name`. + Exception, measured: `android-emulator-parity` stays hosted. SwiftShader renders its GPU on the CPU, and on a + 4-thread slot run 37082733117 spent 41 of the job's 45 minutes still running APKs, where hosted takes ~30. - [ ] Waiting time meets AC-2. proof: the AC-2 measurement on a full-board PR after this phase. diff --git a/docs/PRDs/CI/PRD-481-ci-does-each-piece-of-work-once.md b/docs/PRDs/CI/PRD-481-ci-does-each-piece-of-work-once.md index 9b47962169..1c51cf2d52 100644 --- a/docs/PRDs/CI/PRD-481-ci-does-each-piece-of-work-once.md +++ b/docs/PRDs/CI/PRD-481-ci-does-each-piece-of-work-once.md @@ -160,7 +160,7 @@ skip it. "Take the compiled workspace from this run's producer" (download-artifact). - [ ] The native host cache restores on a PR from a `develop`-warmed key. proof: a `test-native` log with a cache hit and a build time under 60 s. -- [ ] No template or unit shard runs longer than 6 min. proof: per-job durations of one full CI run. +- [x] No template or unit shard runs longer than 6 min. proof: per-job durations of one full CI run. Measured on run 37071464562, two legs were over: `test-unit (4/4)` at 481s and `template-nonvisual (sailing, 2/2)` at 467s, with `rain (3/5)` 358s and `rain (2/5)` 299s next and every other unit and template leg under 360s. This commit takes sailing from two slices to three @@ -171,6 +171,8 @@ skip it. `sha1(path)` partition over that run's per-file durations puts every unit leg at ~137-240s of test time and sailing's worst slice at ~335s. The box stays unticked: its proof is a full CI run's per-job durations, and nothing here has run on GitHub. + Evidence: hosted CI run 37078784853 (PR #405 head 8e1436307, 2026-10-02): slowest legs test-unit (1/4) 329 s, + rain 4/5 319 s, sailing 2/3 289 s; every template and unit shard under 360 s. #### Phase 3: Triggers fire only when they prove something diff --git a/docs/verification/native-runtime-census-2026-08-16.md b/docs/verification/native-runtime-census-2026-08-16.md index ebb40e53e1..330c22b0c5 100644 --- a/docs/verification/native-runtime-census-2026-08-16.md +++ b/docs/verification/native-runtime-census-2026-08-16.md @@ -17,7 +17,7 @@ considered, and a KEEP/DELETE verdict. | --- | ---: | --- | --- | --- | --- | | `src/` | 57,757 | PRD-045, PRD-047, PRD-048, PRD-050, PRD-053, PRD-116, PRD-143, PRD-153, PRD-154, PRD-155, PRD-292 | `src/physics/native_bindings.cpp:586`; desktop V8 and native runtime commands | Move host shims into each game or delete the native host | **KEEP** — this is the owned native host and its physics boundary. | | `conformance/` | 9,477 | PRD-053, PRD-054, PRD-055, PRD-076 | `conformance/run-conformance.mjs`; root `pnpm parity` | Replace cross-target registry/proofs with untested per-game scripts | **KEEP** — shared executable conformance evidence. | -| `tests/` | 57,849 | PRD-045, PRD-046, PRD-048, PRD-049, PRD-050, PRD-053, PRD-054, PRD-055, PRD-076, PRD-116, PRD-127, PRD-143, PRD-153, PRD-154, PRD-155, PRD-292 | `tests/physics_actuation_bindings_test.cpp:141`; runtime-native Vitest and native tests | Delete fail-closed tests to reduce the trigger | **KEEP** — removal would conceal regressions. | +| `tests/` | 57,853 | PRD-045, PRD-046, PRD-048, PRD-049, PRD-050, PRD-053, PRD-054, PRD-055, PRD-076, PRD-116, PRD-127, PRD-143, PRD-153, PRD-154, PRD-155, PRD-292 | `tests/physics_actuation_bindings_test.cpp:141`; runtime-native Vitest and native tests | Delete fail-closed tests to reduce the trigger | **KEEP** — removal would conceal regressions. | | `__tests__/` | 936 | PRD-228 | `__tests__/host-gap-gpu-drain.spec.ts`; the runtime-native Vitest suite via `vitest.config.ts` | Fold it into `tests/`, where every other spec in this package lives | **KEEP** — it is one source-reading spec proving the GPU-drain profile stays diagnostic-only and never ships, and it is picked up by the same suite either way. Its one-file directory is an accident of where it landed, not a design; moving it into `tests/` is a tidy-up and not this row's argument. | | `scripts/` | 30,902 | PRD-045, PRD-048, PRD-049, PRD-050, PRD-053, PRD-054, PRD-116, PRD-127, PRD-153, PRD-155, PRD-292 | `scripts/verify-desktop-physics.mjs:206-228`; build, platform, and device-condition verifiers | Make every game own packaging, device, and verifier orchestration | **KEEP** — no smaller shared alternative preserves the proof. | | `scenarios/` | 109 | PRD-217 | `scenarios/starter-ui-overlay-desktop.playtest.json`; run by the hosted `native-platforms` desktop-core job through `scripts/verify-starter-ui-overlay.mjs` | Keep the starter-HUD input assertion inside the desktop-core job script instead of a standalone scenario file | **KEEP** — the scenario is the portable playtest that proves the starter HUD receives real input on the packed desktop runtime; the hosted desktop-core job is its caller, so the area is one owned proof rather than reusable game content. | @@ -35,7 +35,7 @@ considered, and a KEEP/DELETE verdict. | `js-engine-versions.json` | 38 | PRD-222 | `tests/js-engine-version-skew.test.mjs`; `scripts/download-deps.mjs` consumes the pinned engine versions | Leave engine/version compatibility as prose or duplicate pins across download scripts | **KEEP** — one fail-closed manifest prevents desktop, Android, and iOS engine pins from drifting silently. | | `build-matrix.json` | 231 | PRD-235 | `scripts/check-build-matrix.ts` from `scripts/check-budgets.ts:467` via `pnpm budgets`; proven by `scripts/__tests__/check-build-matrix.spec.ts` | Leave "which build directory does this test executable live in" as prose in `AGENTS.md` and tribal knowledge | **KEEP** — the prose answer was already wrong once: a contract test that only links in `build/tn-linux` is invisible to anyone reading `CMakeLists.txt`. This is the machine-readable contract, and the gate cross-checks every preset, cache variable, and test executable it names against CMake, so either side drifting is a named failure. | | `native-deps.lock.json` | 1,050 | PRD-059 | `scripts/download-deps.mjs` reads it before any acquisition; `tests/dependency-provenance.test.mjs` proves lock coverage | Leave dependency URLs/versions/hashes as inline mutable code in `download-deps.mjs` | **KEEP** — the reviewed immutable payload matrix is what makes reconstruction mean the same bytes; inline authority already served one 404. | -| **Total** | **180,355** | | `pnpm budgets` current measurement | | **No area rejected.** | +| **Total** | **180,359** | | `pnpm budgets` current measurement | | **No area rejected.** | **Regenerated 2026-09-02 for PRD-292 recovery:** the native physics ABI, its contract test, and the continuous-collision measurement harness changed the counted `src/`, `include/`, `tests/`, diff --git a/packages/runtime-native/tests/starter-desktop.test.mjs b/packages/runtime-native/tests/starter-desktop.test.mjs index f1be41f965..024f0b506f 100644 --- a/packages/runtime-native/tests/starter-desktop.test.mjs +++ b/packages/runtime-native/tests/starter-desktop.test.mjs @@ -387,7 +387,11 @@ test('native workflow proves the Linux release container on x64 and arm64', () = assert.ok(starter); // The arm64 leg is the point of the matrix: a free hosted ARM runner, the same release steps, // and the same scope selection as the x64 leg because it is the same job. - assert.match(starter, /- platform: linux-x64\n\s+runner: ubuntu-24\.04/u); + // linux-x64 routes through the TN_RUNNER switch (PRD-480) and falls back to the same hosted image. + assert.match( + starter, + /- platform: linux-x64\n\s+runner: \$\{\{ \(github\.event\.pull_request\.head\.repo\.fork \|\| !vars\.TN_RUNNER\) && 'ubuntu-24\.04' \|\| vars\.TN_RUNNER \}\}/u, + ); assert.match(starter, /- platform: linux-arm64\n\s+runner: ubuntu-24\.04-arm/u); assert.match(starter, /runs-on: \$\{\{ matrix\.runner \}\}/u); // The runner has no GPU, so the software Vulkan ICD and its X display are provisioned. diff --git a/scripts/__tests__/ci-structure.spec.ts b/scripts/__tests__/ci-structure.spec.ts index 460389e081..7193bafec3 100644 --- a/scripts/__tests__/ci-structure.spec.ts +++ b/scripts/__tests__/ci-structure.spec.ts @@ -544,16 +544,23 @@ function triggerSection(source: string): string { } function kvmProvisioning(source: string): readonly string[] { - return source - .split("\n") - .map((line) => line.trim()) - .filter( - (line) => - line.includes('KERNEL=="kvm"') || - line === "| sudo tee /etc/udev/rules.d/99-kvm4all.rules" || - line === "sudo udevadm control --reload-rules" || - line === "sudo udevadm trigger --name-match=kvm", - ); + return ( + source + .split("\n") + .map((line) => line.trim()) + // A trailing `|| true` is not part of the provisioning. native-platforms.yml's copy carries one + // because it is the lane PRD-480 routes onto the `tn-local` container, where no udev runs and + // /sys is read-only, so the commands cannot succeed there — while native-release.yml stays + // hosted and keeps them strict. The two lanes must still provision KVM the same way. + .map((line) => line.replace(/ \|\| true$/u, "")) + .filter( + (line) => + line.includes('KERNEL=="kvm"') || + line === "| sudo tee /etc/udev/rules.d/99-kvm4all.rules" || + line === "sudo udevadm control --reload-rules" || + line === "sudo udevadm trigger --name-match=kvm", + ) + ); } function commandText(section: string): string { @@ -1009,6 +1016,116 @@ describe("CI pipeline structure", () => { } }); + // PRD-480. Every Linux job that can move picks its runner from one expression, so the pool is + // changed by setting or deleting a repository variable and nothing else. A job left at a bare + // `ubuntu-latest` still runs, which is exactly why this has to be a spec: the pool comes up and + // the queue does not shrink, and nothing in a run says which half of the board ignored it. + const routing = + "${{ (github.event.pull_request.head.repo.fork || !vars.TN_RUNNER) && 'ubuntu-24.04' || vars.TN_RUNNER }}"; + const lightRouting = + "${{ (github.event.pull_request.head.repo.fork || !vars.TN_RUNNER_LIGHT) && 'ubuntu-24.04' || vars.TN_RUNNER_LIGHT }}"; + // The hosted jobs that stay hosted, per workflow. `supply-chain` runs gitleaks through + // `docker run`; a self-hosted container would need the host Docker socket mounted to do that, + // which would hand every job root on the owner's machine. `publish-android-v8` is the release + // publish itself — `permissions: contents: write` and a `gh release upload` — and PRD-480 keeps + // release writes on GitHub's own machines. + const hosted = new Map>([ + [".github/workflows/ci.yml", new Set(["supply-chain"])], + [ + ".github/workflows/native-platforms.yml", + // publish-android-v8 writes the release; android-emulator-parity is CPU-bound SwiftShader that + // overran its 45-minute budget on a pinned 4-thread slot (run 37082733117). + new Set(["android-emulator-parity", "publish-android-v8"]), + ], + ]); + // PRD-480's light lane: the jobs whose whole work is a script or a summary — no workspace build, + // no `pnpm install` and no test suite. `scope` classifies the diff, `build` and `golden-path` + // assert an upstream verdict, `ci-required` is the merge verdict and `run-summary` writes the + // report; in native-platforms.yml `networking-matrix` is the one join that needs no toolchain at + // all. They join on `tn-local-light` (1 CPU, 2 GB) so a ten-second verdict does not queue behind + // five twenty-minute builds. Both directions are load-bearing and both are asserted below: a + // heavy job here would build the workspace in one core, and a light job off the list would queue + // behind the heavy pool — which is the wait the lane exists to remove. + const light = new Set([ + "build", + "ci-required", + "golden-path", + "networking-matrix", + "paths", + "run-summary", + "scope", + ]); + + // A matrix job routes per row: `runs-on: ${{ matrix.runner }}` is only a pointer, and what decides + // where a leg lands is the `runner:` value inside `strategy.matrix.include`. Both native + // matrices carry hosted rows next to movable ones (`linux-x64` moves, `linux-arm64` cannot), so + // the row is named `job/row` here and that is the key the hosted allow-list would use. + function matrixRunnerRows(job: string, section: string): readonly (readonly [string, string])[] { + const rows: [string, string][] = []; + for (const row of section.split(/^ {10}- /mu).slice(1)) { + const runner = row.match(/^ {12}runner: (.+)$/mu)?.[1]; + if (runner !== undefined) { + const platform = row.match(/^\s*platform: (.+)$/mu)?.[1] ?? runner; + rows.push([`${job}/${platform}`, runner]); + } + } + return rows; + } + + it("routes every movable Linux job through exactly one TN_RUNNER switch", async () => { + const directory = path.join(repo, ".github/workflows"); + const routed = [ + ".github/workflows/ci.yml", + ".github/workflows/native-platforms.yml", + ...(await readdir(directory)) + .filter( + (entry) => + entry === "integration.yml" || + (entry.startsWith("integration-") && entry.endsWith(".yml")), + ) + .map((entry) => `.github/workflows/${entry}`), + ].sort(); + expect(routed.length).toBeGreaterThan(2); + for (const relative of routed) { + const keepHosted = hosted.get(relative) ?? new Set(); + const source = await readFile(path.join(repo, relative), "utf8"); + for (const [job, section] of jobSections(source)) { + const runsOn = section.match(/^\s+runs-on:.*$/mu)?.[0].trim(); + if (runsOn === undefined) continue; + const rows = matrixRunnerRows(job, section); + const targets: readonly (readonly [string, string])[] = + rows.length > 0 ? rows : [[job, runsOn.replace(/^runs-on: /u, "")]]; + for (const [name, runner] of targets) { + const target = `${relative} ${name}`; + // macOS, Windows and arm64 have no tn-local counterpart, so the expression must never + // appear on one of them. + if (/macos|windows|arm/u.test(runner)) { + expect(runner, `${target} is not Linux`).not.toContain(routing); + expect(runner, `${target} is not Linux`).not.toContain(lightRouting); + continue; + } + const onHeavy = runner === routing; + const onLight = runner === lightRouting; + if (keepHosted.has(name)) { + expect( + onHeavy || onLight, + `${target} is on the hosted allow-list but selects ${runner}`, + ).toBe(false); + continue; + } + expect( + onHeavy || onLight, + `${target} runs on \`${runner}\`: use the TN_RUNNER routing expression, or name it on the hosted allow-list`, + ).toBe(true); + expect( + light.has(name), + `${target} is ${onLight ? "on" : "off"} the light lane: only ${[...light].join(", ")} may select tn-local-light`, + ).toBe(onLight); + } + } + } + }); + // A draft PR spends no runner: every job needs `scope`, so skipping it and the always() gate // skips the board, and `ready_for_review` starts it once the draft is marked ready. it("runs nothing on draft pull requests until they are marked ready", async () => { @@ -1418,7 +1535,10 @@ describe("CI pipeline structure", () => { // Runs on every event since 2026-09-01 (owner call): the PR skip reported nothing on the // branch where the regression was written, and the merge that shipped it reported too late. expect(job).not.toContain("github.event_name == 'push'"); - expect(job).not.toContain("pull_request"); + // An `if:` gate naming the event, not any mention of it: the TN_RUNNER routing expression + // (PRD-480) reads `github.event.pull_request.head.repo.fork` in every routed job and gates + // nothing on it. What this forbids is a gate that skips pull requests. + expect(job).not.toMatch(/^\s+if:.*pull_request/mu); expect(job).toContain('TN_PLAYTEST_ALLOW_SOFTWARE: "1"'); expect(job).toContain("non-visual-scenarios.mjs"); expect(job).toContain("threenative-playtest"); diff --git a/scripts/ci-runners.sh b/scripts/ci-runners.sh new file mode 100755 index 0000000000..567932fc9a --- /dev/null +++ b/scripts/ci-runners.sh @@ -0,0 +1,214 @@ +#!/usr/bin/env bash +# The `tn-local` self-hosted runner pool (PRD-480). +# +# scripts/ci-runners.sh up [N] build the image, run N disposable heavy runner containers plus +# one light one, then set the TN_RUNNER and TN_RUNNER_LIGHT +# repository variables so the routing expressions in ci.yml and +# integration-*.yml send Linux jobs here +# scripts/ci-runners.sh down clear both variables first, then stop everything +# scripts/ci-runners.sh status what is set, what is running, what is online +# +# `down` deletes the variables before it stops anything, so the kill switch is the same command as +# the teardown: jobs go back to hosted runners the moment it is deleted, whether or not the +# containers are still there. +# +# The operator's token lives in an untracked env file outside the repository and is never printed. +# It reaches the containers only through `docker run --env-file`, and the container's entrypoint +# unsets it before `run.sh` starts, so no job ever sees it. +set -euo pipefail +cd "$(dirname -- "${BASH_SOURCE[0]}")/.." + +IMAGE=tn-ci-runner +LABEL=tn-local +LIGHT_LABEL=tn-local-light +VARIABLE=TN_RUNNER +LIGHT_VARIABLE=TN_RUNNER_LIGHT +ENV_FILE="${TN_RUNNERS_ENV:-${XDG_CONFIG_HOME:-$HOME/.config}/threenative/runners.env}" +STATE_DIR="${XDG_STATE_HOME:-$HOME/.local/state}/threenative/ci-runners" +DEFAULT_SLOTS=5 +# Cores never given to a slot, so the owner's desktop stays responsive under a full board. +HOST_CORES=2 +ONLINE_TIMEOUT_SECONDS=300 +# The light lane is a quota, not a reservation: one core of whatever a heavy slot is not using, +# 2 GB, and the highest OOM score of any container, so a small join is what gets killed under +# memory pressure rather than a 20-minute build. +LIGHT_SHAPE="--cpus 1 --memory 2g --memory-swap 2g --oom-score-adj 900" +# `/dev/kvm` for the heavy slots, and only where the host has one: `android-emulator-parity` boots a +# checksum-locked APK under `reactivecircus/android-emulator-runner`, which reads the device for +# read and write and otherwise falls back to a software boot. No KVM, no flag — a host without it +# still runs the pool, just without hardware acceleration, which that lane already reports instead +# of asserting. The light lane never gets it: nothing it runs emulates anything. +KVM_SHAPE="" +if [ -e /dev/kvm ]; then + KVM_SHAPE="--device /dev/kvm" +fi + +fail() { + local code="$1" + shift + printf 'TN_CI_RUNNERS: %s\n' "$*" >&2 + exit "$code" +} + +need() { + command -v "$1" >/dev/null 2>&1 || fail 2 "$1 is not on PATH" +} + +# Fail closed before anything is built or started. A pool that came up without a token would +# register nothing and leave TN_RUNNER set for hosted jobs that never arrive. +load_config() { + [ -n "$ENV_FILE" ] || fail 2 "TN_RUNNERS_ENV is empty" + [ -f "$ENV_FILE" ] || fail 2 "no runner env file at $ENV_FILE; set TN_RUNNERS_ENV or create it with RUNNER_ADMIN_TOKEN=..." + set -a + # shellcheck disable=SC1090 # the operator's own untracked file + . "$ENV_FILE" + set +a + # Read once for validation, then drop it: the token travels to the containers through + # `--env-file`, never through this process's environment. + [ -n "${RUNNER_ADMIN_TOKEN:-}" ] || fail 2 "RUNNER_ADMIN_TOKEN is unset or empty in $ENV_FILE" + unset RUNNER_ADMIN_TOKEN +} + +repo_name() { + gh repo view --json nameWithOwner --jq .nameWithOwner 2>/dev/null \ + || fail 2 "gh could not resolve this checkout's owner/repo" +} + +# One slot is a host-side loop, not a compose service: `docker run --rm` gives each job a fresh +# container filesystem and a fresh hostname, which `restart: always` on a long-lived container +# cannot. The loop is what makes the pool survive the exit. `shape` carries the docker resource +# flags word-split on purpose (there is no array to expand through `bash -c`), and `labels` is the +# one label the container registers with — `--no-default-labels` means that label is all it has. +start_slot() { + local slot="$1" state="$2" env_file="$3" repo="$4" shape="$5" labels="$6" + setsid nohup bash -c ' + while [ ! -e "$1/stop" ]; do + docker run --rm \ + --label tn-ci-runner=1 \ + $4 \ + --env-file "$2" \ + -e TN_RUNNER_REPO="$3" \ + -e TN_RUNNER_LABELS="$5" \ + tn-ci-runner || sleep 10 + done + ' _ "$state" "$env_file" "$repo" "$shape" "$labels" >"$state/$slot.log" 2>&1 & + echo "$!" >"$state/$slot.pid" +} + +stop_slots() { + local pid_file + for pid_file in "$STATE_DIR"/*.pid; do + [ -f "$pid_file" ] || continue + kill "$(cat "$pid_file")" 2>/dev/null || true + done + # No `rm -rf` anywhere near a path built from a variable: the stop file is what stops the loops. + : >"$STATE_DIR/stop" + docker ps -q --filter label=tn-ci-runner=1 | xargs -r docker stop +} + +online_count() { + gh api "repos/$1/actions/runners" --paginate --jq " + [.runners[] | select(.status == \"online\") | select(any(.labels[]; .name == \"$2\"))] | length + " 2>/dev/null || echo 0 +} + +up() { + local slots="${1:-${TN_RUNNERS:-$DEFAULT_SLOTS}}" + case "$slots" in ''|*[!0-9]*) fail 2 "N must be a positive integer, got '$slots'" ;; esac + [ "$slots" -ge 1 ] || fail 2 "N must be at least 1, got '$slots'" + need docker + need gh + load_config + local repo + repo="$(repo_name)" + + docker build -t "$IMAGE" tools/ci-runners + mkdir -p "$STATE_DIR" + # A leftover stop file from a previous teardown would make every loop exit before it starts. + rm -f "$STATE_DIR/stop" + + # Pinned CPUs, not a `--cpus 4` quota: under a quota `nproc` and os.availableParallelism() still + # report every host CPU, so vitest sized its pool for 24 cores inside a 4-core budget and three + # unit tests timed out (PR #404). Each slot takes two whole cores, both hyperthreads, as a hosted + # 4-vCPU runner does. + # ponytail: assumes Linux's usual sibling numbering (CPU k and k + nproc/2 share a core); read + # /sys/devices/system/cpu/cpu*/topology/thread_siblings_list if a host numbers them otherwise. + local half=$(( $(nproc) / 2 )) + [ $(( slots * 2 + HOST_CORES )) -le "$half" ] \ + || fail 2 "$slots slots need $(( slots * 2 )) cores plus $HOST_CORES for the host; this host has $half" + local slot base + for slot in $(seq 1 "$slots"); do + base=$(( (slot - 1) * 2 )) + start_slot "slot-$slot" "$STATE_DIR" "$ENV_FILE" "$repo" \ + "--cpuset-cpus $base,$(( base + 1 )),$(( base + half )),$(( base + half + 1 )) --memory 12g --memory-swap 12g --oom-score-adj 800 $KVM_SHAPE" \ + "$LABEL" + done + # One light slot, unpinned and quota-limited, labelled so heavy jobs cannot select it: a 10-second + # join behind five 20-minute builds is the wait this lane exists to remove. It never reserves a + # core, so the check above still describes exactly the cores this pool owns. + start_slot light "$STATE_DIR" "$ENV_FILE" "$repo" "$LIGHT_SHAPE" "$LIGHT_LABEL" + printf '%s heavy + 1 light runner container(s) starting; logs in %s\n' "$slots" "$STATE_DIR" + + # Only advertise the labels once the runners are really there: TN_RUNNER set with an empty pool + # is a queue that never drains. Both labels wait, because a light runner that never came online + # leaves its jobs queued behind the heavy pool forever. On timeout, take the whole thing back down + # rather than leaving half a pool registered. + local deadline=$(( $(date +%s) + ONLINE_TIMEOUT_SECONDS )) online light_online + while :; do + online="$(online_count "$repo" "$LABEL")" + light_online="$(online_count "$repo" "$LIGHT_LABEL")" + if [ "$online" -ge "$slots" ] && [ "$light_online" -ge 1 ]; then break; fi + if [ "$(date +%s)" -ge "$deadline" ]; then + gh variable delete "$VARIABLE" --repo "$repo" 2>/dev/null || true + gh variable delete "$LIGHT_VARIABLE" --repo "$repo" 2>/dev/null || true + stop_slots + fail 1 "$online of $slots $LABEL and $light_online $LIGHT_LABEL runner(s) came online within ${ONLINE_TIMEOUT_SECONDS}s" + fi + sleep 10 + done + + gh variable set "$VARIABLE" --body "$LABEL" --repo "$repo" + gh variable set "$LIGHT_VARIABLE" --body "$LIGHT_LABEL" --repo "$repo" + printf '%s %s + %s %s online; %s=%s and %s=%s, so deleting either routes its jobs back\n' \ + "$online" "$LABEL" "$light_online" "$LIGHT_LABEL" \ + "$VARIABLE" "$LABEL" "$LIGHT_VARIABLE" "$LIGHT_LABEL" +} + +down() { + need gh + local repo + repo="$(repo_name)" + # First, so no job is ever routed to a pool that is on its way out. + local variable + for variable in "$VARIABLE" "$LIGHT_VARIABLE"; do + gh variable delete "$variable" --repo "$repo" 2>/dev/null \ + || printf 'TN_CI_RUNNERS: %s was already unset\n' "$variable" >&2 + done + mkdir -p "$STATE_DIR" + stop_slots + printf '%s and %s cleared; runners stopping\n' "$VARIABLE" "$LIGHT_VARIABLE" +} + +status() { + need gh + local repo + repo="$(repo_name)" + local variable label + for variable in "$VARIABLE" "$LIGHT_VARIABLE"; do + printf '%s=%s\n' "$variable" "$(gh variable get "$variable" --repo "$repo" 2>/dev/null || echo unset)" + done + printf 'containers: %s\n' "$(docker ps -q --filter label=tn-ci-runner=1 | wc -l | tr -d ' ')" + for label in "$LABEL" "$LIGHT_LABEL"; do + printf 'online %s runners (name, status, busy):\n' "$label" + gh api "repos/$repo/actions/runners" --paginate --jq \ + ".runners[] | select(any(.labels[]; .name == \"$label\")) | \" \\(.name) \\(.status) busy=\\(.busy)\"" \ + 2>/dev/null || echo " none reported" + done +} + +case "${1:-}" in + up) shift; up "${1:-}" ;; + down) down ;; + status) status ;; + *) fail 2 "usage: ci-runners.sh up [N] | down | status" ;; +esac diff --git a/tools/ci-runners/Dockerfile b/tools/ci-runners/Dockerfile new file mode 100644 index 0000000000..96995b0e61 --- /dev/null +++ b/tools/ci-runners/Dockerfile @@ -0,0 +1,95 @@ +# The `tn-local` self-hosted runner image (PRD-480). One container per job: `scripts/ci-runners.sh` +# starts a host-side loop of `docker run --rm`, so nothing survives a job but the image itself. +# +# Ubuntu 24.04 to match hosted `ubuntu-24.04`, with the `runner` user's passwordless sudo intact so +# the workflow's own `sudo apt-get install` steps keep working. The Docker CLI is deliberately +# absent and the socket is never mounted: `supply-chain` needs `docker run` for gitleaks and stays +# on a hosted runner for exactly that reason. +FROM ghcr.io/actions/actions-runner:latest + +USER root + +# Build and test toolchain the moving Linux jobs take from the hosted image. `git`, `curl`, `jq` +# and `unzip` are the runner's own prerequisites; the rest is what ci.yml's jobs actually invoke. +# Playwright's Chromium dependencies are the exact `tools` and `chromium` lists Playwright 1.62.1 +# resolves for ubuntu24.04-x64 (playwright-core's bundled nativeDeps), transcribed so the apt layer +# does not depend on the Node layer below. +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + build-essential cmake ninja-build ccache clang lld pkg-config \ + git curl jq unzip zip xz-utils xvfb xauth python3 python3-pip \ + 7zip ffmpeg imagemagick zstd file lsof procps openjdk-17-jdk-headless \ + fonts-noto-color-emoji fonts-unifont fonts-liberation fonts-freefont-ttf \ + libfontconfig1 libfreetype6 \ + libasound2t64 libatk-bridge2.0-0t64 libatk1.0-0t64 libatspi2.0-0t64 \ + libcairo2 libcups2t64 libdbus-1-3 libdrm2 libgbm1 libglib2.0-0t64 \ + libnspr4 libnss3 libpango-1.0-0 \ + libx11-6 libxcb1 libxcomposite1 libxdamage1 libxext6 libxfixes3 libxkbcommon0 libxrandr2 \ + && rm -rf /var/lib/apt/lists/* + +# Hosted ubuntu-24.04 ships Node 20 and gh on PATH, and jobs rely on both before any setup step: +# the `scope` job runs `node scripts/ci-change-scope.mjs` straight after checkout. +ARG NODE_VERSION=20.19.6 +RUN curl -fsSL "https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-x64.tar.xz" \ + | tar -xJ -C /usr/local --strip-components=1 --exclude=CHANGELOG.md --exclude=README.md --exclude=LICENSE \ + && curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg \ + -o /usr/share/keyrings/githubcli-archive-keyring.gpg \ + && echo "deb [arch=amd64 signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" \ + > /etc/apt/sources.list.d/github-cli.list \ + && apt-get update && apt-get install -y --no-install-recommends gh \ + && rm -rf /var/lib/apt/lists/* + +# Android platform-tools where hosted ubuntu-24.04 keeps its SDK: the playtest doctor resolves adb +# through ANDROID_HOME before PATH. +ENV ANDROID_HOME=/usr/local/lib/android/sdk ANDROID_SDK_ROOT=/usr/local/lib/android/sdk +# Hosted runners run in C.UTF-8; without it javac reads sources as US-ASCII and rejects an em dash. +ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 +# The emulator lane writes a udev rule before it starts; hosted has the directory, the base image does not. +RUN mkdir -p /etc/udev/rules.d "$ANDROID_HOME" \ + && curl -fsSL https://dl.google.com/android/repository/platform-tools-latest-linux.zip -o /tmp/pt.zip \ + && unzip -q /tmp/pt.zip -d "$ANDROID_HOME" && rm /tmp/pt.zip \ + && ln -s "$ANDROID_HOME/platform-tools/adb" /usr/local/bin/adb + +# The SDK every routed native-platforms Android step calls into. `sdkmanager` is the whole of it: +# `android-actions/setup-android` uses an already-installed `cmdline-tools/latest` when the version +# matches, the workflows run their own `sdkmanager "platforms;android-35" "build-tools;35.0.0" +# "ndk;28.2.13676358"`, and `android-emulator-runner` installs what it is missing from it. The +# versions are the ones this repository pins — build-tools 35.0.0, which is also the emulator +# action's own constant; platforms 35 for the parity lane's api-level and 36, which +# packages/runtime-native/android/app/build.gradle.kts declares as compileSdk; and the NDK, which +# that same file names in `ndkVersion` and which the Android V8 build refuses to start without. +# Licenses are accepted at build time so none of those job-time installs stalls on a prompt. +# `emulator` and the system image are deliberately absent: android-emulator-runner installs both +# itself, which is the only Android job that wants them. +ARG CMDLINE_TOOLS=12266719 +RUN set -eu; \ + curl -fsSL "https://dl.google.com/android/repository/commandlinetools-linux-${CMDLINE_TOOLS}_latest.zip" -o /tmp/cli.zip; \ + mkdir -p "$ANDROID_HOME/cmdline-tools"; \ + unzip -q /tmp/cli.zip -d "$ANDROID_HOME/cmdline-tools"; \ + rm /tmp/cli.zip; \ + mv "$ANDROID_HOME/cmdline-tools/cmdline-tools" "$ANDROID_HOME/cmdline-tools/latest"; \ + export PATH="$ANDROID_HOME/cmdline-tools/latest/bin:$PATH"; \ + yes | sdkmanager --licenses > /dev/null; \ + sdkmanager --install "platform-tools" "platforms;android-35" "platforms;android-36" \ + "build-tools;35.0.0" "ndk;28.2.13676358" > /dev/null; \ + rm -rf "$ANDROID_HOME/.temp"; \ + # `runner` owns the SDK, because sdkmanager writes into ANDROID_HOME at job time: the workflows \ + # install their own packages and android-emulator-runner installs the emulator, the platform and \ + # the system image, and a root-owned SDK fails the write as the unprivileged job user. Chowned \ + # here rather than in a later layer, which would copy the whole SDK up and double its size. \ + chown -R runner:runner "$ANDROID_HOME" +ENV PATH=/usr/local/lib/android/sdk/cmdline-tools/latest/bin:$PATH + +COPY entrypoint.sh /home/runner/entrypoint.sh + +# Rust as hosted ubuntu-24.04 has it (stable, on PATH): the native physics ABI is built with cargo. +# Installed as `runner` so cargo can write its own registry and target caches. +USER runner +RUN curl -fsSL https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain stable +ENV PATH=/home/runner/.cargo/bin:$PATH +USER root +RUN chmod 0755 /home/runner/entrypoint.sh + +USER runner +WORKDIR /home/runner +ENTRYPOINT ["/home/runner/entrypoint.sh"] diff --git a/tools/ci-runners/entrypoint.sh b/tools/ci-runners/entrypoint.sh new file mode 100755 index 0000000000..4a9675d418 --- /dev/null +++ b/tools/ci-runners/entrypoint.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# Registers this container as an ephemeral `tn-local` runner — or `tn-local-light` for the light +# lane — and takes one job. +# +# The container is the disposable half of `scripts/ci-runners.sh up`: it configures, runs exactly +# one job, exits, and the host loop starts a fresh one. Nothing on the filesystem crosses jobs. +set -euo pipefail + +# `docker run --device /dev/kvm` creates the node 0660 root:kvm, which the `runner` user cannot +# open: `android-emulator-runner` reads `/dev/kvm` for read and write before it starts anything and +# falls back to a software boot, which is the 474-second one native-platforms.yml's parity lane +# exists to avoid. udev cannot fix it here — no udev runs in the container and its /sys is +# read-only, which is why that lane's `udevadm` lines are tolerant — so the mode is set here, +# before the job can look. Root through the runner image's passwordless sudo; no-op without KVM. +if [ -e /dev/kvm ]; then + sudo chmod 0666 /dev/kvm +fi + +# Fail closed and say which variable is missing. An ephemeral runner that registers without +# TN_RUNNER_REPO would point itself at whatever repository the token defaults to. +: "${RUNNER_ADMIN_TOKEN:?RUNNER_ADMIN_TOKEN is not set; pass the operator env file with --env-file}" +: "${TN_RUNNER_REPO:?TN_RUNNER_REPO is not set; the host loop passes owner/repo}" + +registration_token="$( + curl --fail --silent --show-error --request POST \ + --header "Authorization: Bearer $RUNNER_ADMIN_TOKEN" \ + --header "Accept: application/vnd.github+json" \ + "https://api.github.com/repos/${TN_RUNNER_REPO}/actions/runners/registration-token" \ + | jq --exit-status --raw-output '.token' +)" + +# The admin token mints this one registration token and has no other business existing. Dropping it +# from this shell's environment is what keeps it out of every job `run.sh` hands to a workflow. +unset RUNNER_ADMIN_TOKEN + +./config.sh \ + --unattended \ + --ephemeral \ + --replace \ + --url "https://github.com/${TN_RUNNER_REPO}" \ + --token "$registration_token" \ + --name "tn-$(hostname)" \ + --labels "${TN_RUNNER_LABELS:-tn-local}" \ + --no-default-labels + +exec ./run.sh