diff --git a/.claude/skills/ci-speed-loop/SKILL.md b/.claude/skills/ci-speed-loop/SKILL.md new file mode 100644 index 0000000000..bd3cc1a139 --- /dev/null +++ b/.claude/skills/ci-speed-loop/SKILL.md @@ -0,0 +1,46 @@ +--- +name: ci-speed-loop +description: Run ThreeNative's CI speed loop — measure how long a change takes to merge, name the biggest cost from the first failing jobs and queue wait, change one thing, re-measure the same window, record the iteration and refresh the HTML dashboard. Use when asked to speed up CI or the merge queue, work PRD-550, check progress toward the 30-minute merge goal, or refresh docs/ci-speed/index.html. Not for runtime FPS (perf-loop). +--- + +# CI speed loop + +Goal (PRD-550): a change merges within 30 minutes of its last push. One turn is one measured window, +one named cost, one change, one re-measure. Record a turn that did not move the number too, with the reason. + +| Metric | Goal | +| --- | --- | +| Last push to merge, median / p90 | ≤ 30 / ≤ 45 min | +| Merge queue passes on the first attempt | ≥ 90% | +| Merge-group board wall time, median | ≤ 15 min | +| Runner queue wait, p90 | ≤ 3 min | + +```sh +node scripts/ci-merge-latency.mjs --since 2026-10-09 # measure only, prints the table +node scripts/ci-speed-loop.mjs record --since 2026-10-09 --label "gating board" --notes "..." [--decision keep] +node scripts/ci-speed-loop.mjs show # the ledger +node scripts/ci-speed-loop.mjs discard --iteration 3 --reason "queue outage, 2 PRs only" +node scripts/ci-speed-loop.mjs report # regenerate the HTML only +``` + +`docs/ci-speed/ledger.json` is the record. `docs/ci-speed/index.html` is generated from it and opens with +`file://`. Never edit either by hand. Commit both with the change they measure. + +## The turn + +1. **Measure.** `record --since `. The first iteration is the baseline. Do not + mix PRs from before and after a change in one window. +2. **Name the cost.** Read the "first failing job" chart and the PR table. Name one job or one wait. "CI is slow" is not an answer. + A red that repeats across unrelated PRs is a load red: fix it at its timeout source, not with a retry. +3. **Change one thing.** One PR per PRD (root `AGENTS.md`). Prove it locally first. A CI round trip costs over an hour. +4. **Re-measure the same metrics** once enough PRs merged (the PRD ACs need 10 PRs and 20 merge groups). + A window with fewer than 5 PRs is `provisional`, not `keep`. +5. **Decide.** `keep`, `reject`, `provisional`, `inconclusive` or `invalid`. A contended or outage window is `discard`ed with a reason. It stays in the table and leaves the charts. + +## Rules + +- A value the API cannot supply is `n/a`, never 0. A PR that skipped the queue has no enqueue time. +- Last push is the head commit's committer date, so a rebase moves it. Say so when a number looks low. +- "First attempt" counts the first merge-group run per PR. Cancelled first runs count as not passed. +- Queue wait is job created to job started over all merge-group jobs, including hosted legs. It is not the gating-only wait until PRD-550 phase 3 routes them. +- Do not claim a goal from one window. Cite the iteration id and its PR count. diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 11fad31441..0b4411ee1b 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -98,7 +98,8 @@ jobs: name: csg if: ${{ needs.paths.outputs.csg == 'true' && !github.event.pull_request.draft }} needs: paths - runs-on: ${{ (github.event.pull_request.head.repo.fork || !vars.TN_RUNNER) && 'ubuntu-24.04' || vars.TN_RUNNER }} + # PRD-550: generic lanes run hosted so the local build/test capacity stays free for the heavy jobs. + runs-on: ubuntu-24.04 timeout-minutes: 10 defaults: run: @@ -134,7 +135,7 @@ jobs: name: ik if: ${{ needs.paths.outputs.ik == 'true' && !github.event.pull_request.draft }} needs: paths - runs-on: ${{ (github.event.pull_request.head.repo.fork || !vars.TN_RUNNER) && 'ubuntu-24.04' || vars.TN_RUNNER }} + runs-on: ubuntu-24.04 timeout-minutes: 10 defaults: run: @@ -166,7 +167,7 @@ jobs: name: vegetation if: ${{ needs.paths.outputs.vegetation == 'true' && !github.event.pull_request.draft }} needs: paths - runs-on: ${{ (github.event.pull_request.head.repo.fork || !vars.TN_RUNNER) && 'ubuntu-24.04' || vars.TN_RUNNER }} + runs-on: ubuntu-24.04 timeout-minutes: 10 defaults: run: @@ -198,7 +199,7 @@ jobs: name: decals if: ${{ needs.paths.outputs.decals == 'true' && !github.event.pull_request.draft }} needs: paths - runs-on: ${{ (github.event.pull_request.head.repo.fork || !vars.TN_RUNNER) && 'ubuntu-24.04' || vars.TN_RUNNER }} + runs-on: ubuntu-24.04 timeout-minutes: 30 steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 @@ -314,7 +315,7 @@ jobs: name: exposure if: ${{ needs.paths.outputs.tone == 'true' && !github.event.pull_request.draft }} needs: paths - runs-on: ${{ (github.event.pull_request.head.repo.fork || !vars.TN_RUNNER) && 'ubuntu-24.04' || vars.TN_RUNNER }} + runs-on: ubuntu-24.04 timeout-minutes: 15 steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 @@ -358,7 +359,7 @@ jobs: name: temporal if: ${{ !github.event.pull_request.draft && needs.paths.outputs.temporal == 'true' }} needs: paths - runs-on: ${{ (github.event.pull_request.head.repo.fork || !vars.TN_RUNNER) && 'ubuntu-24.04' || vars.TN_RUNNER }} + runs-on: ubuntu-24.04 timeout-minutes: 20 steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 @@ -405,7 +406,7 @@ jobs: name: velocity if: ${{ !github.event.pull_request.draft && needs.paths.outputs.velocity == 'true' }} needs: paths - runs-on: ${{ (github.event.pull_request.head.repo.fork || !vars.TN_RUNNER) && 'ubuntu-24.04' || vars.TN_RUNNER }} + runs-on: ubuntu-24.04 timeout-minutes: 15 steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 @@ -426,7 +427,10 @@ jobs: - name: Check the exact source-wired fixture types run: pnpm exec tsc --noEmit -p examples/abyss-framework/velocity.tsconfig.json - name: Render actual authored batch velocity and missing-history control - run: pnpm exec tsx scripts/verify-velocity-history.ts + # PRD-550 phase 1. The temporal-off cost arm lives in performance-regression.yml: it prices + # actual frame submission against split-control noise, which a runner shared by twenty jobs + # cannot hold, and its `p95NoiseExcessMs` red was other people's load. + run: pnpm exec tsx scripts/verify-velocity-history.ts --variant=history - name: Retain runtime PNGs, readbacks and exact adapter provenance if: always() uses: actions/upload-artifact@v7 @@ -451,7 +455,7 @@ jobs: name: animation if: ${{ needs.paths.outputs.animation == 'true' && !github.event.pull_request.draft }} needs: paths - runs-on: ${{ (github.event.pull_request.head.repo.fork || !vars.TN_RUNNER) && 'ubuntu-24.04' || vars.TN_RUNNER }} + runs-on: ubuntu-24.04 timeout-minutes: 25 steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 @@ -575,7 +579,7 @@ jobs: name: fog capture if: ${{ needs.paths.outputs.fog == 'true' && !github.event.pull_request.draft }} needs: paths - runs-on: ${{ (github.event.pull_request.head.repo.fork || !vars.TN_RUNNER) && 'ubuntu-24.04' || vars.TN_RUNNER }} + runs-on: ubuntu-24.04 timeout-minutes: 20 steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 @@ -709,7 +713,7 @@ jobs: name: capture if: ${{ !github.event.pull_request.draft && needs.paths.outputs.world-capture == 'true' }} needs: paths - runs-on: ${{ (github.event.pull_request.head.repo.fork || !vars.TN_RUNNER) && 'ubuntu-24.04' || vars.TN_RUNNER }} + runs-on: ubuntu-24.04 timeout-minutes: 20 steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 @@ -759,7 +763,7 @@ jobs: name: fluid collision if: ${{ needs.paths.outputs.fluid-particles == 'true' && !github.event.pull_request.draft }} needs: paths - runs-on: ${{ (github.event.pull_request.head.repo.fork || !vars.TN_RUNNER) && 'ubuntu-24.04' || vars.TN_RUNNER }} + runs-on: ubuntu-24.04 timeout-minutes: 15 steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 @@ -961,7 +965,7 @@ jobs: name: auto-exposure if: ${{ !github.event.pull_request.draft && needs.paths.outputs.exposure == 'true' }} needs: paths - runs-on: ${{ (github.event.pull_request.head.repo.fork || !vars.TN_RUNNER) && 'ubuntu-24.04' || vars.TN_RUNNER }} + runs-on: ubuntu-24.04 timeout-minutes: 30 steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 @@ -1005,7 +1009,7 @@ jobs: name: auto-exposure cold boot if: ${{ !github.event.pull_request.draft && needs.paths.outputs.cold-boot == 'true' }} needs: paths - runs-on: ${{ (github.event.pull_request.head.repo.fork || !vars.TN_RUNNER) && 'ubuntu-24.04' || vars.TN_RUNNER }} + runs-on: ubuntu-24.04 timeout-minutes: 30 steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 diff --git a/.github/workflows/native-platforms.yml b/.github/workflows/native-platforms.yml index 9f25361a85..2a0a2a3f90 100644 --- a/.github/workflows/native-platforms.yml +++ b/.github/workflows/native-platforms.yml @@ -258,7 +258,7 @@ jobs: name: Native collector evidence coverage needs: [scope, android-emulator-parity, desktop-parity, desktop, ios-simulator] if: ${{ !cancelled() && needs.scope.outputs.selection == 'full' }} - runs-on: ${{ (github.event.pull_request.head.repo.fork || !vars.TN_RUNNER) && 'ubuntu-24.04' || vars.TN_RUNNER }} + runs-on: ${{ (github.event.pull_request.head.repo.fork || !vars.TN_RUNNER_LIGHT) && 'ubuntu-24.04' || vars.TN_RUNNER_LIGHT }} timeout-minutes: 5 env: TN_CI_SHA: ${{ needs.scope.outputs.candidate_sha }} @@ -831,7 +831,9 @@ jobs: if: >- needs.scope.outputs.selection != 'prose' && inputs.ios_only == false - runs-on: ${{ (github.event.pull_request.head.repo.fork || !vars.TN_RUNNER) && 'ubuntu-24.04' || vars.TN_RUNNER }} + # Report assembly only - artifact download, `pnpm install` and JSON reports, no compile or GPU + # leg (unlike desktop-parity:906) - so it joins the light lane instead of queuing behind builds. + runs-on: ${{ (github.event.pull_request.head.repo.fork || !vars.TN_RUNNER_LIGHT) && 'ubuntu-24.04' || vars.TN_RUNNER_LIGHT }} timeout-minutes: 20 env: TN_CI_SHA: ${{ needs.scope.outputs.candidate_sha }} diff --git a/.github/workflows/performance-regression.yml b/.github/workflows/performance-regression.yml index 8691e0a880..558fb9e243 100644 --- a/.github/workflows/performance-regression.yml +++ b/.github/workflows/performance-regression.yml @@ -493,3 +493,33 @@ jobs: run: | echo "required-check-promotion requires maintainer review after calibration evidence" echo "baseline-regeneration is a separate-reviewed-change and never runs in this gate" + + velocity-cost: + name: Temporal-off velocity submission cost + # PRD-550 phase 1. This arm prices actual frame submission against split-control noise, so any + # other job on the machine reports its load as render cost. The merge path's integration lane + # therefore runs only `--variant=history`, and this measurement runs on the scheduled lane. + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: ./.github/actions/pnpm + - uses: actions/setup-node@v7 + with: + node-version: 20 + cache: pnpm + - run: pnpm install --frozen-lockfile + - uses: ./.github/actions/playwright-chromium + - name: Build this commit's public bridge and runner + run: pnpm --filter @threenative/playtest build + - name: Measure the temporal-off CPU submission cost + run: pnpm exec tsx scripts/verify-velocity-history.ts --variant=cost + - uses: actions/upload-artifact@v7 + if: always() + with: + name: velocity-cost-${{ github.run_id }}-${{ github.run_attempt }} + path: artifacts/velocity-history/ + retention-days: 30 + if-no-files-found: warn diff --git a/AGENTS.md b/AGENTS.md index 0a4103b09a..d2ddad09f0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -77,7 +77,7 @@ pnpm native:build # opt-in; downloads deps, compiles th pnpm native:verify:desktop # 300 native frames + a non-blank screenshot ``` -**CI.** The pre-push hook runs `pnpm ci:fast` drift checks only; it proves no runtime behaviour. `pnpm ci:local --full` runs the full board; `pnpm ci:local --affected --base origin/develop --target develop` runs what CI would select (dirty or unknown inputs fall back to full). A change touching only Markdown that no fixture consumes (not `AGENTS.md`/`CLAUDE.md`) runs no product CI job. Ordinary PRs into develop use the complete candidate diff: CI-only changes run CI contracts, exact template changes run scaffolder contracts and affected templates, and shared executable inputs keep the full safety board with starter as the default template smoke plus explicitly changed runtime/scenario kits. Template instruction and Markdown changes retain their contract lanes without adding runtime kits. Merge groups, explicit full runs, unknown inputs, main qualification and scheduled audits use fresh exhaustive qualification: all shipped templates, both golden journeys and the full supported native matrix. `ci-required` independently rejects a review plan for these events; Integration is a required reusable join on the same candidate, with each selected canonical job proven by a current-run/attempt receipt and independently checked completed-success API job; missing, skipped, cancelled, stale or unmapped receipts fail closed. Native per-leg authoritative completion and cold-path queue feasibility remain follow-ups before deferring any Android PR coverage. Template scenarios run once per selected kit; develop changes run one representative golden journey and full qualification retains both starter and platformer journeys. `ci-required` rejects missing or failed selected checks; retained contexts include `supply-chain`, `template-nonvisual`, `golden-path`, and native `desktop-parity` when applicable. Native host/platform tests run for native or shared runtime consumers, main qualification and unresolved scopes. Unsafe matrix/reusable verdict reuse stays disabled until authoritative complete coverage and per-leg routing exist. A new candidate runs fresh tests; only exact-run immutable build artifacts are shared. Details: `docs/PRDs/done/PRD-483-impact-driven-ci-without-repeated-template-setup.md` and `docs/PRDs/done/PRD-373-selective-ci-and-develop-promotion.md`. +**CI.** The pre-push hook runs `pnpm ci:fast` drift checks only; it proves no runtime behaviour. `pnpm ci:local --full` runs the full board; `pnpm ci:local --affected --base origin/develop --target develop` runs what CI would select (dirty or unknown inputs fall back to full). A change touching only Markdown that no fixture consumes (not `AGENTS.md`/`CLAUDE.md`) runs no product CI job. Ordinary PRs into develop use the complete candidate diff: CI-only changes run CI contracts, exact template changes run scaffolder contracts and affected templates, and shared executable inputs keep the full safety board with starter as the default template smoke plus explicitly changed runtime/scenario kits. Template instruction and Markdown changes retain their contract lanes without adding runtime kits. Merge groups run the same selected gating board as the PR, so a change merges within 30 minutes (PRD-550). Develop pushes, explicit full runs, unknown inputs, main qualification and scheduled audits use fresh exhaustive qualification: all shipped templates, both golden journeys and the full supported native matrix. `ci-required` independently rejects a review plan for these events; Integration is a required reusable join on the same candidate, with each selected canonical job proven by a current-run/attempt receipt and independently checked completed-success API job; missing, skipped, cancelled, stale or unmapped receipts fail closed. Native per-leg authoritative completion and cold-path queue feasibility remain follow-ups before deferring any Android PR coverage. Template scenarios run once per selected kit; develop changes run one representative golden journey and full qualification retains both starter and platformer journeys. `ci-required` rejects missing or failed selected checks; retained contexts include `supply-chain`, `template-nonvisual`, `golden-path`, and native `desktop-parity` when applicable. Native host/platform tests run for native or shared runtime consumers, main qualification and unresolved scopes. Unsafe matrix/reusable verdict reuse stays disabled until authoritative complete coverage and per-leg routing exist. A new candidate runs fresh tests; only exact-run immutable build artifacts are shared. Details: `docs/PRDs/done/PRD-483-impact-driven-ci-without-repeated-template-setup.md` and `docs/PRDs/done/PRD-373-selective-ci-and-develop-promotion.md`. Registry commands read auth from `~/.npmrc`; pass a checkout-local untracked `.npmrc` with `npm --userconfig .npmrc `. Never print either file. TypeScript 7 `strict` (the native `tsc`); `typescript` imports use the official TypeScript 6 compatibility API for AST scripts and tsup declarations, **ESM only**; relative imports end in `.js`. Versions come from the `catalog:` in `pnpm-workspace.yaml`, templates excepted. Biome formats — never hand-format. Interfaces are `I`-prefixed; classes and type aliases are not. Unit tests: `/__tests__/*.spec.ts`, vitest, node environment, DOM and GPU stubbed. diff --git a/CLAUDE.md b/CLAUDE.md index 936b6a54c4..7afe3b15ba 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -79,7 +79,7 @@ pnpm native:build # opt-in; downloads deps, compiles th pnpm native:verify:desktop # 300 native frames + a non-blank screenshot ``` -**CI.** The pre-push hook runs `pnpm ci:fast` drift checks only; it proves no runtime behaviour. `pnpm ci:local --full` runs the full board; `pnpm ci:local --affected --base origin/develop --target develop` runs what CI would select (dirty or unknown inputs fall back to full). A change touching only Markdown that no fixture consumes (not `AGENTS.md`/`CLAUDE.md`) runs no product CI job. Ordinary PRs into develop use the complete candidate diff: CI-only changes run CI contracts, exact template changes run scaffolder contracts and affected templates, and shared executable inputs keep the full safety board with starter as the default template smoke plus explicitly changed runtime/scenario kits. Template instruction and Markdown changes retain their contract lanes without adding runtime kits. Merge groups, explicit full runs, unknown inputs, main qualification and scheduled audits use fresh exhaustive qualification: all shipped templates, both golden journeys and the full supported native matrix. `ci-required` independently rejects a review plan for these events; Integration is a required reusable join on the same candidate, with each selected canonical job proven by a current-run/attempt receipt and independently checked completed-success API job; missing, skipped, cancelled, stale or unmapped receipts fail closed. Native per-leg authoritative completion and cold-path queue feasibility remain follow-ups before deferring any Android PR coverage. Template scenarios run once per selected kit; develop changes run one representative golden journey and full qualification retains both starter and platformer journeys. `ci-required` rejects missing or failed selected checks; retained contexts include `supply-chain`, `template-nonvisual`, `golden-path`, and native `desktop-parity` when applicable. Native host/platform tests run for native or shared runtime consumers, main qualification and unresolved scopes. Unsafe matrix/reusable verdict reuse stays disabled until authoritative complete coverage and per-leg routing exist. A new candidate runs fresh tests; only exact-run immutable build artifacts are shared. Details: `docs/PRDs/done/PRD-483-impact-driven-ci-without-repeated-template-setup.md` and `docs/PRDs/done/PRD-373-selective-ci-and-develop-promotion.md`. +**CI.** The pre-push hook runs `pnpm ci:fast` drift checks only; it proves no runtime behaviour. `pnpm ci:local --full` runs the full board; `pnpm ci:local --affected --base origin/develop --target develop` runs what CI would select (dirty or unknown inputs fall back to full). A change touching only Markdown that no fixture consumes (not `AGENTS.md`/`CLAUDE.md`) runs no product CI job. Ordinary PRs into develop use the complete candidate diff: CI-only changes run CI contracts, exact template changes run scaffolder contracts and affected templates, and shared executable inputs keep the full safety board with starter as the default template smoke plus explicitly changed runtime/scenario kits. Template instruction and Markdown changes retain their contract lanes without adding runtime kits. Merge groups run the same selected gating board as the PR, so a change merges within 30 minutes (PRD-550). Develop pushes, explicit full runs, unknown inputs, main qualification and scheduled audits use fresh exhaustive qualification: all shipped templates, both golden journeys and the full supported native matrix. `ci-required` independently rejects a review plan for these events; Integration is a required reusable join on the same candidate, with each selected canonical job proven by a current-run/attempt receipt and independently checked completed-success API job; missing, skipped, cancelled, stale or unmapped receipts fail closed. Native per-leg authoritative completion and cold-path queue feasibility remain follow-ups before deferring any Android PR coverage. Template scenarios run once per selected kit; develop changes run one representative golden journey and full qualification retains both starter and platformer journeys. `ci-required` rejects missing or failed selected checks; retained contexts include `supply-chain`, `template-nonvisual`, `golden-path`, and native `desktop-parity` when applicable. Native host/platform tests run for native or shared runtime consumers, main qualification and unresolved scopes. Unsafe matrix/reusable verdict reuse stays disabled until authoritative complete coverage and per-leg routing exist. A new candidate runs fresh tests; only exact-run immutable build artifacts are shared. Details: `docs/PRDs/done/PRD-483-impact-driven-ci-without-repeated-template-setup.md` and `docs/PRDs/done/PRD-373-selective-ci-and-develop-promotion.md`. Registry commands read auth from `~/.npmrc`; pass a checkout-local untracked `.npmrc` with `npm --userconfig .npmrc `. Never print either file. TypeScript 7 `strict` (the native `tsc`); `typescript` imports use the official TypeScript 6 compatibility API for AST scripts and tsup declarations, **ESM only**; relative imports end in `.js`. Versions come from the `catalog:` in `pnpm-workspace.yaml`, templates excepted. Biome formats — never hand-format. Interfaces are `I`-prefixed; classes and type aliases are not. Unit tests: `/__tests__/*.spec.ts`, vitest, node environment, DOM and GPU stubbed. diff --git a/docs/PRDs/CI/PRD-550-a-change-merges-within-30-minutes.md b/docs/PRDs/CI/PRD-550-a-change-merges-within-30-minutes.md index a23acca745..a499bf2c45 100644 --- a/docs/PRDs/CI/PRD-550-a-change-merges-within-30-minutes.md +++ b/docs/PRDs/CI/PRD-550-a-change-merges-within-30-minutes.md @@ -121,22 +121,38 @@ qualifying run. A PR can still opt in to qualifying on the merge path with a lab ## Execution Phases #### Phase 1: No load reds in the gate -**Status:** NOT STARTED +**Status:** IN PROGRESS **Files:** `packages/create-threenative/__tests__/fixtures/auto-exposure/proof.ts`, `.github/workflows/integration.yml`, `.github/workflows/performance-regression.yml`, the `test-unit` specs that phase 1 classifies as load reds. -**Implementation:** Wait on rendered updates with a timeout scaled to the measured frame rate, not a fixed 60 s. Move `velocity` from `integration.yml` to `performance-regression.yml`. For each repeated `test-unit` red, read three failing logs; a red that also fails on an idle `tn-local` run is a code red and leaves this PRD. +**Implementation:** Wait on rendered updates with a timeout scaled to the measured frame rate, not a fixed 60 s. Move `velocity` from `integration.yml` to `performance-regression.yml`. For each repeated `test-unit` red, read three failing logs; a red that also fails on an idle `tn-local` run is a code red and leaves this PRD. Committed so far: the temporal-off cost arm. The two red merge groups of 2026-10-09 (37874560157, 37883642740) both fail first on `integration / velocity` with `TN_PLAYTEST_RESOURCE_ASSERTION_FAILED … 'p95NoiseExcessMs'` (`scripts/verify-velocity-history.ts:189`), which is a measurement of noise on a runner twenty other jobs share. That arm now runs on the scheduled `performance-regression.yml` lane (`--variant=cost`) and the merge path runs only the correctness arm (`--variant=history`), so the lane keeps typing and unit-checking the cost code without measuring on a loaded machine. The auto-exposure frame-rate budget and the `test-unit` triage are still open, and all three boxes below need a CI run id. +**Load-red triage** (2026-10-09): three failing `test-unit` shards (jobs 113669815836, 113540990094, 113540990242) hold seven failing items and every one is a wall-clock expiry or a native-host wait — `Hook timed out in 30000ms`, `Test timed out in 15000ms`, two builds killed at a 60 s ceiling, `world-gpu-scene.spec.ts` and `template-assets-compile-1.spec.ts` at 60 s, `TN_PLAYTEST_NATIVE_SCREENSHOT_UNAVAILABLE` — so no code red leaves this PRD from this sample, and one repeated signature, `TN_CSS_SOURCEMAP_AMBIGUOUS`, is explained by no failing item. On an idle machine all four named specs pass (125 cases, exit 0, 218 s). The compile specs are the ones whose budget does not fit their work: 193.7 s and 215.0 s for 22 and 18 cases, 25–45 s per template against the shared 60 s default, now 180 s per case. `world-gpu-scene.spec.ts` is 70 cases in 60.7 s and `check-capability-docs.spec.ts` 15 in 13.9 s, so their reds are the shared-runner class — phase 3 capacity, not a timeout. +**Budget note (2026-10-10):** the `should not ship recast in a build that never imports the navigation entry` case in `packages/create-threenative/__tests__/scaffold.spec.ts` carried an incidental `15_000` timeout added by the PRD-036 replay-review commit `4956609cf`; the shared default is the existing `testTimeout: 60_000` hang detector (`vitest.config.ts:26`). That override expired at 15 s on a genuine CI red ([run 38066515075 job 114255822601](https://github.com/ThreeNativeHQ/threenative-engine/actions/runs/38066515075/job/114255822601)) before the artifact assertions ran, while the case passes locally in 1.152 s. Only the override is removed to inherit the shared budget; assertions, helper and config are unchanged. Focused file after the change: `pnpm exec vitest run packages/create-threenative/__tests__/scaffold.spec.ts` 66/66 passed, 8.73 s. Why CI slowed is unproved; fresh CI and merge latency stay open. +**Import note (2026-10-10):** a CI red ([run 38069329212 job 114264328770](https://github.com/ThreeNativeHQ/threenative-engine/actions/runs/38069329212/job/114264328770)) killed the `does not launch the capture workflow when imported for its classifier` case in `scripts/__tests__/exposure-ab.spec.ts` at its 5 s child timeout; the child import failed at 5.444 s under the unchanged 5 s subprocess budget. The three capture-only imports (`@playwright/test`, `playtest/src/runner/browser.js`, `runtime-native/conformance/metrics.mjs`) move to dynamic `import()` in `main`; the budget, thresholds, assertions and CLI guard are unchanged. Focused 7/7 passed, 1.62 s; smoke 639 ms; typecheck, lint and docs passed. CLI capture exit 0: control mismatch 0 / deltaE 0, exposure mismatch 1 / deltaE 5.81; `adapter` `{}` so hardware identity is unverified. Fresh CI stability open. + - [ ] `integration / auto-exposure` passes 10 of 10 runs on a `tn-local` slot under a parallel full board. proof: CI run ids. - [ ] `velocity` runs on the hardware lane and is absent from `merge_group` jobs. proof: `performance-regression.yml` run id plus a merge-group job list. - [ ] No `test-unit` spec times out in 10 consecutive merge groups. proof: `node scripts/ci-merge-latency.mjs` first-failing-job column. #### Phase 2: One gating board, run once -**Status:** NOT STARTED +**Status:** IN PROGRESS **Files:** `.github/workflows/ci.yml`, `scripts/ci-change-scope.mjs`, `scripts/ci-required.mjs`, `.github/workflows/ci-janitor.yml`, root `AGENTS.md` (+ `pnpm sync:agents`), `scripts/__tests__/ci-structure.spec.ts`. -**Implementation:** Add a `gating` scope for `merge_group`; keep `full` for develop push, schedule, `workflow_dispatch` and `main`. Prove `test-unit` shard expansion from the run's job list and drop it from `UNPROVEN_REUSE_BOARDS`. Extend `ci-janitor.yml` to open an issue and a revert PR on a red qualifying `develop` run. +**Implementation:** Add a `gating` scope for `merge_group`; keep `full` for develop push, schedule, `workflow_dispatch` and `main`. Prove `test-unit` shard expansion from the run's job list and drop it from `UNPROVEN_REUSE_BOARDS`. Extend `ci-janitor.yml` to open an issue and a revert PR on a red qualifying `develop` run. Committed so far: a `merge_group` on develop selects the same narrowed plan a pull request earns (it reviews the exact tree a merge would produce) and a develop push selects `full`, so the exhaustive board moved from the queue to the promotion. Both outcome boxes below stay open until a CI run proves them. +**Repair (2026-10-09):** `validationProfile` compared a raw `baseRef` against `main` and then ORed any `merge_group` to `develop`, so an under-scoped review plan was accepted for `refs/heads/main` and for unknown or missing queue targets. The shared fix normalizes `refs/heads/` once inside `validationProfile` and classifies only explicit `main`/`develop`, removing the blanket `eventName === "merge_group"` fallback so an unproven target stays `other`. Reproduction: the committed regression test in `scripts/__tests__/ci-qualification.spec.ts`. The gate callers (`validateEventPlan`, `sourceVerdict`, `currentRun`) keep explicit develop/main behavior. +**Shared assertion (2026-10-10):** the pull-request base/head parent assertion moved out of `ci-required.mjs` into `scripts/ci-change-scope.mjs` as `assertMergeParentCandidate`, called by `ci-required` in the same place with every condition, error code and error behavior preserved. The shared helper lets an early classifier reject a wrong-parent candidate before the fanout instead of after it. The guard runs only for the live GitHub projection (`--format github`, `pull_request`, not `--local`); local and offline classification keep their existing behavior. Proof: the focused scope, template-selection, efficiency, qualification, structure, needs, runners and dashboard specs pass 431/431 in 8 files (`/tmp/pr476-preflight-focused.log`). Red before green: against the original unguarded classifier, before the guard existed, the new stale-base test failed (1 failed, 53 skipped; the classifier exited 0) (`/tmp/opencode/ci-stale-base-red.log`); the added guard makes it pass. Biome reports 0 errors on the 3 changed code files (4 pre-existing complexity warnings), and `node --check` passes on both JS files. Actual gate note: `pnpm lint` and `pnpm typecheck` passed after rebuilding stale core declarations and running the shipped strata bake; the full local suite remains unqualified; the live CI outcome is pending. The live early-rejection outcome and the phase latency boxes stay open. +- [x] `main` (bare/prefixed) and unknown/missing queue targets reject review plans; `develop` bare/prefixed preserves the valid reduced review. proof: `pnpm exec vitest run scripts/__tests__/ci-qualification.spec.ts` 19/19 after fix; new test was red before fix (18 passed, 1 failed). Profile caller/receipt/template selection specs 281/281 and `ci-structure.spec.ts` 159/159. Broader local gates: `pnpm lint` and `pnpm typecheck` passed; `pnpm test` failed (491 s) in 22 unchanged native cases, including an unexplained 120 s dry-run timeout, missing host binaries, and a missing mailbox response. Fresh shared CI passed on candidate `77bb0ee577ba39c514cd1262c48c8291f2210de2`: [run 38033595736, attempt 1](https://github.com/ThreeNativeHQ/threenative-engine/actions/runs/38033595736), 69 successful jobs including `ci-required`, completed 2026-10-10T07:56:51Z. This single PR run does not close the phase or its repeated merge-group requirements. - [ ] A merge group runs only gating jobs and finishes in ≤ 15 min wall. proof: CI run id. - [ ] A red qualifying run on `develop` opens an issue and a revert PR. proof: `ci-janitor.yml` run id on a seeded red branch. #### Phase 3: Capacity and the latency audit -**Status:** NOT STARTED -**Files:** `scripts/ci-merge-latency.mjs` (new), `.github/workflows/ci.yml` `runs-on` expressions, `docs/PRDs/CI/EXECUTION-ORDER.md`. -**Implementation:** Route gating jobs to `vars.TN_RUNNER` with hosted fallback; route qualifying jobs to hosted. Write the audit script with `gh api` and no new dependency. +**Status:** IN PROGRESS +**Files:** `scripts/ci-merge-latency.mjs` (new), `scripts/ci-speed-loop.mjs` (new), `docs/ci-speed/` (ledger and HTML dashboard), `.claude/skills/ci-speed-loop/`, `.github/workflows/ci.yml` and `.github/workflows/integration.yml` `runs-on` expressions, `scripts/__tests__/ci-structure.spec.ts`, `docs/PRDs/CI/EXECUTION-ORDER.md`. +**Implementation:** Route gating jobs to `vars.TN_RUNNER` with hosted fallback; route qualifying jobs to hosted. Write the audit script with `gh api` and no new dependency. `scripts/ci-speed-loop.mjs` records each audit window in `docs/ci-speed/ledger.json` and renders `docs/ci-speed/index.html`, a dashboard of every metric against its goal (skill: `ci-speed-loop`). +- [x] The audit and the dashboard exist, reproduce this PRD's own table, and hold the baseline. proof: `pnpm exec vitest run scripts/__tests__/ci-speed-loop.spec.ts` 14 of 14; `node scripts/ci-merge-latency.mjs --since 2026-10-08` gives #461 104.2, #462 242.9, #457 432.7, #449 593.9, #455 626.9, #456 892.1, #465 131.9 min enqueue → merge (this PRD's table) and 8 `auto-exposure` first failures; ledger iteration #1 = baseline. +**Investigation (2026-10-10):** The current hosted placement of the `template-nonvisual` template legs is intentional — the workflow comment pins them to hosted on purpose, to keep one routing variable from sending the whole burst to a single pool. No routing change is accepted pending queue and capacity proof. A rejected attempt to move the job onto `vars.TN_RUNNER` was reverted in full (`.github/workflows/ci.yml`, `scripts/__tests__/ci-structure.spec.ts`); nothing about this job's routing is changed here. +**Repair (2026-10-10, balancer state):** `scripts/ci-runners.sh` `balance` caches `have_heavy`/`have_light` from its own last write, so a routing variable deleted outside the loop (author unknown; `up` documents the balancer as owner and `down` stops the balancer before it clears them) stays unset while the pool is idle and capacity routes to hosted. In scope: `scripts/ci-runners.sh` (`balance` only) and `scripts/__tests__/ci-runners.spec.ts`. Failure hypothesis: a cached `have_*` never re-reads the actual variable, so an external delete is invisible. Fix: one bounded `gh api repos/$repo/actions/variables` read per poll filtered to the two known routing names, exact expected-label compare, re-advertise while capacity stays debounced-on, skip advertising when the read fails. Live effect observed 2026-10-10T13:34:48Z: both routing variables are restored to their expected labels (`tn-local`, `tn-local-light`) with one `balance` controller running. On candidate `d6c1af145` (run 38056537726) the local build, test, integration and native jobs did run in CI, so variable restoration and job uptake are now observed, while latency acceptance stays open; the `exposure` lane measured 19.5 min in queue. That board failed cold boot: `verifySnap` timed out with `state.cutSampleFrames` expected 180 but last 89 in 60 sec, while the 20-launch `verifyColdBoot` check passed. Whether the cause is speed or the game is unproved and no assertion is loosened. The Windows failure triage is pending, so no full-green board is claimed. The local fake-`gh` harness stays the only behavioural proof of the balancer itself. +**Placement (2026-10-10):** On candidate `d6c1af145` (run 38056537726) the generic Integration lanes shared the six local heavy slots with the builds, test shards and native host builds, so a lane assigned at 13:42:05 waited to 14:01:37 (~19.5 min) and cold-boot to 13:59:06 (~17 min); four test/browser/budgets jobs stayed queued after the controller withdrew `TN_RUNNER` at 13:42:07, which cannot move an already assigned job. The generic Integration lanes now pin `ubuntu-24.04`, splitting hosted integration from the local build/test pool without oversubscription: `paths` and `completion` keep `TN_RUNNER_LIGHT`, every native ARM label, job condition, receipt and reusable input is unchanged, and no variable, input or helper is added. The deliberate `template-nonvisual` hosted split in `ci.yml` is untouched. +- [x] `balance` re-advertises a routing variable deleted while idle capacity stays debounced-on, clears a wrong-label variable while the pool is busy, corrects a wrong label to the expected one once idle is confirmed, skips advertising when the variable list is unreadable, and never re-advertises after the stop file exists. proof: `pnpm exec vitest run scripts/__tests__/ci-runners.spec.ts` 16/16 green after fix; original pre-fix harness was red (4 failed, 10 passed), including the external-delete re-advertisement case. The two wrong-label cases additionally caught an intermediate implementation error (1 failed, 1 passed), not a separate original-HEAD regression; `ci-structure.spec.ts` 159/159 and `ci-needs.spec.ts` 47/47 green; five focused CI/dashboard specs 256/256, lint and typecheck passed; full local `pnpm test` failed in 21 native-host/mailbox cases (159 s), so no local full-suite pass is claimed. Fresh read-only review passed. Logs `/tmp/pr476-balancer-red.log`, `/tmp/pr476-balancer-invalid-green.log`; live pool effect unverified. +- [x] Generic Integration lanes pin GitHub-hosted `ubuntu-24.04` while `paths`/`completion` keep the light switch and native ARM legs keep their label. proof: `pnpm exec vitest run scripts/__tests__/ci-structure.spec.ts` contract green after the pre-change red; live queue-wait effect unverified. +**Placement (2026-10-10, release reports):** On candidate `d6c1af145` (run 38056537726) the native `release-reports` job queued from 14:23:09 behind the heavy build pool while light slots stayed idle, though it only checks out, `pnpm install`s, downloads the android/desktop parity artifacts and writes JSON reports — no compile or GPU leg (unlike `desktop-parity`, which builds). It now selects `TN_RUNNER_LIGHT`; its condition, timeout, steps, receipts and every native/parity build label are unchanged, and no variable, input or helper is added. +- [x] `release-reports` joins the light lane while native build and parity legs keep their heavy routing. proof: `pnpm exec vitest run scripts/__tests__/ci-structure.spec.ts` — contract red before the one-line workflow change (`release-reports is off the light lane`, 1 failed / 159 passed) and green after (160 passed); live queue-wait effect unverified. +**Placement (2026-10-10, collector coverage):** Live run `38069329212` showed the native `performance-coverage` job still queued on `tn-local` at 17:29:25 for at least 7 min while all five heavy slots were busy and three light slots idle, though it only checks out, `pnpm install`s, downloads the `native-*` artifacts and runs `tsx scripts/performance-regression/ci-summary.ts` — a JSON aggregator with no compile or GPU leg. It now selects `TN_RUNNER_LIGHT`; its condition, timeout, steps, receipts, coverage list and every native/parity build label are unchanged, and no variable, input or helper is added. Test-first: the exact routing contract was red before the one-line change (`performance-coverage is off the light lane`, 1 failed / 159 skipped, `/tmp/pr476-collector-route-red.log`) and the five focused CI specs run 331/331 after (`/tmp/pr476-collector-route-green.log`); `biome check scripts/__tests__/ci-structure.spec.ts` exit 0 (pre-existing complexity warnings only) and `git diff --check` clean. New-CI outcome pending, so no box is closed. - [ ] Gating jobs in a merge group wait ≤ 3 min p90 for a runner. proof: `node scripts/ci-merge-latency.mjs` queue column. diff --git a/docs/ci-speed/index.html b/docs/ci-speed/index.html new file mode 100644 index 0000000000..51542cd4b6 --- /dev/null +++ b/docs/ci-speed/index.html @@ -0,0 +1,83 @@ + + +CI speed loop +
+

CI speed loop

+

Latest live iteration #20 of 20: Current queue comparison — PR476 still unmerged. Snapshot 2026-10-10T13:30:11.249Z. Regenerate with node scripts/ci-speed-loop.mjs report, then reload; the page is a static render of docs/ci-speed/ledger.json. Goal: a change merges within 30 minutes of its last push (PRD-550).

+

Progress toward the goal

+
+ + + + +
MetricBaselineNowGoalProgress
Last push to merge, median280.4 min474.3 min≤ 30 min 0%open
Last push to merge, p90838.3 min1071.3 min≤ 45 min 0%open
Merge queue passes on the first attempt33.3%22.2%≥ 90 % 0%open
Merge-group board wall time, median77.8 min95.1 min≤ 15 min 0%open
Runner queue wait, p9041.4 min39.6 min≤ 3 min 5%open
+

Trend per iteration

+
Last push to merge, median (lower is better)
+ + + +30 +goal ≤ 30 min +745 +#1: 280.4 min#1#5: 474.3 min#5#6: 475.9 min#6#7: 475.9 min#7#8: 475.9 min#8#9: 475.9 min#9#10: 475.9 min#10#11: 475.9 min#11#12: 475.9 min#12#13: 475.9 min#13#14: 475.9 min#14#15: 475.9 min#15#16: 477.4 min#16#17: 477.4 min#17#18: 744.5 min#18#19: 475.9 min#19#20: 474.3 min#20
Last push to merge, p90 (lower is better)
+ + + +45 +goal ≤ 45 min +1130 +#1: 838.3 min#1#5: 474.3 min#5#6: 477.1 min#6#7: 477.1 min#7#8: 477.1 min#8#9: 477.1 min#9#10: 477.1 min#10#11: 477.1 min#11#12: 477.1 min#12#13: 477.1 min#13#14: 477.1 min#14#15: 477.1 min#15#16: 1023.9 min#16#17: 1023.9 min#17#18: 1130.1 min#18#19: 1090.9 min#19#20: 1071.3 min#20
Merge queue passes on the first attempt (higher is better)
+ + + +90 +goal ≥ 90 % + +#1: 33.3%#1#5: 0%#5#6: 50%#6#7: 50%#7#8: 50%#8#9: 50%#9#10: 50%#10#11: 50%#11#12: 50%#12#13: 50%#13#14: 50%#14#15: 50%#15#16: 40%#16#17: 40%#17#18: 33.3%#18#19: 25%#19#20: 22.2%#20
Merge-group board wall time, median (lower is better)
+ + + +15 +goal ≤ 15 min +119 +#1: 77.8 min#1#2: 119.1 min#2#3: 113.3 min#3#4: 113.3 min#4#5: 111.6 min#5#6: 109.9 min#6#7: 109.9 min#7#8: 109.9 min#8#9: 109.9 min#9#10: 109.9 min#10#11: 102.5 min#11#12: 95.1 min#12#13: 95.1 min#13#14: 95.1 min#14#15: 95.1 min#15#16: 95.1 min#16#17: 95.1 min#17#18: 107.9 min#18#19: 107.9 min#19#20: 95.1 min#20
Runner queue wait, p90 (lower is better)
+ + + +3 +goal ≤ 3 min +51 +#1: 41.4 min#1#2: 50.9 min#2#3: 49.9 min#3#4: 46.9 min#4#5: 47.2 min#5#6: 45 min#6#7: 42.9 min#7#8: 41.5 min#8#9: 40.7 min#9#10: 39 min#10#11: 38.9 min#11#12: 38.7 min#12#13: 38.5 min#13#14: 38.7 min#14#15: 38.6 min#15#16: 36.9 min#16#17: 41.5 min#17#18: 42.3 min#18#19: 42 min#19#20: 39.6 min#20
First failing job per red merge group, latest iteration
integration / velocity7
test-unit (2/4)5
test-unit (3/4)5
test-browser3
golden-path-template (platformer)2
integration / auto-exposure2
integration / auto-exposure cold boot2
native-platforms / macOS desktop core2
+

Iterations

+
#DateCommitChangeDecisionWindow fromPush→merge p50Push→merge p90First attemptBoard wallQueue wait p90
#12026-10-0911cafc003baseline: exhaustive board on every merge group
Before PRD-550. 2026-10-08 to 2026-10-09 window, same period as the PRD Context.		baseline2026-10-08280.4838.333.3%77.841.4
#22026-10-0990dee0c83no change landed: the queue still runs the exhaustive board
Window since 2026-10-09. 0 PRs merged, so push->merge, p90 and first-attempt are n/a; board wall and queue wait measure the exhaustive merge-group board this change removes. Iteration #1 stays the baseline. Re-measure once 5 or more PRs merged.		provisional2026-10-09n/an/an/a119.150.9
#32026-10-09a47851026phase 1 in review
Same window as iteration 2: 0 PRs merged, 9 merge-group runs, board median 113.3 min, queue wait p90 49.9 min. Named cost: integration / velocity was the first failing job in 2 of 4 red merge groups, because its cost arm priced shared-runner load as render cost. The change is PRD-550 phase 1 (481824f3c, a47851026) on PR 476. It moves that arm to a scheduled lane and widens two compile budgets. The compile specs pass locally, 40 of 40 in 220 s. Re-measure after it merges. The PR head also shows an intermittent iOS simulator red, seen in 2 of 10 recent runs with different errors, on code this branch does not change.		provisional2026-10-09n/an/an/a113.349.9
#42026-10-096ec34f52cPR 476 green on phase 1 head
Same window: 0 PRs merged, 12 merge-group runs. PR 476 head 6ec34f52c is green: 69 checks pass and 12 skip, and merge state is CLEAN. integration / velocity passes on that head, and the iOS simulator job passes there too, so the iOS red from iteration 3 did not recur. Merge-group metrics do not move until phase 1 reaches develop. Named costs still on develop: integration / velocity and test-unit (3/4), each first in 2 red merge groups. The compile-spec budget change (a47851026) targets the test-unit timeouts. Re-measure after the merge.		provisional2026-10-09n/an/an/a113.346.9
#52026-10-093ca761875hourly refresh
One PR merged since the last tick: 458, last push to merge 474.3 min, first attempt failed at test-unit (2/4). The window now has 13 merge-group runs, board median 111.6 min, and queue wait p90 47.2 min over 885 jobs. First failing job in the red groups: integration / velocity 2, test-unit (3/4) 2, test-unit (2/4) 1, integration / capture 1, test-playtest 1. PR 476 is still open, so phase 1 is not on develop yet. Provisional: fewer than 5 PRs merged.		provisional2026-10-09474.3474.30%111.647.2
#62026-10-097789bf328hourly refresh
One new PR merged since the last tick, 470, after 4 merge-group attempts. Its first failing job was integration / velocity. The window now has 2 merged PRs, 470 and 458. Last push to merge median 475.9 min, p90 477.1 min. First attempt merge 50 percent. 16 merge-group runs, board median 109.9 min, queue wait p90 45 min over 1086 jobs. First failing job in the red groups: integration / velocity 2, test-unit (3/4) 2, integration / capture 1, template-nonvisual platformer 1. PR 476 is still open, so phase 1 is not on develop yet. Provisional: fewer than 5 PRs merged.		provisional2026-10-09475.9477.150%109.945
#72026-10-09aa13e02c1hourly refresh
No new PR merged since the last tick. The window still has 2 merged PRs, 470 and 458. It now has 18 merge-group runs, up from 16. Board median 109.9 min. Queue wait p90 42.9 min over 1227 jobs. Run success fell to 31.3 percent. First failing job in the 8 red groups: integration / velocity 4, test-unit (3/4) 2, integration / capture 1, template-nonvisual platformer 1. integration / velocity is the largest named cost, and phase 1 on PR 476 targets it. PR 476 is still open, so phase 1 is not on develop yet. Provisional: fewer than 5 PRs merged.		provisional2026-10-09475.9477.150%109.942.9
#82026-10-099f36ac3c8hourly refresh
No new PR merged since the last tick. The window still has 2 merged PRs, 470 and 458. It now has 20 merge-group runs, up from 18. Run success fell to 27.8 percent. Board median 109.9 min. Queue wait p90 41.6 min over 1376 jobs. First failing job in the red groups: integration / velocity 4, test-unit (3/4) 3, test-browser 2, integration / capture 1. test-browser is new in this list. integration / velocity is still the largest named cost, and phase 1 on PR 476 targets it. PR 476 is still open. Provisional: fewer than 5 PRs merged.		provisional2026-10-09475.9477.150%109.941.5
#92026-10-09cbd2e662ehourly refresh
No new PR merged since the last tick. The window still has 2 merged PRs, 470 and 458. It now has 24 merge-group runs, up from 20. Run success fell to 23.8 percent. Board median 109.9 min. Queue wait p90 41.2 min over 1579 jobs. First failing job in the red groups: integration / velocity 4, test-unit (3/4) 4, test-browser 2, integration / auto-exposure cold boot 1. integration / velocity and test-unit (3/4) are tied as the largest named costs. Phase 1 on PR 476 targets integration / velocity, and its compile-spec budget change targets test-unit timeouts. PR 476 is still open. Provisional: fewer than 5 PRs merged.		provisional2026-10-09475.9477.150%109.940.7
#102026-10-092e100adc4hourly refresh
No new PR merged since the last tick. The window still has 2 merged PRs, 470 and 458. It now has 26 merge-group runs, up from 24. Run success fell to 20.8 percent, the fourth straight drop. Board median 109.9 min. Queue wait p90 39 min over 1785 jobs, down from 41.2. First failing job in the red groups: test-unit (3/4) 5, integration / velocity 4, test-browser 2, integration / auto-exposure cold boot 1. test-unit (3/4) is now the largest named cost. Phase 1 on PR 476 includes a compile-spec budget change for test-unit timeouts, and it is not on develop yet. PR 476 is still open. Provisional: fewer than 5 PRs merged.		provisional2026-10-09475.9477.150%109.939
#112026-10-09f9a01bbe4hourly refresh
No new PR merged since the last tick. The window still has 2 merged PRs, 470 and 458. It now has 29 merge-group runs, up from 26. Run success is 22.2 percent, up from 20.8. Board median is 102.5 min, down from 109.9. Queue wait p90 is 38.4 min over 1993 jobs. First failing job in the 12 red groups: test-unit (3/4) 5, integration / velocity 4, test-browser 2, golden-path template platformer 1. The named costs are unchanged. A separate develop push run at b12b257f1 failed in test-native, and it is not counted in these groups. PR 476 is still open. Provisional: fewer than 5 PRs merged.		provisional2026-10-09475.9477.150%102.538.9
#122026-10-09f1e868d47hourly refresh
Catch-up tick after the 09:00 stop. No new PR merged. The window still has 2 merged PRs, 470 and 458. It now has 31 merge-group runs, up from 29. Run success is 24.1 percent, up from 22.2. Board median is 95.1 min, down from 102.5. Queue wait p90 is 38.7 min over 2118 jobs. First failing job in the 13 red groups: test-unit (3/4) 5, integration / velocity 4, test-browser 2, test-unit (1/4) 2. test-unit (1/4) is new in this list. The named costs are unchanged. PR 476 is still open. Provisional: fewer than 5 PRs merged.		provisional2026-10-09475.9477.150%95.138.7
#132026-10-091310367b6hourly refresh
No new PR merged since the last tick. The window still has 2 merged PRs, 470 and 458. It now has 33 merge-group runs, up from 31. Run success is 22.6 percent, down from 24.1. Board median is 95.1 min, unchanged. Queue wait p90 is 38.5 min over 2258 jobs. First failing job in the 14 red groups: integration / velocity 5, test-unit (3/4) 5, test-browser 2, test-unit (1/4) 2. integration / velocity rose by one. PR 476 is still open. Provisional: fewer than 5 PRs merged.		provisional2026-10-09475.9477.150%95.138.5
#142026-10-096639a5bafhourly refresh
No new PR merged since the last tick. The window still has 2 merged PRs, 470 and 458. It now has 35 merge-group runs, up from 33. Run success is 21.2 percent, down from 22.6. Board median is 95.1 min, unchanged. Queue wait p90 is 38.7 min over 2396 jobs. First failing job in the 15 red groups: integration / velocity 5, test-unit (3/4) 5, test-unit (2/4) 3, integration / auto-exposure cold boot 2. The list changed: test-unit (2/4) and auto-exposure cold boot appear, and test-browser and test-unit (1/4) drop out. The first attempt hit a transient GitHub 502, and the rerun succeeded. Develop still sits at b12b257f1, so the queue has no new merges. PR 476 is still open. Provisional: fewer than 5 PRs merged.		provisional2026-10-09475.9477.150%95.138.7
#152026-10-094c1bbd890hourly refresh
No new PR merged since the last tick. The window still has 2 merged PRs, 470 and 458. Develop still sits at b12b257f1, so the queue has no new merges. It now has 36 merge-group runs, up from 35. Run success is 20 percent, down from 21.2. Board median is 95.1 min, unchanged. Queue wait p90 is 38.6 min over 2476 jobs. First failing job in the 16 red groups: integration / velocity 5, test-unit (3/4) 5, test-browser 3, test-unit (2/4) 3. test-browser is back in the list. PR 476 is still open. Provisional: fewer than 5 PRs merged.		provisional2026-10-09475.9477.150%95.138.6
#162026-10-094d81de73chourly refresh
The window merged five PRs, which are 458, 470, 471, 473, and 468, and this matches the merged PR line that shows 5 PRs. The window held 42 merge group runs, with a run success of 19.4 percent and a first attempt merge of 40 percent. The merge group board median wall was 95.1 minutes, and the last push to merge median was 477.4 minutes with a p90 of 1023.9 minutes. The runner queue wait p90 was 36.9 minutes over 2845 jobs. The first failing job counts, largest first, were 5 for integration / velocity, 5 for test-unit (3/4), 4 for test-unit (2/4), 3 for test-browser, 2 for integration / auto-exposure, 2 for integration / auto-exposure cold boot, 2 for test-unit (1/4), and 1 for each of golden-path-template (platformer), integration / capture, native-platforms / iOS simulator runtime and no-Xcode consumer handoff, native-platforms / macOS desktop core, template-nonvisual (platformer), and test-playtest. Develop moved from b12b257f1 to 6b18d913e, and its CI run on that head was still queued when this row was written. None of these PRs carries the phase 1 change on PR 476, which is still open, so this window cannot judge that change, and the decision is inconclusive.		inconclusive2026-10-09477.41023.940%95.136.9
#172026-10-090a4ce848chourly refresh
Merged into develop since 2026-10-09 are 5 PRs, 473, 471, 470, 468 and 458. The merge-group run count is 42 and the run success percent is 18.4 with the first-attempt merge percent at 40. The board median wall is 95.1 minutes, and the last push to merge median is 477.4 minutes and p90 is 1023.9 minutes. The queue wait p90 is 40.4 minutes over 2873 jobs. The first failing job counts are 6 for integration / velocity, 5 for test-unit (2/4), 5 for test-unit (3/4), 3 for test-browser, 2 for integration / auto-exposure, 2 for integration / auto-exposure cold boot, 2 for test-unit (1/4), and 1 each for golden-path-template (platformer), integration / capture, native-platforms / iOS simulator runtime and no-Xcode consumer handoff, native-platforms / macOS desktop core, template-nonvisual (platformer) and test-playtest. Develop head is 6b18d913e and its CI run has finished with success. None of these PRs carry the phase 1 change on PR 476, which is still open, so this window cannot judge that change, and local commit 0a4ce848c adds a test only diagnostic and does not change any merge-group result, giving an inconclusive decision.		inconclusive2026-10-09477.41023.940%95.141.5
#182026-10-101652670eePR476 unmerged baseline refresh
PR476 remains unmerged. This is a fresh observation of develop, not a measured effect of PR476. Candidate CI run 38025989864 failed sailing diagnostics; ci-required correctly rejected it.		provisional2026-10-09744.51130.133.3%107.942.3
#192026-10-1077bb0ee57PR476 candidate 77bb in CI — unmerged baseline
Fresh develop baseline while PR476 remains unmerged. CI run38033595736 targets candidate77bb; verification is pending. This audit does not measure PR476 effect.		provisional2026-10-09475.91090.925%107.942
#202026-10-1077bb0ee57Current queue comparison — PR476 still unmerged
Refreshed after user requested current queue comparison and continued speed work. PR476 remains unmerged; no measured impact attribution.		provisional2026-10-09474.31071.322.2%95.139.6
+

Merged PRs, iteration #20 window

+
PRTitlePush to mergeEnqueue to mergeAttemptsFirst attemptFirst failing job
#479PRD-567: terrain layers blend by their own height, and the game owns the curve66.4 min64.9 min1cancelled
#477feat(core): shadow casters sort themselves into static and moving (PRD-572)91.8 min56.5 min1cancelled
#473fix(world): PRD-494 — bundle-safety walk stops reading pixels; main pass draw budget measured317.9 min454.8 min5failurenative-platforms / macOS desktop core
#472PRD-297: capability recall is a gated number, PRD done1228.1 min1160.9 min12failureintegration / velocity
#471fix(capabilities): every manifest import resolves in every scaffolded template (PRD-354)1011.6 min1015.5 min8successtest-playtest
#470PRD-461: view distance basics — terrain radius, near-only colliders, recipe fog477.4 min363.9 min4successintegration / velocity
#469fix(shooter): clear ADS sight window, remove GTAO square behind smoke125.9 min1091.6 min7failuretest-unit (2/4)
#468feat(core): ship threenative-sculpt-mcp 0.2.11032.1 min631.5 min7failuretest-browser
#458test(PRD-536): tests close the fail-open and regression holes474.3 min143.1 min2failuretest-unit (2/4)
+
diff --git a/docs/ci-speed/ledger.json b/docs/ci-speed/ledger.json new file mode 100644 index 0000000000..487dcd27b4 --- /dev/null +++ b/docs/ci-speed/ledger.json @@ -0,0 +1,2050 @@ +{ + "schemaVersion": 1, + "goals": [ + { + "key": "lastPushToMergeMedianMin", + "short": "Push→merge p50", + "label": "Last push to merge, median", + "unit": "min", + "goal": 30, + "better": "lower" + }, + { + "key": "lastPushToMergeP90Min", + "short": "Push→merge p90", + "label": "Last push to merge, p90", + "unit": "min", + "goal": 45, + "better": "lower" + }, + { + "key": "firstAttemptSuccessPct", + "short": "First attempt", + "label": "Merge queue passes on the first attempt", + "unit": "%", + "goal": 90, + "better": "higher" + }, + { + "key": "boardWallMedianMin", + "short": "Board wall", + "label": "Merge-group board wall time, median", + "unit": "min", + "goal": 15, + "better": "lower" + }, + { + "key": "queueWaitP90Min", + "short": "Queue wait p90", + "label": "Runner queue wait, p90", + "unit": "min", + "goal": 3, + "better": "lower" + } + ], + "iterations": [ + { + "id": 1, + "at": "2026-10-09T05:09:15.030Z", + "commit": "11cafc003", + "label": "baseline: exhaustive board on every merge group", + "notes": "Before PRD-550. 2026-10-08 to 2026-10-09 window, same period as the PRD Context.", + "decision": "baseline", + "source": "measured", + "window": { + "since": "2026-10-08", + "base": "develop" + }, + "metrics": { + "lastPushToMergeMedianMin": 280.4, + "lastPushToMergeP90Min": 838.3, + "firstAttemptSuccessPct": 33.3, + "boardWallMedianMin": 77.8, + "queueWaitP90Min": 41.4 + }, + "extra": { + "prCount": 12, + "mergeGroupRuns": 45, + "mergeGroupRunSuccessPct": 24.2, + "enqueueToMergeMedianMin": 432.7, + "queueWaitJobs": 2759 + }, + "prs": [ + { + "number": 467, + "title": "docs(prds): renumber the low-resolution temporal PRD to 539", + "lastPushToMergeMin": 58.7, + "enqueueToMergeMin": null, + "attempts": 0, + "firstAttempt": null, + "firstFailingJob": null + }, + { + "number": 466, + "title": "docs(prd): point asset-mcp and sculpt-mcp links at ThreeNativeHQ", + "lastPushToMergeMin": 103.9, + "enqueueToMergeMin": null, + "attempts": 0, + "firstAttempt": null, + "firstFailingJob": null + }, + { + "number": 465, + "title": "docs(guides): list native-css-support.md in the guides index", + "lastPushToMergeMin": 134.5, + "enqueueToMergeMin": 131.9, + "attempts": 3, + "firstAttempt": "failure", + "firstFailingJob": "test" + }, + { + "number": 464, + "title": "docs(prds): file PRD-537 and PRD-538", + "lastPushToMergeMin": 7.5, + "enqueueToMergeMin": null, + "attempts": 0, + "firstAttempt": null, + "firstFailingJob": null + }, + { + "number": 462, + "title": "PRD-210: Android continue stays desktop-only, PRD done", + "lastPushToMergeMin": 307.8, + "enqueueToMergeMin": 242.9, + "attempts": 5, + "firstAttempt": "failure", + "firstFailingJob": "test-unit (1/4)" + }, + { + "number": 461, + "title": "PRD-210: Pixel 8 audio-suspend rung and relaunch cycles pass", + "lastPushToMergeMin": 111.4, + "enqueueToMergeMin": 104.2, + "attempts": 2, + "firstAttempt": "failure", + "firstFailingJob": "build-artifacts" + }, + { + "number": 457, + "title": "PRD-210: native host survives backgrounding and crashes — device proofs", + "lastPushToMergeMin": 436, + "enqueueToMergeMin": 432.7, + "attempts": 4, + "firstAttempt": "success", + "firstFailingJob": "integration / auto-exposure" + }, + { + "number": 456, + "title": "docs(prd-211): Android APK boots from real Ogg assets — close PRD-211", + "lastPushToMergeMin": 764.7, + "enqueueToMergeMin": 892.1, + "attempts": 9, + "firstAttempt": "cancelled", + "firstFailingJob": "test-native" + }, + { + "number": 455, + "title": "fix(PRD-196): a clean machine builds Android from npm; hosted clean room proves it", + "lastPushToMergeMin": 846.5, + "enqueueToMergeMin": 626.9, + "attempts": 3, + "firstAttempt": "success", + "firstFailingJob": "integration / auto-exposure" + }, + { + "number": 449, + "title": "Bake procedural wolves and follow accepted physics", + "lastPushToMergeMin": 870.6, + "enqueueToMergeMin": 593.9, + "attempts": 7, + "firstAttempt": "success", + "firstFailingJob": "integration / Linux native locomotion correctness" + }, + { + "number": 448, + "title": "feat(avbd): prototype bounded secondary ropes and sails", + "lastPushToMergeMin": 349.2, + "enqueueToMergeMin": 116.8, + "attempts": 2, + "firstAttempt": "failure", + "firstFailingJob": "integration / auto-exposure" + }, + { + "number": 398, + "title": "feat(render): consolidate motion history and temporal reconstruction (PRDs 269, 455)", + "lastPushToMergeMin": 252.9, + "enqueueToMergeMin": 545.7, + "attempts": 4, + "firstAttempt": "failure", + "firstFailingJob": "native-platforms / macOS desktop core" + } + ], + "firstFailingJobs": [ + { + "job": "integration / auto-exposure", + "count": 8 + }, + { + "job": "test", + "count": 3 + }, + { + "job": "native-platforms / macOS desktop core", + "count": 2 + }, + { + "job": "test-unit (1/4)", + "count": 2 + }, + { + "job": "test-unit (3/4)", + "count": 2 + }, + { + "job": "budgets", + "count": 1 + }, + { + "job": "build-artifacts", + "count": 1 + }, + { + "job": "integration / Linux native fluid correctness", + "count": 1 + }, + { + "job": "integration / Linux native locomotion correctness", + "count": 1 + }, + { + "job": "integration / velocity", + "count": 1 + }, + { + "job": "native-platforms / iOS simulator runtime and no-Xcode consumer handoff", + "count": 1 + }, + { + "job": "test-native", + "count": 1 + }, + { + "job": "test-unit (4/4)", + "count": 1 + } + ] + }, + { + "id": 2, + "at": "2026-10-09T05:47:36.259Z", + "commit": "90dee0c83", + "label": "no change landed: the queue still runs the exhaustive board", + "notes": "Window since 2026-10-09. 0 PRs merged, so push->merge, p90 and first-attempt are n/a; board wall and queue wait measure the exhaustive merge-group board this change removes. Iteration #1 stays the baseline. Re-measure once 5 or more PRs merged.", + "decision": "provisional", + "source": "measured", + "window": { + "since": "2026-10-09", + "base": "develop" + }, + "metrics": { + "lastPushToMergeMedianMin": null, + "lastPushToMergeP90Min": null, + "firstAttemptSuccessPct": null, + "boardWallMedianMin": 119.1, + "queueWaitP90Min": 50.9 + }, + "extra": { + "prCount": 0, + "mergeGroupRuns": 7, + "mergeGroupRunSuccessPct": 50, + "enqueueToMergeMedianMin": null, + "queueWaitJobs": 462 + }, + "prs": [], + "firstFailingJobs": [ + { + "job": "integration / velocity", + "count": 2 + } + ] + }, + { + "id": 3, + "at": "2026-10-09T07:02:02.167Z", + "commit": "a47851026", + "label": "phase 1 in review", + "notes": "Same window as iteration 2: 0 PRs merged, 9 merge-group runs, board median 113.3 min, queue wait p90 49.9 min. Named cost: integration / velocity was the first failing job in 2 of 4 red merge groups, because its cost arm priced shared-runner load as render cost. The change is PRD-550 phase 1 (481824f3c, a47851026) on PR 476. It moves that arm to a scheduled lane and widens two compile budgets. The compile specs pass locally, 40 of 40 in 220 s. Re-measure after it merges. The PR head also shows an intermittent iOS simulator red, seen in 2 of 10 recent runs with different errors, on code this branch does not change.", + "decision": "provisional", + "source": "measured", + "window": { + "since": "2026-10-09", + "base": "develop" + }, + "metrics": { + "lastPushToMergeMedianMin": null, + "lastPushToMergeP90Min": null, + "firstAttemptSuccessPct": null, + "boardWallMedianMin": 113.3, + "queueWaitP90Min": 49.9 + }, + "extra": { + "prCount": 0, + "mergeGroupRuns": 9, + "mergeGroupRunSuccessPct": 42.9, + "enqueueToMergeMedianMin": null, + "queueWaitJobs": 602 + }, + "prs": [], + "firstFailingJobs": [ + { + "job": "integration / velocity", + "count": 2 + }, + { + "job": "test-playtest", + "count": 1 + }, + { + "job": "test-unit (3/4)", + "count": 1 + } + ] + }, + { + "id": 4, + "at": "2026-10-09T08:09:52.392Z", + "commit": "6ec34f52c", + "label": "PR 476 green on phase 1 head", + "notes": "Same window: 0 PRs merged, 12 merge-group runs. PR 476 head 6ec34f52c is green: 69 checks pass and 12 skip, and merge state is CLEAN. integration / velocity passes on that head, and the iOS simulator job passes there too, so the iOS red from iteration 3 did not recur. Merge-group metrics do not move until phase 1 reaches develop. Named costs still on develop: integration / velocity and test-unit (3/4), each first in 2 red merge groups. The compile-spec budget change (a47851026) targets the test-unit timeouts. Re-measure after the merge.", + "decision": "provisional", + "source": "measured", + "window": { + "since": "2026-10-09", + "base": "develop" + }, + "metrics": { + "lastPushToMergeMedianMin": null, + "lastPushToMergeP90Min": null, + "firstAttemptSuccessPct": null, + "boardWallMedianMin": 113.3, + "queueWaitP90Min": 46.9 + }, + "extra": { + "prCount": 0, + "mergeGroupRuns": 12, + "mergeGroupRunSuccessPct": 33.3, + "enqueueToMergeMedianMin": null, + "queueWaitJobs": 774 + }, + "prs": [], + "firstFailingJobs": [ + { + "job": "integration / velocity", + "count": 2 + }, + { + "job": "test-unit (3/4)", + "count": 2 + }, + { + "job": "test-playtest", + "count": 1 + }, + { + "job": "test-unit (2/4)", + "count": 1 + } + ] + }, + { + "id": 5, + "at": "2026-10-09T09:15:07.994Z", + "commit": "3ca761875", + "label": "hourly refresh", + "notes": "One PR merged since the last tick: 458, last push to merge 474.3 min, first attempt failed at test-unit (2/4). The window now has 13 merge-group runs, board median 111.6 min, and queue wait p90 47.2 min over 885 jobs. First failing job in the red groups: integration / velocity 2, test-unit (3/4) 2, test-unit (2/4) 1, integration / capture 1, test-playtest 1. PR 476 is still open, so phase 1 is not on develop yet. Provisional: fewer than 5 PRs merged.", + "decision": "provisional", + "source": "measured", + "window": { + "since": "2026-10-09", + "base": "develop" + }, + "metrics": { + "lastPushToMergeMedianMin": 474.3, + "lastPushToMergeP90Min": 474.3, + "firstAttemptSuccessPct": 0, + "boardWallMedianMin": 111.6, + "queueWaitP90Min": 47.2 + }, + "extra": { + "prCount": 1, + "mergeGroupRuns": 13, + "mergeGroupRunSuccessPct": 36.4, + "enqueueToMergeMedianMin": 143.1, + "queueWaitJobs": 886 + }, + "prs": [ + { + "number": 458, + "title": "test(PRD-536): tests close the fail-open and regression holes", + "lastPushToMergeMin": 474.3, + "enqueueToMergeMin": 143.1, + "attempts": 2, + "firstAttempt": "failure", + "firstFailingJob": "test-unit (2/4)" + } + ], + "firstFailingJobs": [ + { + "job": "integration / velocity", + "count": 2 + }, + { + "job": "test-unit (3/4)", + "count": 2 + }, + { + "job": "integration / capture", + "count": 1 + }, + { + "job": "test-playtest", + "count": 1 + }, + { + "job": "test-unit (2/4)", + "count": 1 + } + ] + }, + { + "id": 6, + "at": "2026-10-09T10:16:15.565Z", + "commit": "7789bf328", + "label": "hourly refresh", + "notes": "One new PR merged since the last tick, 470, after 4 merge-group attempts. Its first failing job was integration / velocity. The window now has 2 merged PRs, 470 and 458. Last push to merge median 475.9 min, p90 477.1 min. First attempt merge 50 percent. 16 merge-group runs, board median 109.9 min, queue wait p90 45 min over 1086 jobs. First failing job in the red groups: integration / velocity 2, test-unit (3/4) 2, integration / capture 1, template-nonvisual platformer 1. PR 476 is still open, so phase 1 is not on develop yet. Provisional: fewer than 5 PRs merged.", + "decision": "provisional", + "source": "measured", + "window": { + "since": "2026-10-09", + "base": "develop" + }, + "metrics": { + "lastPushToMergeMedianMin": 475.9, + "lastPushToMergeP90Min": 477.1, + "firstAttemptSuccessPct": 50, + "boardWallMedianMin": 109.9, + "queueWaitP90Min": 45 + }, + "extra": { + "prCount": 2, + "mergeGroupRuns": 16, + "mergeGroupRunSuccessPct": 35.7, + "enqueueToMergeMedianMin": 253.5, + "queueWaitJobs": 1086 + }, + "prs": [ + { + "number": 470, + "title": "PRD-461: view distance basics — terrain radius, near-only colliders, recipe fog", + "lastPushToMergeMin": 477.4, + "enqueueToMergeMin": 363.9, + "attempts": 4, + "firstAttempt": "success", + "firstFailingJob": "integration / velocity" + }, + { + "number": 458, + "title": "test(PRD-536): tests close the fail-open and regression holes", + "lastPushToMergeMin": 474.3, + "enqueueToMergeMin": 143.1, + "attempts": 2, + "firstAttempt": "failure", + "firstFailingJob": "test-unit (2/4)" + } + ], + "firstFailingJobs": [ + { + "job": "integration / velocity", + "count": 2 + }, + { + "job": "test-unit (3/4)", + "count": 2 + }, + { + "job": "integration / capture", + "count": 1 + }, + { + "job": "template-nonvisual (platformer)", + "count": 1 + }, + { + "job": "test-browser", + "count": 1 + }, + { + "job": "test-playtest", + "count": 1 + }, + { + "job": "test-unit (2/4)", + "count": 1 + } + ] + }, + { + "id": 7, + "at": "2026-10-09T11:15:40.652Z", + "commit": "aa13e02c1", + "label": "hourly refresh", + "notes": "No new PR merged since the last tick. The window still has 2 merged PRs, 470 and 458. It now has 18 merge-group runs, up from 16. Board median 109.9 min. Queue wait p90 42.9 min over 1227 jobs. Run success fell to 31.3 percent. First failing job in the 8 red groups: integration / velocity 4, test-unit (3/4) 2, integration / capture 1, template-nonvisual platformer 1. integration / velocity is the largest named cost, and phase 1 on PR 476 targets it. PR 476 is still open, so phase 1 is not on develop yet. Provisional: fewer than 5 PRs merged.", + "decision": "provisional", + "source": "measured", + "window": { + "since": "2026-10-09", + "base": "develop" + }, + "metrics": { + "lastPushToMergeMedianMin": 475.9, + "lastPushToMergeP90Min": 477.1, + "firstAttemptSuccessPct": 50, + "boardWallMedianMin": 109.9, + "queueWaitP90Min": 42.9 + }, + "extra": { + "prCount": 2, + "mergeGroupRuns": 18, + "mergeGroupRunSuccessPct": 31.3, + "enqueueToMergeMedianMin": 253.5, + "queueWaitJobs": 1228 + }, + "prs": [ + { + "number": 470, + "title": "PRD-461: view distance basics — terrain radius, near-only colliders, recipe fog", + "lastPushToMergeMin": 477.4, + "enqueueToMergeMin": 363.9, + "attempts": 4, + "firstAttempt": "success", + "firstFailingJob": "integration / velocity" + }, + { + "number": 458, + "title": "test(PRD-536): tests close the fail-open and regression holes", + "lastPushToMergeMin": 474.3, + "enqueueToMergeMin": 143.1, + "attempts": 2, + "firstAttempt": "failure", + "firstFailingJob": "test-unit (2/4)" + } + ], + "firstFailingJobs": [ + { + "job": "integration / velocity", + "count": 4 + }, + { + "job": "test-unit (3/4)", + "count": 2 + }, + { + "job": "integration / capture", + "count": 1 + }, + { + "job": "template-nonvisual (platformer)", + "count": 1 + }, + { + "job": "test-browser", + "count": 1 + }, + { + "job": "test-playtest", + "count": 1 + }, + { + "job": "test-unit (2/4)", + "count": 1 + } + ] + }, + { + "id": 8, + "at": "2026-10-09T12:15:46.448Z", + "commit": "9f36ac3c8", + "label": "hourly refresh", + "notes": "No new PR merged since the last tick. The window still has 2 merged PRs, 470 and 458. It now has 20 merge-group runs, up from 18. Run success fell to 27.8 percent. Board median 109.9 min. Queue wait p90 41.6 min over 1376 jobs. First failing job in the red groups: integration / velocity 4, test-unit (3/4) 3, test-browser 2, integration / capture 1. test-browser is new in this list. integration / velocity is still the largest named cost, and phase 1 on PR 476 targets it. PR 476 is still open. Provisional: fewer than 5 PRs merged.", + "decision": "provisional", + "source": "measured", + "window": { + "since": "2026-10-09", + "base": "develop" + }, + "metrics": { + "lastPushToMergeMedianMin": 475.9, + "lastPushToMergeP90Min": 477.1, + "firstAttemptSuccessPct": 50, + "boardWallMedianMin": 109.9, + "queueWaitP90Min": 41.5 + }, + "extra": { + "prCount": 2, + "mergeGroupRuns": 21, + "mergeGroupRunSuccessPct": 26.3, + "enqueueToMergeMedianMin": 253.5, + "queueWaitJobs": 1379 + }, + "prs": [ + { + "number": 470, + "title": "PRD-461: view distance basics — terrain radius, near-only colliders, recipe fog", + "lastPushToMergeMin": 477.4, + "enqueueToMergeMin": 363.9, + "attempts": 4, + "firstAttempt": "success", + "firstFailingJob": "integration / velocity" + }, + { + "number": 458, + "title": "test(PRD-536): tests close the fail-open and regression holes", + "lastPushToMergeMin": 474.3, + "enqueueToMergeMin": 143.1, + "attempts": 2, + "firstAttempt": "failure", + "firstFailingJob": "test-unit (2/4)" + } + ], + "firstFailingJobs": [ + { + "job": "integration / velocity", + "count": 4 + }, + { + "job": "test-unit (3/4)", + "count": 3 + }, + { + "job": "test-browser", + "count": 2 + }, + { + "job": "integration / auto-exposure cold boot", + "count": 1 + }, + { + "job": "integration / capture", + "count": 1 + }, + { + "job": "template-nonvisual (platformer)", + "count": 1 + }, + { + "job": "test-playtest", + "count": 1 + }, + { + "job": "test-unit (2/4)", + "count": 1 + } + ] + }, + { + "id": 9, + "at": "2026-10-09T13:16:32.659Z", + "commit": "cbd2e662e", + "label": "hourly refresh", + "notes": "No new PR merged since the last tick. The window still has 2 merged PRs, 470 and 458. It now has 24 merge-group runs, up from 20. Run success fell to 23.8 percent. Board median 109.9 min. Queue wait p90 41.2 min over 1579 jobs. First failing job in the red groups: integration / velocity 4, test-unit (3/4) 4, test-browser 2, integration / auto-exposure cold boot 1. integration / velocity and test-unit (3/4) are tied as the largest named costs. Phase 1 on PR 476 targets integration / velocity, and its compile-spec budget change targets test-unit timeouts. PR 476 is still open. Provisional: fewer than 5 PRs merged.", + "decision": "provisional", + "source": "measured", + "window": { + "since": "2026-10-09", + "base": "develop" + }, + "metrics": { + "lastPushToMergeMedianMin": 475.9, + "lastPushToMergeP90Min": 477.1, + "firstAttemptSuccessPct": 50, + "boardWallMedianMin": 109.9, + "queueWaitP90Min": 40.7 + }, + "extra": { + "prCount": 2, + "mergeGroupRuns": 24, + "mergeGroupRunSuccessPct": 23.8, + "enqueueToMergeMedianMin": 253.5, + "queueWaitJobs": 1610 + }, + "prs": [ + { + "number": 470, + "title": "PRD-461: view distance basics — terrain radius, near-only colliders, recipe fog", + "lastPushToMergeMin": 477.4, + "enqueueToMergeMin": 363.9, + "attempts": 4, + "firstAttempt": "success", + "firstFailingJob": "integration / velocity" + }, + { + "number": 458, + "title": "test(PRD-536): tests close the fail-open and regression holes", + "lastPushToMergeMin": 474.3, + "enqueueToMergeMin": 143.1, + "attempts": 2, + "firstAttempt": "failure", + "firstFailingJob": "test-unit (2/4)" + } + ], + "firstFailingJobs": [ + { + "job": "integration / velocity", + "count": 4 + }, + { + "job": "test-unit (3/4)", + "count": 4 + }, + { + "job": "test-browser", + "count": 2 + }, + { + "job": "integration / auto-exposure cold boot", + "count": 1 + }, + { + "job": "integration / capture", + "count": 1 + }, + { + "job": "native-platforms / iOS simulator runtime and no-Xcode consumer handoff", + "count": 1 + }, + { + "job": "template-nonvisual (platformer)", + "count": 1 + }, + { + "job": "test-playtest", + "count": 1 + }, + { + "job": "test-unit (2/4)", + "count": 1 + } + ] + }, + { + "id": 10, + "at": "2026-10-09T14:16:12.255Z", + "commit": "2e100adc4", + "label": "hourly refresh", + "notes": "No new PR merged since the last tick. The window still has 2 merged PRs, 470 and 458. It now has 26 merge-group runs, up from 24. Run success fell to 20.8 percent, the fourth straight drop. Board median 109.9 min. Queue wait p90 39 min over 1785 jobs, down from 41.2. First failing job in the red groups: test-unit (3/4) 5, integration / velocity 4, test-browser 2, integration / auto-exposure cold boot 1. test-unit (3/4) is now the largest named cost. Phase 1 on PR 476 includes a compile-spec budget change for test-unit timeouts, and it is not on develop yet. PR 476 is still open. Provisional: fewer than 5 PRs merged.", + "decision": "provisional", + "source": "measured", + "window": { + "since": "2026-10-09", + "base": "develop" + }, + "metrics": { + "lastPushToMergeMedianMin": 475.9, + "lastPushToMergeP90Min": 477.1, + "firstAttemptSuccessPct": 50, + "boardWallMedianMin": 109.9, + "queueWaitP90Min": 39 + }, + "extra": { + "prCount": 2, + "mergeGroupRuns": 26, + "mergeGroupRunSuccessPct": 20.8, + "enqueueToMergeMedianMin": 253.5, + "queueWaitJobs": 1787 + }, + "prs": [ + { + "number": 470, + "title": "PRD-461: view distance basics — terrain radius, near-only colliders, recipe fog", + "lastPushToMergeMin": 477.4, + "enqueueToMergeMin": 363.9, + "attempts": 4, + "firstAttempt": "success", + "firstFailingJob": "integration / velocity" + }, + { + "number": 458, + "title": "test(PRD-536): tests close the fail-open and regression holes", + "lastPushToMergeMin": 474.3, + "enqueueToMergeMin": 143.1, + "attempts": 2, + "firstAttempt": "failure", + "firstFailingJob": "test-unit (2/4)" + } + ], + "firstFailingJobs": [ + { + "job": "test-unit (3/4)", + "count": 5 + }, + { + "job": "integration / velocity", + "count": 4 + }, + { + "job": "test-browser", + "count": 2 + }, + { + "job": "integration / auto-exposure cold boot", + "count": 1 + }, + { + "job": "integration / capture", + "count": 1 + }, + { + "job": "native-platforms / iOS simulator runtime and no-Xcode consumer handoff", + "count": 1 + }, + { + "job": "native-platforms / macOS desktop core", + "count": 1 + }, + { + "job": "template-nonvisual (platformer)", + "count": 1 + }, + { + "job": "test-playtest", + "count": 1 + }, + { + "job": "test-unit (1/4)", + "count": 1 + }, + { + "job": "test-unit (2/4)", + "count": 1 + } + ] + }, + { + "id": 11, + "at": "2026-10-09T15:17:36.022Z", + "commit": "f9a01bbe4", + "label": "hourly refresh", + "notes": "No new PR merged since the last tick. The window still has 2 merged PRs, 470 and 458. It now has 29 merge-group runs, up from 26. Run success is 22.2 percent, up from 20.8. Board median is 102.5 min, down from 109.9. Queue wait p90 is 38.4 min over 1993 jobs. First failing job in the 12 red groups: test-unit (3/4) 5, integration / velocity 4, test-browser 2, golden-path template platformer 1. The named costs are unchanged. A separate develop push run at b12b257f1 failed in test-native, and it is not counted in these groups. PR 476 is still open. Provisional: fewer than 5 PRs merged.", + "decision": "provisional", + "source": "measured", + "window": { + "since": "2026-10-09", + "base": "develop" + }, + "metrics": { + "lastPushToMergeMedianMin": 475.9, + "lastPushToMergeP90Min": 477.1, + "firstAttemptSuccessPct": 50, + "boardWallMedianMin": 102.5, + "queueWaitP90Min": 38.9 + }, + "extra": { + "prCount": 2, + "mergeGroupRuns": 29, + "mergeGroupRunSuccessPct": 22.2, + "enqueueToMergeMedianMin": 253.5, + "queueWaitJobs": 1994 + }, + "prs": [ + { + "number": 470, + "title": "PRD-461: view distance basics — terrain radius, near-only colliders, recipe fog", + "lastPushToMergeMin": 477.4, + "enqueueToMergeMin": 363.9, + "attempts": 4, + "firstAttempt": "success", + "firstFailingJob": "integration / velocity" + }, + { + "number": 458, + "title": "test(PRD-536): tests close the fail-open and regression holes", + "lastPushToMergeMin": 474.3, + "enqueueToMergeMin": 143.1, + "attempts": 2, + "firstAttempt": "failure", + "firstFailingJob": "test-unit (2/4)" + } + ], + "firstFailingJobs": [ + { + "job": "test-unit (3/4)", + "count": 5 + }, + { + "job": "integration / velocity", + "count": 4 + }, + { + "job": "test-browser", + "count": 2 + }, + { + "job": "golden-path-template (platformer)", + "count": 1 + }, + { + "job": "integration / auto-exposure", + "count": 1 + }, + { + "job": "integration / auto-exposure cold boot", + "count": 1 + }, + { + "job": "integration / capture", + "count": 1 + }, + { + "job": "native-platforms / iOS simulator runtime and no-Xcode consumer handoff", + "count": 1 + }, + { + "job": "native-platforms / macOS desktop core", + "count": 1 + }, + { + "job": "template-nonvisual (platformer)", + "count": 1 + }, + { + "job": "test-playtest", + "count": 1 + }, + { + "job": "test-unit (1/4)", + "count": 1 + }, + { + "job": "test-unit (2/4)", + "count": 1 + } + ] + }, + { + "id": 12, + "at": "2026-10-09T16:04:17.913Z", + "commit": "f1e868d47", + "label": "hourly refresh", + "notes": "Catch-up tick after the 09:00 stop. No new PR merged. The window still has 2 merged PRs, 470 and 458. It now has 31 merge-group runs, up from 29. Run success is 24.1 percent, up from 22.2. Board median is 95.1 min, down from 102.5. Queue wait p90 is 38.7 min over 2118 jobs. First failing job in the 13 red groups: test-unit (3/4) 5, integration / velocity 4, test-browser 2, test-unit (1/4) 2. test-unit (1/4) is new in this list. The named costs are unchanged. PR 476 is still open. Provisional: fewer than 5 PRs merged.", + "decision": "provisional", + "source": "measured", + "window": { + "since": "2026-10-09", + "base": "develop" + }, + "metrics": { + "lastPushToMergeMedianMin": 475.9, + "lastPushToMergeP90Min": 477.1, + "firstAttemptSuccessPct": 50, + "boardWallMedianMin": 95.1, + "queueWaitP90Min": 38.7 + }, + "extra": { + "prCount": 2, + "mergeGroupRuns": 31, + "mergeGroupRunSuccessPct": 24.1, + "enqueueToMergeMedianMin": 253.5, + "queueWaitJobs": 2118 + }, + "prs": [ + { + "number": 470, + "title": "PRD-461: view distance basics — terrain radius, near-only colliders, recipe fog", + "lastPushToMergeMin": 477.4, + "enqueueToMergeMin": 363.9, + "attempts": 4, + "firstAttempt": "success", + "firstFailingJob": "integration / velocity" + }, + { + "number": 458, + "title": "test(PRD-536): tests close the fail-open and regression holes", + "lastPushToMergeMin": 474.3, + "enqueueToMergeMin": 143.1, + "attempts": 2, + "firstAttempt": "failure", + "firstFailingJob": "test-unit (2/4)" + } + ], + "firstFailingJobs": [ + { + "job": "test-unit (3/4)", + "count": 5 + }, + { + "job": "integration / velocity", + "count": 4 + }, + { + "job": "test-browser", + "count": 2 + }, + { + "job": "test-unit (1/4)", + "count": 2 + }, + { + "job": "golden-path-template (platformer)", + "count": 1 + }, + { + "job": "integration / auto-exposure", + "count": 1 + }, + { + "job": "integration / auto-exposure cold boot", + "count": 1 + }, + { + "job": "integration / capture", + "count": 1 + }, + { + "job": "native-platforms / iOS simulator runtime and no-Xcode consumer handoff", + "count": 1 + }, + { + "job": "native-platforms / macOS desktop core", + "count": 1 + }, + { + "job": "template-nonvisual (platformer)", + "count": 1 + }, + { + "job": "test-playtest", + "count": 1 + }, + { + "job": "test-unit (2/4)", + "count": 1 + } + ] + }, + { + "id": 13, + "at": "2026-10-09T17:18:18.223Z", + "commit": "1310367b6", + "label": "hourly refresh", + "notes": "No new PR merged since the last tick. The window still has 2 merged PRs, 470 and 458. It now has 33 merge-group runs, up from 31. Run success is 22.6 percent, down from 24.1. Board median is 95.1 min, unchanged. Queue wait p90 is 38.5 min over 2258 jobs. First failing job in the 14 red groups: integration / velocity 5, test-unit (3/4) 5, test-browser 2, test-unit (1/4) 2. integration / velocity rose by one. PR 476 is still open. Provisional: fewer than 5 PRs merged.", + "decision": "provisional", + "source": "measured", + "window": { + "since": "2026-10-09", + "base": "develop" + }, + "metrics": { + "lastPushToMergeMedianMin": 475.9, + "lastPushToMergeP90Min": 477.1, + "firstAttemptSuccessPct": 50, + "boardWallMedianMin": 95.1, + "queueWaitP90Min": 38.5 + }, + "extra": { + "prCount": 2, + "mergeGroupRuns": 33, + "mergeGroupRunSuccessPct": 22.6, + "enqueueToMergeMedianMin": 253.5, + "queueWaitJobs": 2259 + }, + "prs": [ + { + "number": 470, + "title": "PRD-461: view distance basics — terrain radius, near-only colliders, recipe fog", + "lastPushToMergeMin": 477.4, + "enqueueToMergeMin": 363.9, + "attempts": 4, + "firstAttempt": "success", + "firstFailingJob": "integration / velocity" + }, + { + "number": 458, + "title": "test(PRD-536): tests close the fail-open and regression holes", + "lastPushToMergeMin": 474.3, + "enqueueToMergeMin": 143.1, + "attempts": 2, + "firstAttempt": "failure", + "firstFailingJob": "test-unit (2/4)" + } + ], + "firstFailingJobs": [ + { + "job": "integration / velocity", + "count": 5 + }, + { + "job": "test-unit (3/4)", + "count": 5 + }, + { + "job": "test-browser", + "count": 2 + }, + { + "job": "test-unit (1/4)", + "count": 2 + }, + { + "job": "test-unit (2/4)", + "count": 2 + }, + { + "job": "golden-path-template (platformer)", + "count": 1 + }, + { + "job": "integration / auto-exposure", + "count": 1 + }, + { + "job": "integration / auto-exposure cold boot", + "count": 1 + }, + { + "job": "integration / capture", + "count": 1 + }, + { + "job": "native-platforms / iOS simulator runtime and no-Xcode consumer handoff", + "count": 1 + }, + { + "job": "native-platforms / macOS desktop core", + "count": 1 + }, + { + "job": "template-nonvisual (platformer)", + "count": 1 + }, + { + "job": "test-playtest", + "count": 1 + } + ] + }, + { + "id": 14, + "at": "2026-10-09T18:19:50.825Z", + "commit": "6639a5baf", + "label": "hourly refresh", + "notes": "No new PR merged since the last tick. The window still has 2 merged PRs, 470 and 458. It now has 35 merge-group runs, up from 33. Run success is 21.2 percent, down from 22.6. Board median is 95.1 min, unchanged. Queue wait p90 is 38.7 min over 2396 jobs. First failing job in the 15 red groups: integration / velocity 5, test-unit (3/4) 5, test-unit (2/4) 3, integration / auto-exposure cold boot 2. The list changed: test-unit (2/4) and auto-exposure cold boot appear, and test-browser and test-unit (1/4) drop out. The first attempt hit a transient GitHub 502, and the rerun succeeded. Develop still sits at b12b257f1, so the queue has no new merges. PR 476 is still open. Provisional: fewer than 5 PRs merged.", + "decision": "provisional", + "source": "measured", + "window": { + "since": "2026-10-09", + "base": "develop" + }, + "metrics": { + "lastPushToMergeMedianMin": 475.9, + "lastPushToMergeP90Min": 477.1, + "firstAttemptSuccessPct": 50, + "boardWallMedianMin": 95.1, + "queueWaitP90Min": 38.7 + }, + "extra": { + "prCount": 2, + "mergeGroupRuns": 35, + "mergeGroupRunSuccessPct": 21.2, + "enqueueToMergeMedianMin": 253.5, + "queueWaitJobs": 2396 + }, + "prs": [ + { + "number": 470, + "title": "PRD-461: view distance basics — terrain radius, near-only colliders, recipe fog", + "lastPushToMergeMin": 477.4, + "enqueueToMergeMin": 363.9, + "attempts": 4, + "firstAttempt": "success", + "firstFailingJob": "integration / velocity" + }, + { + "number": 458, + "title": "test(PRD-536): tests close the fail-open and regression holes", + "lastPushToMergeMin": 474.3, + "enqueueToMergeMin": 143.1, + "attempts": 2, + "firstAttempt": "failure", + "firstFailingJob": "test-unit (2/4)" + } + ], + "firstFailingJobs": [ + { + "job": "integration / velocity", + "count": 5 + }, + { + "job": "test-unit (3/4)", + "count": 5 + }, + { + "job": "test-unit (2/4)", + "count": 3 + }, + { + "job": "integration / auto-exposure cold boot", + "count": 2 + }, + { + "job": "test-browser", + "count": 2 + }, + { + "job": "test-unit (1/4)", + "count": 2 + }, + { + "job": "golden-path-template (platformer)", + "count": 1 + }, + { + "job": "integration / auto-exposure", + "count": 1 + }, + { + "job": "integration / capture", + "count": 1 + }, + { + "job": "native-platforms / iOS simulator runtime and no-Xcode consumer handoff", + "count": 1 + }, + { + "job": "native-platforms / macOS desktop core", + "count": 1 + }, + { + "job": "template-nonvisual (platformer)", + "count": 1 + }, + { + "job": "test-playtest", + "count": 1 + } + ] + }, + { + "id": 15, + "at": "2026-10-09T19:18:15.486Z", + "commit": "4c1bbd890", + "label": "hourly refresh", + "notes": "No new PR merged since the last tick. The window still has 2 merged PRs, 470 and 458. Develop still sits at b12b257f1, so the queue has no new merges. It now has 36 merge-group runs, up from 35. Run success is 20 percent, down from 21.2. Board median is 95.1 min, unchanged. Queue wait p90 is 38.6 min over 2476 jobs. First failing job in the 16 red groups: integration / velocity 5, test-unit (3/4) 5, test-browser 3, test-unit (2/4) 3. test-browser is back in the list. PR 476 is still open. Provisional: fewer than 5 PRs merged.", + "decision": "provisional", + "source": "measured", + "window": { + "since": "2026-10-09", + "base": "develop" + }, + "metrics": { + "lastPushToMergeMedianMin": 475.9, + "lastPushToMergeP90Min": 477.1, + "firstAttemptSuccessPct": 50, + "boardWallMedianMin": 95.1, + "queueWaitP90Min": 38.6 + }, + "extra": { + "prCount": 2, + "mergeGroupRuns": 36, + "mergeGroupRunSuccessPct": 20, + "enqueueToMergeMedianMin": 253.5, + "queueWaitJobs": 2476 + }, + "prs": [ + { + "number": 470, + "title": "PRD-461: view distance basics — terrain radius, near-only colliders, recipe fog", + "lastPushToMergeMin": 477.4, + "enqueueToMergeMin": 363.9, + "attempts": 4, + "firstAttempt": "success", + "firstFailingJob": "integration / velocity" + }, + { + "number": 458, + "title": "test(PRD-536): tests close the fail-open and regression holes", + "lastPushToMergeMin": 474.3, + "enqueueToMergeMin": 143.1, + "attempts": 2, + "firstAttempt": "failure", + "firstFailingJob": "test-unit (2/4)" + } + ], + "firstFailingJobs": [ + { + "job": "integration / velocity", + "count": 5 + }, + { + "job": "test-unit (3/4)", + "count": 5 + }, + { + "job": "test-browser", + "count": 3 + }, + { + "job": "test-unit (2/4)", + "count": 3 + }, + { + "job": "integration / auto-exposure", + "count": 2 + }, + { + "job": "integration / auto-exposure cold boot", + "count": 2 + }, + { + "job": "test-unit (1/4)", + "count": 2 + }, + { + "job": "golden-path-template (platformer)", + "count": 1 + }, + { + "job": "integration / capture", + "count": 1 + }, + { + "job": "native-platforms / iOS simulator runtime and no-Xcode consumer handoff", + "count": 1 + }, + { + "job": "native-platforms / macOS desktop core", + "count": 1 + }, + { + "job": "template-nonvisual (platformer)", + "count": 1 + }, + { + "job": "test-playtest", + "count": 1 + } + ] + }, + { + "id": 16, + "at": "2026-10-09T20:30:36.009Z", + "commit": "4d81de73c", + "label": "hourly refresh", + "notes": "The window merged five PRs, which are 458, 470, 471, 473, and 468, and this matches the merged PR line that shows 5 PRs. The window held 42 merge group runs, with a run success of 19.4 percent and a first attempt merge of 40 percent. The merge group board median wall was 95.1 minutes, and the last push to merge median was 477.4 minutes with a p90 of 1023.9 minutes. The runner queue wait p90 was 36.9 minutes over 2845 jobs. The first failing job counts, largest first, were 5 for integration / velocity, 5 for test-unit (3/4), 4 for test-unit (2/4), 3 for test-browser, 2 for integration / auto-exposure, 2 for integration / auto-exposure cold boot, 2 for test-unit (1/4), and 1 for each of golden-path-template (platformer), integration / capture, native-platforms / iOS simulator runtime and no-Xcode consumer handoff, native-platforms / macOS desktop core, template-nonvisual (platformer), and test-playtest. Develop moved from b12b257f1 to 6b18d913e, and its CI run on that head was still queued when this row was written. None of these PRs carries the phase 1 change on PR 476, which is still open, so this window cannot judge that change, and the decision is inconclusive.", + "decision": "inconclusive", + "source": "measured", + "window": { + "since": "2026-10-09", + "base": "develop" + }, + "metrics": { + "lastPushToMergeMedianMin": 477.4, + "lastPushToMergeP90Min": 1023.9, + "firstAttemptSuccessPct": 40, + "boardWallMedianMin": 95.1, + "queueWaitP90Min": 36.9 + }, + "extra": { + "prCount": 5, + "mergeGroupRuns": 42, + "mergeGroupRunSuccessPct": 19.4, + "enqueueToMergeMedianMin": 454.8, + "queueWaitJobs": 2846 + }, + "prs": [ + { + "number": 473, + "title": "fix(world): PRD-494 — bundle-safety walk stops reading pixels; main pass draw budget measured", + "lastPushToMergeMin": 317.9, + "enqueueToMergeMin": 454.8, + "attempts": 5, + "firstAttempt": "failure", + "firstFailingJob": "native-platforms / macOS desktop core" + }, + { + "number": 471, + "title": "fix(capabilities): every manifest import resolves in every scaffolded template (PRD-354)", + "lastPushToMergeMin": 1011.6, + "enqueueToMergeMin": 1015.5, + "attempts": 8, + "firstAttempt": "success", + "firstFailingJob": "test-playtest" + }, + { + "number": 470, + "title": "PRD-461: view distance basics — terrain radius, near-only colliders, recipe fog", + "lastPushToMergeMin": 477.4, + "enqueueToMergeMin": 363.9, + "attempts": 4, + "firstAttempt": "success", + "firstFailingJob": "integration / velocity" + }, + { + "number": 468, + "title": "feat(core): ship threenative-sculpt-mcp 0.2.1", + "lastPushToMergeMin": 1032.1, + "enqueueToMergeMin": 631.5, + "attempts": 7, + "firstAttempt": "failure", + "firstFailingJob": "test-browser" + }, + { + "number": 458, + "title": "test(PRD-536): tests close the fail-open and regression holes", + "lastPushToMergeMin": 474.3, + "enqueueToMergeMin": 143.1, + "attempts": 2, + "firstAttempt": "failure", + "firstFailingJob": "test-unit (2/4)" + } + ], + "firstFailingJobs": [ + { + "job": "integration / velocity", + "count": 5 + }, + { + "job": "test-unit (3/4)", + "count": 5 + }, + { + "job": "test-unit (2/4)", + "count": 4 + }, + { + "job": "test-browser", + "count": 3 + }, + { + "job": "integration / auto-exposure", + "count": 2 + }, + { + "job": "integration / auto-exposure cold boot", + "count": 2 + }, + { + "job": "test-unit (1/4)", + "count": 2 + }, + { + "job": "golden-path-template (platformer)", + "count": 1 + }, + { + "job": "integration / capture", + "count": 1 + }, + { + "job": "native-platforms / iOS simulator runtime and no-Xcode consumer handoff", + "count": 1 + }, + { + "job": "native-platforms / macOS desktop core", + "count": 1 + }, + { + "job": "template-nonvisual (platformer)", + "count": 1 + }, + { + "job": "test-playtest", + "count": 1 + } + ] + }, + { + "id": 17, + "at": "2026-10-09T21:26:04.973Z", + "commit": "0a4ce848c", + "label": "hourly refresh", + "notes": "Merged into develop since 2026-10-09 are 5 PRs, 473, 471, 470, 468 and 458. The merge-group run count is 42 and the run success percent is 18.4 with the first-attempt merge percent at 40. The board median wall is 95.1 minutes, and the last push to merge median is 477.4 minutes and p90 is 1023.9 minutes. The queue wait p90 is 40.4 minutes over 2873 jobs. The first failing job counts are 6 for integration / velocity, 5 for test-unit (2/4), 5 for test-unit (3/4), 3 for test-browser, 2 for integration / auto-exposure, 2 for integration / auto-exposure cold boot, 2 for test-unit (1/4), and 1 each for golden-path-template (platformer), integration / capture, native-platforms / iOS simulator runtime and no-Xcode consumer handoff, native-platforms / macOS desktop core, template-nonvisual (platformer) and test-playtest. Develop head is 6b18d913e and its CI run has finished with success. None of these PRs carry the phase 1 change on PR 476, which is still open, so this window cannot judge that change, and local commit 0a4ce848c adds a test only diagnostic and does not change any merge-group result, giving an inconclusive decision.", + "decision": "inconclusive", + "source": "measured", + "window": { + "since": "2026-10-09", + "base": "develop" + }, + "metrics": { + "lastPushToMergeMedianMin": 477.4, + "lastPushToMergeP90Min": 1023.9, + "firstAttemptSuccessPct": 40, + "boardWallMedianMin": 95.1, + "queueWaitP90Min": 41.5 + }, + "extra": { + "prCount": 5, + "mergeGroupRuns": 42, + "mergeGroupRunSuccessPct": 18.4, + "enqueueToMergeMedianMin": 454.8, + "queueWaitJobs": 2877 + }, + "prs": [ + { + "number": 473, + "title": "fix(world): PRD-494 — bundle-safety walk stops reading pixels; main pass draw budget measured", + "lastPushToMergeMin": 317.9, + "enqueueToMergeMin": 454.8, + "attempts": 5, + "firstAttempt": "failure", + "firstFailingJob": "native-platforms / macOS desktop core" + }, + { + "number": 471, + "title": "fix(capabilities): every manifest import resolves in every scaffolded template (PRD-354)", + "lastPushToMergeMin": 1011.6, + "enqueueToMergeMin": 1015.5, + "attempts": 8, + "firstAttempt": "success", + "firstFailingJob": "test-playtest" + }, + { + "number": 470, + "title": "PRD-461: view distance basics — terrain radius, near-only colliders, recipe fog", + "lastPushToMergeMin": 477.4, + "enqueueToMergeMin": 363.9, + "attempts": 4, + "firstAttempt": "success", + "firstFailingJob": "integration / velocity" + }, + { + "number": 468, + "title": "feat(core): ship threenative-sculpt-mcp 0.2.1", + "lastPushToMergeMin": 1032.1, + "enqueueToMergeMin": 631.5, + "attempts": 7, + "firstAttempt": "failure", + "firstFailingJob": "test-browser" + }, + { + "number": 458, + "title": "test(PRD-536): tests close the fail-open and regression holes", + "lastPushToMergeMin": 474.3, + "enqueueToMergeMin": 143.1, + "attempts": 2, + "firstAttempt": "failure", + "firstFailingJob": "test-unit (2/4)" + } + ], + "firstFailingJobs": [ + { + "job": "integration / velocity", + "count": 6 + }, + { + "job": "test-unit (2/4)", + "count": 5 + }, + { + "job": "test-unit (3/4)", + "count": 5 + }, + { + "job": "test-browser", + "count": 3 + }, + { + "job": "integration / auto-exposure", + "count": 2 + }, + { + "job": "integration / auto-exposure cold boot", + "count": 2 + }, + { + "job": "test-unit (1/4)", + "count": 2 + }, + { + "job": "golden-path-template (platformer)", + "count": 1 + }, + { + "job": "integration / capture", + "count": 1 + }, + { + "job": "native-platforms / iOS simulator runtime and no-Xcode consumer handoff", + "count": 1 + }, + { + "job": "native-platforms / macOS desktop core", + "count": 1 + }, + { + "job": "template-nonvisual (platformer)", + "count": 1 + }, + { + "job": "test-playtest", + "count": 1 + } + ] + }, + { + "id": 18, + "at": "2026-10-10T05:58:22.141Z", + "commit": "1652670ee", + "label": "PR476 unmerged baseline refresh", + "notes": "PR476 remains unmerged. This is a fresh observation of develop, not a measured effect of PR476. Candidate CI run 38025989864 failed sailing diagnostics; ci-required correctly rejected it.", + "decision": "provisional", + "source": "measured", + "window": { + "since": "2026-10-09", + "base": "develop" + }, + "metrics": { + "lastPushToMergeMedianMin": 744.5, + "lastPushToMergeP90Min": 1130.1, + "firstAttemptSuccessPct": 33.3, + "boardWallMedianMin": 107.9, + "queueWaitP90Min": 42.3 + }, + "extra": { + "prCount": 6, + "mergeGroupRuns": 43, + "mergeGroupRunSuccessPct": 23.8, + "enqueueToMergeMedianMin": 543.2, + "queueWaitJobs": 2963 + }, + "prs": [ + { + "number": 473, + "title": "fix(world): PRD-494 — bundle-safety walk stops reading pixels; main pass draw budget measured", + "lastPushToMergeMin": 317.9, + "enqueueToMergeMin": 454.8, + "attempts": 5, + "firstAttempt": "failure", + "firstFailingJob": "native-platforms / macOS desktop core" + }, + { + "number": 472, + "title": "PRD-297: capability recall is a gated number, PRD done", + "lastPushToMergeMin": 1228.1, + "enqueueToMergeMin": 1160.9, + "attempts": 12, + "firstAttempt": "failure", + "firstFailingJob": "integration / velocity" + }, + { + "number": 471, + "title": "fix(capabilities): every manifest import resolves in every scaffolded template (PRD-354)", + "lastPushToMergeMin": 1011.6, + "enqueueToMergeMin": 1015.5, + "attempts": 8, + "firstAttempt": "success", + "firstFailingJob": "test-playtest" + }, + { + "number": 470, + "title": "PRD-461: view distance basics — terrain radius, near-only colliders, recipe fog", + "lastPushToMergeMin": 477.4, + "enqueueToMergeMin": 363.9, + "attempts": 4, + "firstAttempt": "success", + "firstFailingJob": "integration / velocity" + }, + { + "number": 468, + "title": "feat(core): ship threenative-sculpt-mcp 0.2.1", + "lastPushToMergeMin": 1032.1, + "enqueueToMergeMin": 631.5, + "attempts": 7, + "firstAttempt": "failure", + "firstFailingJob": "test-browser" + }, + { + "number": 458, + "title": "test(PRD-536): tests close the fail-open and regression holes", + "lastPushToMergeMin": 474.3, + "enqueueToMergeMin": 143.1, + "attempts": 2, + "firstAttempt": "failure", + "firstFailingJob": "test-unit (2/4)" + } + ], + "firstFailingJobs": [ + { + "job": "integration / velocity", + "count": 7 + }, + { + "job": "test-unit (2/4)", + "count": 5 + }, + { + "job": "test-unit (3/4)", + "count": 5 + }, + { + "job": "test-browser", + "count": 3 + }, + { + "job": "integration / auto-exposure", + "count": 2 + }, + { + "job": "integration / auto-exposure cold boot", + "count": 2 + }, + { + "job": "test-unit (1/4)", + "count": 2 + }, + { + "job": "golden-path-template (platformer)", + "count": 1 + }, + { + "job": "integration / capture", + "count": 1 + }, + { + "job": "native-platforms / iOS simulator runtime and no-Xcode consumer handoff", + "count": 1 + }, + { + "job": "native-platforms / macOS desktop core", + "count": 1 + }, + { + "job": "template-nonvisual (platformer)", + "count": 1 + }, + { + "job": "test-playtest", + "count": 1 + } + ] + }, + { + "id": 19, + "at": "2026-10-10T07:20:48.411Z", + "commit": "77bb0ee57", + "label": "PR476 candidate 77bb in CI — unmerged baseline", + "notes": "Fresh develop baseline while PR476 remains unmerged. CI run38033595736 targets candidate77bb; verification is pending. This audit does not measure PR476 effect.", + "decision": "provisional", + "source": "measured", + "window": { + "since": "2026-10-09", + "base": "develop" + }, + "metrics": { + "lastPushToMergeMedianMin": 475.9, + "lastPushToMergeP90Min": 1090.9, + "firstAttemptSuccessPct": 25, + "boardWallMedianMin": 107.9, + "queueWaitP90Min": 42 + }, + "extra": { + "prCount": 8, + "mergeGroupRuns": 44, + "mergeGroupRunSuccessPct": 23.8, + "enqueueToMergeMedianMin": 409.4, + "queueWaitJobs": 3035 + }, + "prs": [ + { + "number": 479, + "title": "PRD-567: terrain layers blend by their own height, and the game owns the curve", + "lastPushToMergeMin": 66.4, + "enqueueToMergeMin": 64.9, + "attempts": 1, + "firstAttempt": "cancelled", + "firstFailingJob": null + }, + { + "number": 477, + "title": "feat(core): shadow casters sort themselves into static and moving (PRD-572)", + "lastPushToMergeMin": 91.8, + "enqueueToMergeMin": 56.5, + "attempts": 1, + "firstAttempt": "cancelled", + "firstFailingJob": null + }, + { + "number": 473, + "title": "fix(world): PRD-494 — bundle-safety walk stops reading pixels; main pass draw budget measured", + "lastPushToMergeMin": 317.9, + "enqueueToMergeMin": 454.8, + "attempts": 5, + "firstAttempt": "failure", + "firstFailingJob": "native-platforms / macOS desktop core" + }, + { + "number": 472, + "title": "PRD-297: capability recall is a gated number, PRD done", + "lastPushToMergeMin": 1228.1, + "enqueueToMergeMin": 1160.9, + "attempts": 12, + "firstAttempt": "failure", + "firstFailingJob": "integration / velocity" + }, + { + "number": 471, + "title": "fix(capabilities): every manifest import resolves in every scaffolded template (PRD-354)", + "lastPushToMergeMin": 1011.6, + "enqueueToMergeMin": 1015.5, + "attempts": 8, + "firstAttempt": "success", + "firstFailingJob": "test-playtest" + }, + { + "number": 470, + "title": "PRD-461: view distance basics — terrain radius, near-only colliders, recipe fog", + "lastPushToMergeMin": 477.4, + "enqueueToMergeMin": 363.9, + "attempts": 4, + "firstAttempt": "success", + "firstFailingJob": "integration / velocity" + }, + { + "number": 468, + "title": "feat(core): ship threenative-sculpt-mcp 0.2.1", + "lastPushToMergeMin": 1032.1, + "enqueueToMergeMin": 631.5, + "attempts": 7, + "firstAttempt": "failure", + "firstFailingJob": "test-browser" + }, + { + "number": 458, + "title": "test(PRD-536): tests close the fail-open and regression holes", + "lastPushToMergeMin": 474.3, + "enqueueToMergeMin": 143.1, + "attempts": 2, + "firstAttempt": "failure", + "firstFailingJob": "test-unit (2/4)" + } + ], + "firstFailingJobs": [ + { + "job": "integration / velocity", + "count": 7 + }, + { + "job": "test-unit (2/4)", + "count": 5 + }, + { + "job": "test-unit (3/4)", + "count": 5 + }, + { + "job": "test-browser", + "count": 3 + }, + { + "job": "integration / auto-exposure", + "count": 2 + }, + { + "job": "integration / auto-exposure cold boot", + "count": 2 + }, + { + "job": "test-unit (1/4)", + "count": 2 + }, + { + "job": "golden-path-template (platformer)", + "count": 1 + }, + { + "job": "integration / capture", + "count": 1 + }, + { + "job": "native-platforms / iOS simulator runtime and no-Xcode consumer handoff", + "count": 1 + }, + { + "job": "native-platforms / macOS desktop core", + "count": 1 + }, + { + "job": "template-nonvisual (platformer)", + "count": 1 + }, + { + "job": "test-playtest", + "count": 1 + } + ] + }, + { + "id": 20, + "at": "2026-10-10T12:55:25.811Z", + "commit": "77bb0ee57", + "label": "Current queue comparison — PR476 still unmerged", + "notes": "Refreshed after user requested current queue comparison and continued speed work. PR476 remains unmerged; no measured impact attribution.", + "decision": "provisional", + "source": "measured", + "window": { + "since": "2026-10-09", + "base": "develop" + }, + "metrics": { + "lastPushToMergeMedianMin": 474.3, + "lastPushToMergeP90Min": 1071.3, + "firstAttemptSuccessPct": 22.2, + "boardWallMedianMin": 95.1, + "queueWaitP90Min": 39.6 + }, + "extra": { + "prCount": 9, + "mergeGroupRuns": 50, + "mergeGroupRunSuccessPct": 27.7, + "enqueueToMergeMedianMin": 454.8, + "queueWaitJobs": 3449 + }, + "prs": [ + { + "number": 479, + "title": "PRD-567: terrain layers blend by their own height, and the game owns the curve", + "lastPushToMergeMin": 66.4, + "enqueueToMergeMin": 64.9, + "attempts": 1, + "firstAttempt": "cancelled", + "firstFailingJob": null + }, + { + "number": 477, + "title": "feat(core): shadow casters sort themselves into static and moving (PRD-572)", + "lastPushToMergeMin": 91.8, + "enqueueToMergeMin": 56.5, + "attempts": 1, + "firstAttempt": "cancelled", + "firstFailingJob": null + }, + { + "number": 473, + "title": "fix(world): PRD-494 — bundle-safety walk stops reading pixels; main pass draw budget measured", + "lastPushToMergeMin": 317.9, + "enqueueToMergeMin": 454.8, + "attempts": 5, + "firstAttempt": "failure", + "firstFailingJob": "native-platforms / macOS desktop core" + }, + { + "number": 472, + "title": "PRD-297: capability recall is a gated number, PRD done", + "lastPushToMergeMin": 1228.1, + "enqueueToMergeMin": 1160.9, + "attempts": 12, + "firstAttempt": "failure", + "firstFailingJob": "integration / velocity" + }, + { + "number": 471, + "title": "fix(capabilities): every manifest import resolves in every scaffolded template (PRD-354)", + "lastPushToMergeMin": 1011.6, + "enqueueToMergeMin": 1015.5, + "attempts": 8, + "firstAttempt": "success", + "firstFailingJob": "test-playtest" + }, + { + "number": 470, + "title": "PRD-461: view distance basics — terrain radius, near-only colliders, recipe fog", + "lastPushToMergeMin": 477.4, + "enqueueToMergeMin": 363.9, + "attempts": 4, + "firstAttempt": "success", + "firstFailingJob": "integration / velocity" + }, + { + "number": 469, + "title": "fix(shooter): clear ADS sight window, remove GTAO square behind smoke", + "lastPushToMergeMin": 125.9, + "enqueueToMergeMin": 1091.6, + "attempts": 7, + "firstAttempt": "failure", + "firstFailingJob": "test-unit (2/4)" + }, + { + "number": 468, + "title": "feat(core): ship threenative-sculpt-mcp 0.2.1", + "lastPushToMergeMin": 1032.1, + "enqueueToMergeMin": 631.5, + "attempts": 7, + "firstAttempt": "failure", + "firstFailingJob": "test-browser" + }, + { + "number": 458, + "title": "test(PRD-536): tests close the fail-open and regression holes", + "lastPushToMergeMin": 474.3, + "enqueueToMergeMin": 143.1, + "attempts": 2, + "firstAttempt": "failure", + "firstFailingJob": "test-unit (2/4)" + } + ], + "firstFailingJobs": [ + { + "job": "integration / velocity", + "count": 7 + }, + { + "job": "test-unit (2/4)", + "count": 5 + }, + { + "job": "test-unit (3/4)", + "count": 5 + }, + { + "job": "test-browser", + "count": 3 + }, + { + "job": "golden-path-template (platformer)", + "count": 2 + }, + { + "job": "integration / auto-exposure", + "count": 2 + }, + { + "job": "integration / auto-exposure cold boot", + "count": 2 + }, + { + "job": "native-platforms / macOS desktop core", + "count": 2 + }, + { + "job": "test-unit (1/4)", + "count": 2 + }, + { + "job": "integration / capture", + "count": 1 + }, + { + "job": "native-platforms / iOS simulator runtime and no-Xcode consumer handoff", + "count": 1 + }, + { + "job": "template-nonvisual (platformer)", + "count": 1 + }, + { + "job": "test-playtest", + "count": 1 + } + ] + } + ] +} diff --git a/package.json b/package.json index 205a2bfc5c..cbfe3b1ae8 100644 --- a/package.json +++ b/package.json @@ -27,9 +27,11 @@ "check:docs": "tsx scripts/check-doc-links.ts", "check:flight-cost": "tsx scripts/check-flight-cost.ts --max-mean-ms 0.35", "ci:fast": "bash scripts/ci-fast.sh", + "ci:latency": "node scripts/ci-merge-latency.mjs", "ci:local": "bash scripts/ci-local.sh", "ci:runners": "bash scripts/ci-runners.sh", "ci:runners:install": "bash scripts/ci-runners.sh install", + "ci:speed": "node scripts/ci-speed-loop.mjs", "coverage": "vitest run --coverage", "gate:doctor": "tsx scripts/gate-cli.ts doctor", "gate:resume": "tsx scripts/gate-cli.ts resume", diff --git a/packages/create-threenative/__tests__/scaffold.spec.ts b/packages/create-threenative/__tests__/scaffold.spec.ts index e1ef4417c9..c2bac8ac73 100644 --- a/packages/create-threenative/__tests__/scaffold.spec.ts +++ b/packages/create-threenative/__tests__/scaffold.spec.ts @@ -1107,7 +1107,7 @@ describe("create-threenative", () => { } finally { await rm(root, { recursive: true, force: true }); } - }, 15_000); + }); it("should parse the no-install and template flags", () => { expect(parseArgs(["my-game", "--template", "minimal", "--no-install"])).toEqual({ diff --git a/packages/create-threenative/__tests__/template-assets-compile-1.spec.ts b/packages/create-threenative/__tests__/template-assets-compile-1.spec.ts index 77b53fe820..dd8bda2473 100644 --- a/packages/create-threenative/__tests__/template-assets-compile-1.spec.ts +++ b/packages/create-threenative/__tests__/template-assets-compile-1.spec.ts @@ -15,7 +15,11 @@ import { templates, } from "./template-assets-compile-cases.js"; -describe("shipped templates", () => { +// A template compile is tens of seconds of real work on an idle machine: the first half measured +// 1.0-44.6 s per template locally, and a merge-group runner shared with twenty jobs is 2-3 times +// slower, so the shared 60 s unit-test ceiling turned a compile that was merely concurrent into +// `Test timed out in 60000ms`. PRD-550 phase 1: the budget follows the work, not the default. +describe("shipped templates", { timeout: 180_000 }, () => { it.each(firstHalf)( "%s reaches the uncooked budget with an eligible source probe", reachesTheUncookedBudgetWithAnEligibleSourceProbe, diff --git a/packages/create-threenative/__tests__/template-assets-compile-2.spec.ts b/packages/create-threenative/__tests__/template-assets-compile-2.spec.ts index 1294f4b2c8..8660ac27bc 100644 --- a/packages/create-threenative/__tests__/template-assets-compile-2.spec.ts +++ b/packages/create-threenative/__tests__/template-assets-compile-2.spec.ts @@ -15,7 +15,9 @@ import { secondHalf, } from "./template-assets-compile-cases.js"; -describe("shipped templates", () => { +// The twin of `template-assets-compile-1.spec.ts`, with the same budget for the same reason: a +// template compile is tens of seconds of real work, not a unit test's 60 s default. +describe("shipped templates", { timeout: 180_000 }, () => { it.each(secondHalf)( "%s reaches the uncooked budget with an eligible source probe", reachesTheUncookedBudgetWithAnEligibleSourceProbe, diff --git a/scripts/__tests__/ci-attempt-receipts.spec.ts b/scripts/__tests__/ci-attempt-receipts.spec.ts index 4e3d53c999..358822bb74 100644 --- a/scripts/__tests__/ci-attempt-receipts.spec.ts +++ b/scripts/__tests__/ci-attempt-receipts.spec.ts @@ -1,4 +1,4 @@ -import { beforeEach, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, expect, it, vi } from "vitest"; const execute = vi.hoisted(() => vi.fn()); vi.mock("node:child_process", () => ({ execFileSync: execute })); const { readAttemptJobs } = await import( @@ -8,6 +8,10 @@ const identity = { repository: "owner/repo", runId: "123", runAttempt: "2" }; const complete = JSON.stringify([{ total_count: 1, jobs: [{ id: 10, run_attempt: 2 }] }]); beforeEach(() => { execute.mockReset(); + vi.spyOn(Atomics, "wait").mockReturnValue("timed-out"); +}); +afterEach(() => { + vi.restoreAllMocks(); }); const failure = (stderr: string) => Object.assign(new Error(stderr), { stderr, status: 1 }); it("retries a transport outage against the same exact attempt endpoint", () => { @@ -19,12 +23,48 @@ it("retries a transport outage against the same exact attempt endpoint", () => { expect(execute).toHaveBeenCalledTimes(2); expect(execute.mock.calls[0]).toEqual(execute.mock.calls[1]); }); -it("fails closed after three connection failures without returning a receipt", () => { +it("fails closed after five connection failures without returning a receipt", () => { execute.mockImplementation(() => { throw failure("error connecting to api.github.com"); }); expect(() => readAttemptJobs(identity)).toThrow("error connecting"); + expect(execute).toHaveBeenCalledTimes(5); +}); +it("resolves a valid one-page response after two connection failures", () => { + execute.mockImplementationOnce(() => { + throw failure("error connecting to api.github.com"); + }); + execute.mockImplementationOnce(() => { + throw failure("error connecting to api.github.com"); + }); + execute.mockReturnValueOnce(complete); + expect(readAttemptJobs(identity)).toEqual({ + totalCount: 1, + jobs: [{ id: 10, run_attempt: 2 }], + }); expect(execute).toHaveBeenCalledTimes(3); + expect(execute.mock.calls[0]).toEqual(execute.mock.calls[1]); + expect(execute.mock.calls[0]).toEqual(execute.mock.calls[2]); + expect(vi.mocked(Atomics.wait).mock.calls.map((call) => call[3])).toEqual([1000, 2000]); +}); +it("throws after five connection failures with four exponential backoffs", () => { + const error = failure("error connecting to api.github.com"); + execute.mockImplementation(() => { + throw error; + }); + expect(() => readAttemptJobs(identity)).toThrow(error); + expect(execute).toHaveBeenCalledTimes(5); + expect(vi.mocked(Atomics.wait).mock.calls.map((call) => call[3])).toEqual([ + 1000, 2000, 4000, 8000, + ]); +}); +it("throws on the first HTTP 403 failure without connection text", () => { + execute.mockImplementation(() => { + throw failure("HTTP 403"); + }); + expect(() => readAttemptJobs(identity)).toThrow("HTTP 403"); + expect(execute).toHaveBeenCalledTimes(1); + expect(Atomics.wait).not.toHaveBeenCalled(); }); it.each([ "HTTP 403: Resource not accessible by integration", diff --git a/scripts/__tests__/ci-efficiency.spec.ts b/scripts/__tests__/ci-efficiency.spec.ts index 08143b2cb8..158567ba4d 100644 --- a/scripts/__tests__/ci-efficiency.spec.ts +++ b/scripts/__tests__/ci-efficiency.spec.ts @@ -220,10 +220,22 @@ describe("PRD-380 an ordinary pull request owes only the Linux native rows", () expect( (ordinary.jobs as Record)["native-platforms"]?.required, ).toBe(true); - expect(classify(["--event-name", "pull_request", "--target", "develop"], "github")).toContain( - "native_tier=full\n", + // An unresolved live pull request rejects before fanout rather than emit a plan it cannot prove. + const unresolved = spawnSync( + process.execPath, + [ + path.join(repo, "scripts/ci-change-scope.mjs"), + ...["--event-name", "pull_request", "--target", "develop"], + ...["--candidate-sha", candidate, "--format", "github"], + ], + { cwd: repo, encoding: "utf8" }, ); - // Queue qualification owes all supported systems regardless of the target branch. + expect(unresolved.status, unresolved.stderr).toBe(2); + expect(unresolved.stderr).toContain("CI_REQUIRED_PR_CANDIDATE_MISMATCH"); + expect(unresolved.stdout).not.toContain("plan="); + // A queue entry whose base/head this classifier cannot resolve still owes every row: with no + // proven candidate identity it has no diff to narrow by. (A resolved develop queue run narrows + // exactly like a pull request — ci-qualification.spec.ts proves that case.) expect( JSON.parse(classify(["--event-name", "merge_group", "--target", "develop"])), ).toMatchObject({ nativeTier: "full" }); @@ -705,6 +717,67 @@ describe("PRD-373 fixed full candidates and current package products", () => { } }); + // A stale recorded base must reject before a plan is emitted, exactly as the verdict's parent check does. + it("rejects a pull-request candidate whose recorded base is stale before emitting a plan", () => { + const root = makeTempDirSync("ci-stale-base-"); + const git = (...args: string[]) => { + const result = spawnSync("git", args, { cwd: root, encoding: "utf8" }); + expect(result.status, result.stderr).toBe(0); + return result.stdout.trim(); + }; + try { + git("init", "-q", "--initial-branch=main"); + git("config", "user.email", "ci@example.invalid"); + git("config", "user.name", "CI fixture"); + writeFileSync(path.join(root, "seed"), "seed"); + git("add", "."); + git("commit", "-qm", "seed"); + git("checkout", "-qb", "develop"); + writeFileSync(path.join(root, "feature"), "feature"); + git("add", "."); + git("commit", "-qm", "feature"); + const head = git("rev-parse", "HEAD"); + git("checkout", "-q", "main"); + writeFileSync(path.join(root, "base"), "base"); + git("add", "."); + git("commit", "-qm", "base"); + const recordedBase = git("rev-parse", "HEAD"); + writeFileSync(path.join(root, "newer"), "newer"); + git("add", "."); + git("commit", "-qm", "moved base"); + const currentBase = git("rev-parse", "HEAD"); + git("merge", "--no-ff", "-qm", "candidate", "develop"); + const live = ["--event-name", "pull_request", "--target", "develop", "--format", "github"]; + const run = (...args: string[]) => + spawnSync(process.execPath, [path.join(repo, "scripts/ci-change-scope.mjs"), ...args], { + cwd: root, + encoding: "utf8", + env: { ...process.env }, + }); + // The same merge fixture proves the guard accepts the candidate's own base and head. + const valid = run("--base", currentBase, "--head", head, ...live); + expect(valid.status, valid.stderr).toBe(0); + expect(valid.stdout).toContain("plan="); + // Every stale or incomplete base/head reaches the live guard, so each rejects before a plan. + const rejectedCases: readonly (readonly string[])[] = [ + ["--base", recordedBase, "--head", head], + ["--base", recordedBase, "--head", head, "--target", "main"], + ["--base", recordedBase, "--head", head, "--full"], + ["--head", head], + ["--base", currentBase], + ]; + for (const args of rejectedCases) { + // live first so a per-case --target (main) overrides the shared develop default. + const rejected = run(...live, ...args); + expect(rejected.status, rejected.stderr).toBe(2); + expect(rejected.stderr).toContain("CI_REQUIRED_PR_CANDIDATE_MISMATCH"); + expect(rejected.stdout).not.toContain("plan="); + } + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + it("does not label-exempt desktop parity after selecting the full family", () => { const native = readFileSync(path.join(repo, ".github/workflows/native-platforms.yml"), "utf8"); const parity = new Map(jobSections(native)).get("desktop-parity") ?? ""; diff --git a/scripts/__tests__/ci-integration-scope.spec.ts b/scripts/__tests__/ci-integration-scope.spec.ts index 1f402ae3b6..93549bfe7e 100644 --- a/scripts/__tests__/ci-integration-scope.spec.ts +++ b/scripts/__tests__/ci-integration-scope.spec.ts @@ -306,7 +306,10 @@ describe("integration exact Git source selection", () => { baseSha: base, }); expect(preflight.candidateSha).toBe(head); - expect(preflight.lanes).toEqual(all); + // The queue is narrowed like a pull request into develop, so its inventory comes from the + // queue's own diff rather than from the whole workflow. + expect(preflight.qualification).toBe(false); + expect(preflight.lanes).toEqual({ ...none, csg: true }); // Deliberate gate retirement must be reflected in every exhaustive event inventory. for (const eventName of ["pull_request", "merge_group", "workflow_dispatch", "push"]) { const target = eventName === "push" ? "main" : "develop"; @@ -330,12 +333,7 @@ describe("integration exact Git source selection", () => { expect(inventory.jobs).toContain("fluid-collision"); expect(inventory.jobs).toContain("fluid-collision-native"); } - expect(preflight.jobs).toEqual( - [...workflow.matchAll(/^ {2}([a-z][a-z0-9-]*):\n/gmu)] - .map((m) => m[1]) - .filter((id) => id !== "paths") - .sort(), - ); + expect(preflight.jobs).toEqual(["csg"]); const reviewPlan = selectionPlan( "ci", "review exemption", @@ -345,12 +343,22 @@ describe("integration exact Git source selection", () => { 0, "develop", ); - expect(() => + // PRD-550: a narrowed plan is a develop review or queue entry. Everything else still owes the + // exhaustive inventory. + expect( integrationCandidatePreflight({ plan: reviewPlan, eventName: "merge_group", target: "develop", baseSha: base, + }).candidateSha, + ).toBe(head); + expect(() => + integrationCandidatePreflight({ + plan: reviewPlan, + eventName: "workflow_dispatch", + target: "develop", + baseSha: base, }), ).toThrow("CI_REQUIRED_QUALIFICATION_MINIMUM"); expect(() => diff --git a/scripts/__tests__/ci-needs.spec.ts b/scripts/__tests__/ci-needs.spec.ts index ba99147896..7e324d6459 100644 --- a/scripts/__tests__/ci-needs.spec.ts +++ b/scripts/__tests__/ci-needs.spec.ts @@ -7,6 +7,8 @@ import { makeTempDirSync } from "../../test-support/temp-dir.js"; import { formatJobTimings, formatRunSummary, summaryRows } from "../ci-run-summary.js"; import { ciJobGraph, ciNeedsFindings, declaredNeeds, jobSections } from "../ci-workflow.js"; +const { selectionPlan } = await import(new URL("../ci-change-scope.mjs", import.meta.url).href); + const repo = path.resolve(import.meta.dirname, "../.."); const ciPath = path.join(repo, ".github/workflows/ci.yml"); @@ -889,8 +891,8 @@ describe("PRD-481 the runner class is compared per job, because every board is m }); }); -/** The verdict gate on a non-reuse plan, with every job's own result supplied by the caller. */ -function verifyWarmPlan( +/** The verdict gate on a narrowed plan, with every job's own result supplied by the caller. */ +function verifyNarrowedPlan( fixture: IReuseFixture, plan: Record, results: Record, @@ -908,8 +910,11 @@ function verifyWarmPlan( env: { ...process.env, TN_CI_NEEDS: JSON.stringify(needs), - TN_CI_EVENT: "push", + // The merge queue is a develop review lane, so it carries the narrowed plans the gate reads. + TN_CI_EVENT: "merge_group", TN_CI_BASE_REF: "develop", + TN_CI_BASE_SHA: fixture.candidate, + TN_CI_HEAD_SHA: fixture.candidate, GITHUB_ACTIONS: "true", GITHUB_REPOSITORY: "three-native/fixture", GITHUB_RUN_ID: String(SELF_RUN_ID), @@ -917,13 +922,13 @@ function verifyWarmPlan( }); } -describe("PRD-481 a develop push only warms the caches every pull request reads", () => { +describe("a develop push qualifies its tree and still warms the caches every pull request reads", () => { // GitHub scopes a pull request's cache to `refs/pull/N/merge`, which no other pull request can // read. Only a run on the base branch publishes something every pull request into develop reads, // and before this lane nothing ran on develop at all — so every pull request paid a full native // rebuild (199 s of host plus ~120 s of V8/QuickJS contract executables) and a workspace dist // build of its own. - it("selects warm on a develop push, with only the two cache producers required", () => { + it("qualifies a develop push with the full board and still requires both cache producers", () => { const fixture = reuseFixture(); fakeActionsApi(fixture); listSourceRun(fixture); @@ -931,16 +936,17 @@ describe("PRD-481 a develop push only warms the caches every pull request reads" event: "push", target: "develop", }); - expect(plan).toMatchObject({ selection: "warm", reusedRunId: 0 }); + // PRD-550: a push promotes an exact tree, so it runs every job and every native row. Both cache + // publishers are part of that board, so the base branch stays warm for every pull request. + expect(plan).toMatchObject({ selection: "full", qualification: true, reusedRunId: 0 }); const jobs = plan.jobs as Record; expect( Object.entries(jobs) .filter(([, job]) => job.required) .map(([name]) => name) .sort(), - ).toEqual(["build-artifacts", "test-native"]); - // No gate ran, so the run owes no native matrix and asks for no reused verdict. - expect(plan.nativeTier).toBe("none"); + ).toEqual(Object.keys(jobs).sort()); + expect(plan.nativeTier).toBe("full"); }); it("keeps a push to main on the full board", () => { @@ -951,7 +957,7 @@ describe("PRD-481 a develop push only warms the caches every pull request reads" expect(plan).toMatchObject({ selection: "full", nativeTier: "full" }); }); - it("leaves a pull request and the merge queue on develop exactly as they were", () => { + it("leaves a pull request and the merge queue on develop on the same plan", () => { const fixture = reuseFixture(); fakeActionsApi(fixture); listSourceRun(fixture); @@ -996,20 +1002,27 @@ describe("PRD-481 a develop push only warms the caches every pull request reads" }); }); - it("passes the warm verdict on its two producers and fails without them", () => { + it("passes a narrowed queue plan on its two producers and fails without them", () => { + // The gate reads the plan's own required set, whatever the event. `warm` still exists as a + // selection, so it still proves the gate asks for its producers and nothing more. const fixture = reuseFixture(); fakeActionsApi(fixture); - const plan = classifyCandidate(fixture, fixture.candidate, { - event: "push", - target: "develop", - }); - const green = verifyWarmPlan(fixture, plan, { + const plan = selectionPlan( + "warm", + "cache publishers only", + [], + fixture.candidate, + false, + 0, + "develop", + ); + const green = verifyNarrowedPlan(fixture, plan, { "build-artifacts": "success", "test-native": "success", }); expect(green.status, green.stdout + green.stderr).toBe(0); expect(green.stdout).toContain("build-artifacts: required (success)"); - const red = verifyWarmPlan(fixture, plan, { "build-artifacts": "success" }); + const red = verifyNarrowedPlan(fixture, plan, { "build-artifacts": "success" }); expect(red.status).toBe(1); expect(red.stderr).toContain("CI_REQUIRED_JOB_NOT_SUCCESS: test-native"); }); diff --git a/scripts/__tests__/ci-qualification.spec.ts b/scripts/__tests__/ci-qualification.spec.ts index b11761ed28..4f45fc9221 100644 --- a/scripts/__tests__/ci-qualification.spec.ts +++ b/scripts/__tests__/ci-qualification.spec.ts @@ -4,7 +4,7 @@ import path from "node:path"; import { describe, expect, it } from "vitest"; import { makeTempDirSync } from "../../test-support/temp-dir.js"; -const { classify, selectionPlan, TEMPLATE_NAMES, validatePlan } = await import( +const { classify, selectionPlan, TEMPLATE_NAMES, validateEventPlan, validatePlan } = await import( new URL("../ci-change-scope.mjs", import.meta.url).href ); const gate = new URL("../ci-required.mjs", import.meta.url).pathname; @@ -64,18 +64,43 @@ describe("exact-candidate qualification minimum", () => { rmSync(f.root, { recursive: true, force: true }); } }); + // PRD-550: the queue is no longer the exhaustive board. A merge group tests the exact tree a merge + // would produce on the base branch, so it earns the same selection plan a pull request into + // develop earns — narrowed by the same changed paths, with the same reduced native tier. it.each([ - "docs/inert.md", - "scripts/ci-required.mjs", - "packages/core/src/scene.ts", - "packages/create-threenative/templates/shooter/src/main.ts", - ])("requires every supported system and kit for queue input %s", (file) => { - const f = fixture(file); + ["docs/inert.md", "prose"], + ["scripts/ci-required.mjs", "ci"], + ["packages/create-threenative/templates/shooter/src/main.ts", "template"], + ["packages/core/src/scene.ts", "full"], + ])( + "gives a develop merge group the plan a develop pull request gets for %s", + (file, selection) => { + const f = fixture(file); + try { + const queue = classify({ + ...f, + candidateSha: f.head, + target: "refs/heads/develop", + eventName: "merge_group", + }); + expect(queue).toMatchObject({ selection, qualification: false }); + expect(queue).toEqual( + classify({ ...f, candidateSha: f.head, target: "develop", eventName: "pull_request" }), + ); + expect(validatePlan(queue)).toEqual(queue); + } finally { + rmSync(f.root, { recursive: true, force: true }); + } + }, + ); + + it.each(["main", "release"])("still qualifies a queue entry targeting %s", (target) => { + const f = fixture("docs/inert.md"); try { const plan = classify({ ...f, candidateSha: f.head, - target: "refs/heads/develop", + target, eventName: "merge_group", }); expect(plan).toMatchObject({ @@ -97,6 +122,23 @@ describe("exact-candidate qualification minimum", () => { } }); + // A develop push promotes an exact tree, so it runs the exhaustive board instead of the cache-only + // lane, and it keeps publishing the two base-branch caches because a full selection runs both. + it.each(["develop", "main"])("qualifies a push to %s with the whole board", (target) => { + const f = fixture("docs/inert.md"); + try { + expect(classify({ ...f, candidateSha: f.head, target, eventName: "push" })).toMatchObject({ + selection: "full", + qualification: true, + native: true, + nativeTier: "full", + reusedRunId: 0, + }); + } finally { + rmSync(f.root, { recursive: true, force: true }); + } + }); + it.each(["schedule", "workflow_dispatch", "unknown"])( "keeps %s exhaustive on develop", (eventName) => { @@ -120,8 +162,77 @@ describe("exact-candidate qualification minimum", () => { }, ); + it("accepts a narrowed develop queue plan and refuses the same plan on every other event", () => { + const f = fixture("docs/inert.md"); + try { + const plan = selectionPlan( + "prose", + "narrow review", + ["docs/inert.md"], + f.head, + false, + 0, + "develop", + ); + expect( + validateEventPlan(plan, { eventName: "merge_group", baseRef: "refs/heads/develop" }), + ).toEqual(plan); + for (const eventName of ["push", "schedule", "workflow_dispatch"]) { + expect(() => validateEventPlan(plan, { eventName, baseRef: "develop" })).toThrow( + "CI_REQUIRED_QUALIFICATION_MINIMUM", + ); + } + expect(() => validateEventPlan(plan, { eventName: "merge_group", baseRef: "main" })).toThrow( + "CI_REQUIRED_QUALIFICATION_MINIMUM", + ); + expect(() => + validateEventPlan(plan, { + eventName: "merge_group", + baseRef: "develop", + forceFull: true, + }), + ).toThrow("CI_REQUIRED_QUALIFICATION_MINIMUM"); + } finally { + rmSync(f.root, { recursive: true, force: true }); + } + }); + + it("requires an explicit develop target to keep a reduced queue plan", () => { + const f = fixture("docs/inert.md"); + try { + const plan = selectionPlan( + "prose", + "narrow review", + ["docs/inert.md"], + f.head, + false, + 0, + "develop", + ); + // develop, bare or prefixed, keeps the reduced review the queue earns. + for (const baseRef of ["develop", "refs/heads/develop"]) { + expect(validateEventPlan(plan, { eventName: "merge_group", baseRef })).toEqual(plan); + } + // main, bare or prefixed, and any unknown or missing target must reject a review plan. + for (const baseRef of [ + "main", + "refs/heads/main", + "refs/heads/release", + "release", + "", + undefined, + ]) { + expect(() => validateEventPlan(plan, { eventName: "merge_group", baseRef })).toThrow( + "CI_REQUIRED_QUALIFICATION_MINIMUM", + ); + } + } finally { + rmSync(f.root, { recursive: true, force: true }); + } + }); + it.each(["prose", "ci", "warm", "full"])( - "independently rejects a planner-generated %s queue exemption", + "independently rejects a planner-generated %s exemption outside a develop review", (selection) => { const f = fixture("docs/inert.md"); try { @@ -149,7 +260,7 @@ describe("exact-candidate qualification minimum", () => { encoding: "utf8", env: { ...process.env, - TN_CI_EVENT: "merge_group", + TN_CI_EVENT: "workflow_dispatch", TN_CI_BASE_REF: "refs/heads/develop", TN_CI_BASE_SHA: f.base, TN_CI_HEAD_SHA: f.head, diff --git a/scripts/__tests__/ci-runners.spec.ts b/scripts/__tests__/ci-runners.spec.ts index 4a558a3527..02c9ff7912 100644 --- a/scripts/__tests__/ci-runners.spec.ts +++ b/scripts/__tests__/ci-runners.spec.ts @@ -143,3 +143,185 @@ describe.skipIf(process.platform !== "linux")("local runner physical core alloca expect(result.stderr).toContain("affinity"); }); }); + +// Runs `balance` against a fake `gh` and `sleep`: the fake `sleep` advances a tick counter and +// applies the scheduled mutation, so the loop ends deterministically through the real stop file. +function runBalance( + options: { + readonly varsFail?: boolean; + readonly deleteHeavyOnTick?: number; + readonly stopOnTick?: number; + readonly stopBefore?: boolean; + readonly stopDuringVarReadAt?: number; + readonly initialVars?: string; + readonly idle?: string; + } = {}, +) { + const fixture = makeTempDirSync("runner-balance-"); + const bin = path.join(fixture, "bin"); + mkdirSync(bin); + const idleFile = path.join(fixture, "idle"); + const varsFile = path.join(fixture, "vars"); + const failFile = path.join(fixture, "vars-fail"); + const ticksFile = path.join(fixture, "ticks"); + const readsFile = path.join(fixture, "var-reads"); + const callsFile = path.join(fixture, "gh-calls"); + const stateDir = path.join(fixture, "threenative", "ci-runners"); + mkdirSync(stateDir, { recursive: true }); + const stopFile = path.join(stateDir, "stop"); + + writeFileSync(idleFile, options.idle ?? "tn-local tn-local-light\n"); + writeFileSync(varsFile, options.initialVars ?? ""); + writeFileSync(callsFile, ""); + if (options.varsFail) writeFileSync(failFile, ""); + if (options.stopBefore) writeFileSync(stopFile, ""); + + const dropVar = (name: string): string => + `grep -v "^${name}=" '${varsFile}' 2>/dev/null > '${varsFile}.tmp' || true\nmv '${varsFile}.tmp' '${varsFile}'`; + + const gh = `#!/bin/sh +printf '%s\\n' "$*" >> '${callsFile}' +case "$*" in + *"repo view"*) printf 'test/repo\\n'; exit 0 ;; +esac +if [ "$1" = "variable" ]; then + shift + case "$1" in + set) + name="$2"; shift 2; body="" + while [ $# -gt 0 ]; do case "$1" in --body) body="$2"; shift 2 ;; *) shift ;; esac; done + grep -v "^$name=" '${varsFile}' 2>/dev/null > '${varsFile}.tmp' || true + printf '%s=%s\\n' "$name" "$body" >> '${varsFile}.tmp' + mv '${varsFile}.tmp' '${varsFile}' + exit 0 ;; + delete) + ${dropVar("$2")} + exit 0 ;; + get) + value=$(grep "^$2=" '${varsFile}' 2>/dev/null | head -1 | cut -d= -f2-) + [ -n "$value" ] && { printf '%s\\n' "$value"; exit 0; } + exit 1 ;; + esac +fi +if [ "$1" = "api" ]; then + case "$*" in + *"actions/variables"*) + reads=$(cat '${readsFile}' 2>/dev/null || echo 0); reads=$((reads + 1)) + echo "$reads" > '${readsFile}' + ${options.stopDuringVarReadAt === undefined ? ":" : `[ "$reads" -ge ${options.stopDuringVarReadAt} ] && : > '${stopFile}'`} + [ -e '${failFile}' ] && exit 1 + # Same shape as the real --jq filter: the name, or NAME:invalid for a wrong label; never the value. + awk -F= '$1 == "TN_RUNNER" { print ($2 == "tn-local") ? "TN_RUNNER" : "TN_RUNNER:invalid" } + $1 == "TN_RUNNER_LIGHT" { print ($2 == "tn-local-light") ? "TN_RUNNER_LIGHT" : "TN_RUNNER_LIGHT:invalid" }' '${varsFile}' 2>/dev/null + exit 0 ;; + *"actions/runners"*) + cat '${idleFile}' 2>/dev/null; exit 0 ;; + esac +fi +exit 0 +`; + + const mutations: string[] = []; + if (options.deleteHeavyOnTick !== undefined) { + mutations.push( + `[ "$ticks" -eq ${options.deleteHeavyOnTick} ] && {\n ${dropVar("TN_RUNNER")}\n}`, + ); + } + if (options.stopOnTick !== undefined) { + mutations.push(`[ "$ticks" -eq ${options.stopOnTick} ] && : > '${stopFile}'`); + } + const sleep = `#!/bin/sh +ticks=$(cat '${ticksFile}' 2>/dev/null || echo 0); ticks=$((ticks + 1)) +echo "$ticks" > '${ticksFile}' +${mutations.join("\n")} +exit 0 +`; + + for (const [name, body] of [ + ["gh", gh], + ["sleep", sleep], + ] as const) { + const command = path.join(bin, name); + writeFileSync(command, body); + chmodSync(command, 0o755); + } + + const result = spawnSync("bash", [script, "balance"], { + encoding: "utf8", + cwd: fixture, + timeout: 20_000, + env: { + ...process.env, + PATH: `${bin}:${process.env.PATH}`, + HOME: fixture, + XDG_STATE_HOME: fixture, + TN_RUNNERS_ENV: path.join(fixture, "missing-env"), + }, + }); + return { + status: result.status, + calls: readFileSync(callsFile, "utf8"), + vars: readFileSync(varsFile, "utf8"), + }; +} + +describe.skipIf(process.platform !== "linux")("local runner balancer routing", () => { + it("advertises idle capacity only after two consecutive idle polls", () => { + const { status, vars } = runBalance({ stopOnTick: 2 }); + expect(status).toBe(0); + expect(vars).toContain("TN_RUNNER=tn-local\n"); + expect(vars).toContain("TN_RUNNER_LIGHT=tn-local-light\n"); + }); + + it("does not advertise before the second idle poll", () => { + const { status, vars } = runBalance({ stopOnTick: 1 }); + expect(status).toBe(0); + expect(vars).not.toContain("TN_RUNNER="); + }); + + it("wrong-label routing variable: clears a stale label while the pool is busy", () => { + const { status, vars } = runBalance({ + initialVars: "TN_RUNNER=some-other-label\n", + idle: "", + stopOnTick: 2, + }); + expect(status).toBe(0); + expect(vars).not.toContain("TN_RUNNER="); + }); + + it("wrong-label routing variable: corrects it to the expected label once idle is confirmed", () => { + const { status, vars } = runBalance({ + initialVars: "TN_RUNNER=some-other-label\n", + stopOnTick: 2, + }); + expect(status).toBe(0); + expect(vars).toContain("TN_RUNNER=tn-local\n"); + }); + + it("re-advertises a routing variable deleted externally while capacity stays idle", () => { + const { status, calls, vars } = runBalance({ deleteHeavyOnTick: 2, stopOnTick: 4 }); + expect(status).toBe(0); + expect(vars).toContain("TN_RUNNER=tn-local\n"); + expect( + calls.split("\n").filter((line) => line.includes("variable set TN_RUNNER ")), + ).toHaveLength(2); + }); + + it("does not advertise when the routing-variable list cannot be read", () => { + const { status, vars } = runBalance({ varsFail: true, stopOnTick: 3 }); + expect(status).toBe(0); + expect(vars).not.toContain("TN_RUNNER"); + }); + + it("never advertises when the stop file already exists", () => { + const { status, calls } = runBalance({ stopBefore: true }); + expect(status).toBe(0); + expect(calls).not.toContain("variable set"); + }); + + it("stops without re-advertising when a teardown begins mid-poll", () => { + const { status, calls } = runBalance({ stopDuringVarReadAt: 2, stopOnTick: 5 }); + expect(status).toBe(0); + expect(calls).not.toContain("variable set"); + }); +}); diff --git a/scripts/__tests__/ci-speed-loop.spec.ts b/scripts/__tests__/ci-speed-loop.spec.ts new file mode 100644 index 0000000000..5df07e84b4 --- /dev/null +++ b/scripts/__tests__/ci-speed-loop.spec.ts @@ -0,0 +1,283 @@ +import { describe, expect, it } from "vitest"; +const { firstFailingJob, quantile, queuePrNumber, summarize } = await import( + new URL("../ci-merge-latency.mjs", import.meta.url).href +); +const { + GOALS, + appendIteration, + discardIteration, + emptyLedger, + isLive, + meetsGoal, + progressToward, + renderHtml, +} = await import(new URL("../ci-speed-loop.mjs", import.meta.url).href); + +const T = (minute: number) => new Date(Date.UTC(2026, 9, 9, 0, minute)).toISOString(); + +const window = summarize({ + prs: [ + { number: 1, title: "red then green", mergedAt: T(300), lastCommitAt: T(0) }, + { number: 2, title: "straight through", mergedAt: T(120), lastCommitAt: T(100) }, + { number: 3, title: "direct merge, no queue", mergedAt: T(50), lastCommitAt: T(40) }, + ], + runs: [ + { + id: 10, + headBranch: "gh-readonly-queue/develop/pr-1-aaa", + createdAt: T(10), + updatedAt: T(80), + conclusion: "failure", + }, + { + id: 11, + headBranch: "gh-readonly-queue/develop/pr-1-bbb", + createdAt: T(200), + updatedAt: T(280), + conclusion: "success", + }, + { + id: 12, + headBranch: "gh-readonly-queue/develop/pr-2-ccc", + createdAt: T(105), + updatedAt: T(115), + conclusion: "success", + }, + { + id: 13, + headBranch: "refs/heads/not-a-queue-branch", + createdAt: T(1), + updatedAt: T(2), + conclusion: "success", + }, + ], + jobsByRun: { + 10: [ + { + name: "ci-required", + conclusion: "failure", + createdAt: T(79), + startedAt: T(79), + completedAt: T(80), + }, + { + name: "integration / auto-exposure", + conclusion: "failure", + createdAt: T(20), + startedAt: T(30), + completedAt: T(60), + }, + { + name: "native-platforms", + conclusion: "skipped", + createdAt: T(20), + startedAt: null, + completedAt: T(20), + }, + ], + 12: [ + { + name: "test", + conclusion: "success", + createdAt: T(105), + startedAt: T(107), + completedAt: T(110), + }, + ], + }, +}); + +describe("ci-merge-latency", () => { + it("interpolates quantiles and ignores non-finite values", () => { + expect(quantile([10, 20, 30, 40], 0.5)).toBe(25); + expect(quantile([null as unknown as number, 5], 0.9)).toBe(5); + expect(quantile([], 0.5)).toBeNull(); + }); + + it("reads the PR number from a merge-queue branch only", () => { + expect(queuePrNumber("gh-readonly-queue/develop/pr-461-1a2b")).toBe(461); + expect(queuePrNumber("feature/pr-461-x")).toBeNull(); + expect(queuePrNumber(undefined)).toBeNull(); + }); + + it("names the earliest failing job, never the join that finishes last", () => { + expect( + firstFailingJob([ + { name: "ci-required", conclusion: "failure", completedAt: T(80) }, + { name: "test-unit (1/4)", conclusion: "failure", completedAt: T(40) }, + ]), + ).toBe("test-unit (1/4)"); + expect(firstFailingJob([{ name: "x", conclusion: "success", completedAt: T(1) }])).toBeNull(); + }); + + it("measures push to merge, enqueue to merge, attempts and the first attempt per PR", () => { + const [one, two, three] = window.prs; + expect(one).toMatchObject({ + lastPushToMergeMin: 300, + enqueueToMergeMin: 290, + attempts: 2, + firstAttempt: "failure", + firstFailingJob: "integration / auto-exposure", + }); + expect(two).toMatchObject({ + lastPushToMergeMin: 20, + enqueueToMergeMin: 15, + attempts: 1, + firstAttempt: "success", + firstFailingJob: null, + }); + // No queue run: unknown, never 0. + expect(three).toMatchObject({ attempts: 0, firstAttempt: null, enqueueToMergeMin: null }); + }); + + it("summarizes the window", () => { + expect(window.summary).toMatchObject({ + prCount: 3, + lastPushToMergeMedianMin: 20, + firstAttemptSuccessPct: 50, + mergeGroupRuns: 3, + mergeGroupRunSuccessPct: 66.7, + boardWallMedianMin: 45, + }); + // Skipped jobs never started, so they are not queue wait. + expect(window.summary.queueWaitJobs).toBe(3); + expect(window.firstFailingJobs).toEqual([{ job: "integration / auto-exposure", count: 1 }]); + }); + + it("returns null, not 0, for an empty window", () => { + const empty = summarize({ prs: [], runs: [] }); + expect(empty.summary.lastPushToMergeMedianMin).toBeNull(); + expect(empty.summary.firstAttemptSuccessPct).toBeNull(); + expect(empty.summary.queueWaitP90Min).toBeNull(); + }); +}); + +const measurement = (overrides: Record = {}) => ({ + since: "2026-10-09", + base: "develop", + at: "2026-10-09T12:00:00.000Z", + ...window, + summary: { ...window.summary, ...overrides }, +}); + +describe("ci-speed-loop ledger", () => { + it("makes the first iteration the baseline and later ones provisional", () => { + const first = appendIteration(emptyLedger(), { measurement: measurement(), label: "baseline" }); + const second = appendIteration(first, { measurement: measurement(), label: "gating board" }); + expect(second.iterations.map((i: { decision: string }) => i.decision)).toEqual([ + "baseline", + "provisional", + ]); + expect(second.iterations.map((i: { id: number }) => i.id)).toEqual([1, 2]); + }); + + it("rejects an unknown decision and a discard without a reason", () => { + expect(() => + appendIteration(emptyLedger(), { measurement: measurement(), label: "x", decision: "win" }), + ).toThrow(/decision/u); + const ledger = appendIteration(emptyLedger(), { measurement: measurement(), label: "x" }); + expect(() => discardIteration(ledger, 1, "")).toThrow(/reason/u); + expect(() => discardIteration(ledger, 9, "why")).toThrow(/no iteration/u); + }); + + it("keeps a superseded iteration in history and out of the live set", () => { + let ledger = appendIteration(emptyLedger(), { + measurement: measurement({ lastPushToMergeMedianMin: 280 }), + label: "baseline", + }); + ledger = appendIteration(ledger, { + measurement: measurement({ lastPushToMergeMedianMin: 5 }), + label: "bad run", + }); + ledger = discardIteration(ledger, 2, "queue outage, thin sample"); + expect(ledger.iterations).toHaveLength(2); + expect(ledger.iterations.filter(isLive).map((i: { id: number }) => i.id)).toEqual([1]); + const html = renderHtml(ledger); + expect(html).toContain("superseded: queue outage, thin sample"); + expect(html).toContain("#2"); + }); + + it("re-rendering after an append keeps every earlier iteration", () => { + let ledger = appendIteration(emptyLedger(), { + measurement: measurement(), + label: "first change", + }); + const before = renderHtml(ledger); + ledger = appendIteration(ledger, { measurement: measurement(), label: "second change" }); + const after = renderHtml(ledger); + expect(before).toContain("first change"); + expect(after).toContain("first change"); + expect(after).toContain("second change"); + }); + + it("scores progress from the baseline to the goal for both directions", () => { + const lower = GOALS[0]; + const higher = GOALS[2]; + expect(progressToward(lower, 280, 155)).toBeCloseTo(0.5, 1); + expect(progressToward(lower, 280, 20)).toBe(1); + expect(progressToward(lower, 280, 400)).toBe(0); + expect(progressToward(higher, 30, 60)).toBeCloseTo(0.5, 1); + expect(progressToward(lower, null, 20)).toBeNull(); + expect(meetsGoal(lower, 30)).toBe(true); + expect(meetsGoal(higher, 89)).toBe(false); + expect(meetsGoal(lower, null)).toBe(false); + }); +}); + +describe("ci-speed-loop dashboard", () => { + it("says baseline pending for an empty ledger and draws every chart", () => { + const html = renderHtml(emptyLedger()); + expect(html).toContain("Baseline pending"); + expect(html.match(/ { + const hostile = { + ...measurement(), + prs: [ + { + number: 7, + title: "", + lastPushToMergeMin: 1, + enqueueToMergeMin: 1, + attempts: 1, + firstAttempt: "success", + firstFailingJob: null, + }, + ], + }; + const html = renderHtml( + appendIteration(emptyLedger(), { measurement: hostile, label: '">' }), + ); + expect(html).not.toContain("1"); + }); + + it("renders a missing value as n/a, never as 0", () => { + const html = renderHtml( + appendIteration(emptyLedger(), { + measurement: measurement({ queueWaitP90Min: null }), + label: "x", + }), + ); + expect(html).toContain("n/a"); + expect(html).not.toMatch(/Runner queue wait, p90<\/td>0 min/u); + }); + + it("keeps the latest window that merged a PR in the PR table", () => { + const first = appendIteration(emptyLedger(), { measurement: measurement(), label: "baseline" }); + const html = renderHtml( + appendIteration(first, { + measurement: { + ...measurement({ lastPushToMergeMedianMin: null }), + prs: [], + firstFailingJobs: [], + }, + label: "no PR merged", + }), + ); + expect(html).toContain("red then green"); + expect(html).toContain("Merged PRs, iteration #1 window"); + }); +}); diff --git a/scripts/__tests__/ci-structure.spec.ts b/scripts/__tests__/ci-structure.spec.ts index 870e2785c6..7e5cf38589 100644 --- a/scripts/__tests__/ci-structure.spec.ts +++ b/scripts/__tests__/ci-structure.spec.ts @@ -785,8 +785,8 @@ describe("CI pipeline structure", () => { "main", "develop", ]); - // The warm lane is the two jobs that publish those caches, plus the reporting job. Nothing else - // runs, so a develop push costs the native build once rather than a whole board. + // The `warm` plan still exists as a selection, and these are the only jobs it runs: the two + // cache publishers plus the reporting job. for (const name of ["build-artifacts", "test-native", "run-summary"]) { expect(requiredJob(ci, name), `${name} never runs on a develop push`).toContain( "needs.scope.outputs.selection == 'warm'", @@ -1101,6 +1101,25 @@ describe("CI pipeline structure", () => { "android-v8-source", ]), ], + [ + ".github/workflows/integration.yml", + // PRD-550: these generic lanes run hosted so they do not queue behind the local build/test pool. + new Set([ + "animation", + "capture", + "cold-boot", + "csg", + "decals", + "exposure", + "fluid-collision", + "fog", + "ik", + "temporal", + "tone", + "vegetation", + "velocity", + ]), + ], ]); it.each([ ["web-reference", "native-web-reference-${{ needs.scope.outputs.candidate_sha }}"], @@ -1132,11 +1151,14 @@ describe("CI pipeline structure", () => { }, ); - // PRD-480's light lane: the jobs whose whole work is a script or a summary — no workspace build, - // no `pnpm install` and no test suite. `scope` classifies the diff, `build` and `golden-path` + // PRD-480's light lane: the jobs whose whole work is a script or a summary — no workspace build + // and no test suite; a JSON aggregator may still `pnpm install` its own toolchain. `scope` + // classifies the diff, `build` and `golden-path` // assert an upstream verdict, `ci-required` is the merge verdict and `run-summary` writes the - // report; in native-platforms.yml `networking-matrix` is the one join that needs no toolchain at - // all. They join on `tn-local-light` (1 CPU, 2 GB) so a ten-second verdict does not queue behind + // report; in native-platforms.yml `networking-matrix` is the one join that needs no toolchain, and + // `release-reports` and `performance-coverage` only download artifacts and write JSON (no compile + // or GPU leg). They join on + // `tn-local-light` (1 CPU, 2 GB) so a ten-second verdict does not queue behind // five twenty-minute builds. Both directions are load-bearing and both are asserted below: a // heavy job here would build the workspace in one core, and a light job off the list would queue // behind the heavy pool — which is the wait the lane exists to remove. @@ -1147,6 +1169,8 @@ describe("CI pipeline structure", () => { "golden-path", "networking-matrix", "paths", + "performance-coverage", + "release-reports", "run-summary", "scope", ]); @@ -1239,6 +1263,43 @@ describe("CI pipeline structure", () => { } }); + // PRD-550: pin the generic lanes hosted to split the pool; the light switch and native ARM labels stay. + it("pins generic Integration lanes to hosted while the light and native legs keep their routing", async () => { + const source = await readFile(path.join(repo, ".github/workflows/integration.yml"), "utf8"); + const runsOn = (job: string) => requiredJob(source, job).match(/^ {4}runs-on: (.+)$/mu)?.[1]; + for (const job of [ + "animation", + "capture", + "cold-boot", + "csg", + "decals", + "exposure", + "fluid-collision", + "fog", + "ik", + "temporal", + "tone", + "vegetation", + "velocity", + ] as const) { + expect(runsOn(job), `integration ${job} must not share the local build/test pool`).toBe( + "ubuntu-24.04", + ); + } + for (const job of ["paths", "completion"] as const) { + expect(runsOn(job), `integration ${job} keeps the light routing switch`).toBe(lightRouting); + } + for (const job of [ + "animation-native", + "decals-native", + "fluid-collision-native", + "fog-native", + "packaged-quickjs", + ] as const) { + expect(runsOn(job), `integration ${job} keeps its native ARM label`).toBe("ubuntu-24.04-arm"); + } + }); + // A draft PR spends no runner: every job needs `scope`, so skipping it and the always() gate // skips the board, and `ready_for_review` starts it once the draft is marked ready. it("runs nothing on draft pull requests until they are marked ready", async () => { @@ -3597,7 +3658,7 @@ describe("PRD-373 selective feature verification", () => { }, ); - it("requires native for a push to main, and warms nothing but caches on a push to develop", async () => { + it("requires native for a push to main, and qualifies a push to develop with the whole board", async () => { const fixture = await scopeFixture(); try { const head = await commitScopeChange( @@ -3613,16 +3674,21 @@ describe("PRD-373 selective feature verification", () => { "push", ]); expect(main).toMatchObject({ selection: "full", native: true }); - // The cache-warm lane (PRD-481 phase 2) is the one narrowed push, and it owes no native row: - // it publishes the base-branch caches and verifies nothing. + // PRD-550: the exhaustive board left the queue for the tree-qualifying events, so a develop + // push runs every lane and every native row. const develop = classifyScope(fixture.root, fixture.base, head, [ "--target", "develop", "--event-name", "push", ]); - expect(develop).toMatchObject({ selection: "warm", native: false }); - expect(develop.jobs).toMatchObject({ "native-platforms": { required: false } }); + expect(develop).toMatchObject({ + selection: "full", + native: true, + qualification: true, + nativeTier: "full", + }); + expect(develop.jobs).toMatchObject({ "native-platforms": { required: true } }); } finally { await removeFixture(fixture.root); } @@ -3694,7 +3760,7 @@ describe("PRD-373 selective feature verification", () => { ]) { expect(classifyScope(fixture.root, fixture.base, head, extra).selection).toBe("full"); } - for (const event of ["schedule", "workflow_dispatch", "merge_group"]) { + for (const event of ["schedule", "workflow_dispatch"]) { expect( classifyScope(fixture.root, fixture.base, head, [ "--target", @@ -3704,25 +3770,17 @@ describe("PRD-373 selective feature verification", () => { ]).selection, ).toBe("full"); } - // The one narrowed event is a push onto develop, and it is narrow only as far as the caches: - // `warm` still requires the two jobs that publish them, and it requires them on main too — - // where the rest of the board joins in. - expect( - classifyScope(fixture.root, fixture.base, head, [ - "--target", - "develop", - "--event-name", - "push", - ]).selection, - ).toBe("warm"); - expect( - classifyScope(fixture.root, fixture.base, head, [ + // A push qualifies the exact tree it promotes, on develop and on main alike. + for (const target of ["develop", "main"]) { + const push = classifyScope(fixture.root, fixture.base, head, [ "--target", - "main", + target, "--event-name", "push", - ]).selection, - ).toBe("full"); + ]); + expect(push.selection).toBe("full"); + expect(push.qualification).toBe(true); + } } finally { await removeFixture(fixture.root); } @@ -3921,9 +3979,10 @@ describe("a CI-configuration-only pull request", () => { }); it.each([ - [["--event-name", "merge_group", "--target", "develop"]], [["--event-name", "schedule", "--target", "develop"]], [["--event-name", "workflow_dispatch", "--target", "develop"]], + [["--event-name", "merge_group", "--target", "main"]], + [["--event-name", "push", "--target", "develop"]], [["--target", "main"]], ])("never narrows %j", async (extra) => { const fixture = await scopeFixture(); @@ -3943,7 +4002,7 @@ describe("a CI-configuration-only pull request", () => { } }); - it("runs a develop push as the cache-warm lane, never the ci narrowing", async () => { + it("runs a develop push as the exhaustive qualification, never the ci narrowing", async () => { const fixture = await scopeFixture(); try { const head = await commitScopeChange( @@ -3952,14 +4011,14 @@ describe("a CI-configuration-only pull request", () => { "name: CI\njobs: {}\n", "ci configuration", ); - expect( - classifyScope(fixture.root, fixture.base, head, [ - "--event-name", - "push", - "--target", - "develop", - ]).selection, - ).toBe("warm"); + const push = classifyScope(fixture.root, fixture.base, head, [ + "--event-name", + "push", + "--target", + "develop", + ]); + expect(push.selection).toBe("full"); + expect(push.qualification).toBe(true); } finally { await removeFixture(fixture.root); } diff --git a/scripts/__tests__/ci-template-selection.spec.ts b/scripts/__tests__/ci-template-selection.spec.ts index 94c321ed4c..f153a5c387 100644 --- a/scripts/__tests__/ci-template-selection.spec.ts +++ b/scripts/__tests__/ci-template-selection.spec.ts @@ -35,36 +35,39 @@ function fixture() { }; } describe("impact-driven template coverage", () => { - it.each(["pull_request"])("selects only an exact changed kit for %s", (eventName) => { - const f = fixture(); - try { - const head = f.change("packages/create-threenative/templates/shooter/src/game.ts"); - const plan = classify({ - root: f.root, - base: f.base, - head, - candidateSha: head, - target: "develop", - eventName, - }); - expect(plan.selection).toBe("template"); - expect(plan.templateMatrix.template).toEqual(["shooter"]); - expect(plan.goldenMatrix.template).toEqual(["shooter"]); - for (const name of ["test-native", "native-platforms", "test-browser", "benchmark"]) - expect(plan.jobs[name].required).toBe(false); - for (const name of [ - "build-artifacts", - "template-nonvisual", - "golden-path-template", - "supply-chain", - ]) - expect(plan.jobs[name].required).toBe(true); - expect(validatePlan(plan)).toEqual(plan); - } finally { - rmSync(f.root, { recursive: true, force: true }); - } - }); - it.each(["pull_request"])("keeps CI and docs narrow on %s", (eventName) => { + it.each(["pull_request", "merge_group"])( + "selects only an exact changed kit for %s", + (eventName) => { + const f = fixture(); + try { + const head = f.change("packages/create-threenative/templates/shooter/src/game.ts"); + const plan = classify({ + root: f.root, + base: f.base, + head, + candidateSha: head, + target: "develop", + eventName, + }); + expect(plan.selection).toBe("template"); + expect(plan.templateMatrix.template).toEqual(["shooter"]); + expect(plan.goldenMatrix.template).toEqual(["shooter"]); + for (const name of ["test-native", "native-platforms", "test-browser", "benchmark"]) + expect(plan.jobs[name].required).toBe(false); + for (const name of [ + "build-artifacts", + "template-nonvisual", + "golden-path-template", + "supply-chain", + ]) + expect(plan.jobs[name].required).toBe(true); + expect(validatePlan(plan)).toEqual(plan); + } finally { + rmSync(f.root, { recursive: true, force: true }); + } + }, + ); + it.each(["pull_request", "merge_group"])("keeps CI and docs narrow on %s", (eventName) => { for (const [file, selection] of [ ["docs/PRDs/inert.md", "prose"], ["scripts/ci-required.mjs", "ci"], @@ -119,7 +122,7 @@ describe("impact-driven template coverage", () => { rmSync(f.root, { recursive: true, force: true }); } }); - it("normalizes the queue target and includes every constituent template change", () => { + it("normalizes the queue target and unions every constituent template change", () => { const f = fixture(); try { f.change("packages/create-threenative/templates/shooter/src/x.ts"); @@ -132,10 +135,14 @@ describe("impact-driven template coverage", () => { target: "refs/heads/develop", eventName: "merge_group", }); - expect(plan.selection).toBe("full"); - expect(plan.templateMatrix.template).toEqual(TEMPLATE_NAMES); - expect(plan.goldenMatrix.template).toEqual(["starter", "platformer"]); - expect(plan.unitMatrix.shard).toEqual(["1/4", "2/4", "3/4", "4/4"]); + // The queue is narrowed exactly like a pull request into develop: the changed kits prove + // themselves and starter exercises the default scaffold route. + expect(plan.selection).toBe("template"); + expect(plan.qualification).toBe(false); + expect(plan.templateMatrix.template).toEqual(["shooter", "snow"]); + expect(plan.goldenMatrix.template).toEqual(["shooter"]); + expect(plan.unitMatrix.shard).toEqual(["1/1"]); + expect(validatePlan(plan)).toEqual(plan); } finally { rmSync(f.root, { recursive: true, force: true }); } @@ -144,7 +151,10 @@ describe("impact-driven template coverage", () => { const f = fixture(); try { const head = f.change("packages/create-threenative/templates/shooter/README.md"); - for (const target of ["develop", "main"]) { + for (const [target, selection] of [ + ["develop", "prose"], + ["main", "full"], + ] as const) { const plan = classify({ root: f.root, base: f.base, @@ -153,7 +163,7 @@ describe("impact-driven template coverage", () => { target, eventName: "merge_group", }); - expect(plan.selection).toBe("full"); + expect(plan.selection).toBe(selection); } } finally { rmSync(f.root, { recursive: true, force: true }); @@ -172,9 +182,9 @@ describe("impact-driven template coverage", () => { target: "develop", eventName: "merge_group", }); - expect(plan.selection).toBe("full"); + expect(plan.selection).toBe("template"); expect(plan.checks.ci).toBe(true); - expect(plan.templateMatrix.template).toEqual(TEMPLATE_NAMES); + expect(plan.templateMatrix.template).toEqual(["shooter"]); } finally { rmSync(f.root, { recursive: true, force: true }); } @@ -206,7 +216,9 @@ describe("impact-driven template coverage", () => { const f = fixture(); const clone = makeTempDirSync("ci-shallow-queue-"); try { - const head = f.change("docs/x.md"); + // A shared-runtime change keeps the queue plan narrow enough for a review lane while still + // requiring the exact-candidate Integration join, which is what this gate re-derives. + const head = f.change("packages/core/src/scene.ts"); const plan = classify({ root: f.root, base: f.base, @@ -215,6 +227,7 @@ describe("impact-driven template coverage", () => { target: "develop", eventName: "merge_group", }); + expect(plan.jobs.integration.required).toBe(true); const cloned = spawnSync( "git", ["clone", "--quiet", "--depth", "1", `file://${f.root}`, clone], @@ -232,6 +245,7 @@ describe("impact-driven template coverage", () => { env: { ...process.env, TN_CI_EVENT: "merge_group", + TN_CI_BASE_REF: "refs/heads/develop", TN_CI_BASE_SHA: f.base, TN_CI_HEAD_SHA: head, TN_CI_NEEDS: JSON.stringify(needs), @@ -279,6 +293,8 @@ describe("impact-driven template coverage", () => { { base: "unknown", head }, { base: f.base, head: f.base }, ] as const) { + // An unresolved base/head identity cannot prove what the queue would merge, so it stays + // exhaustive even though every other develop queue input is narrowed. expect( classify({ root: f.root, @@ -286,8 +302,8 @@ describe("impact-driven template coverage", () => { candidateSha: head, target: "develop", eventName: "merge_group", - }).selection, - ).toBe("full"); + }), + ).toMatchObject({ selection: "full", qualification: true, nativeTier: "full" }); } } finally { rmSync(f.root, { recursive: true, force: true }); diff --git a/scripts/ci-attempt-receipts.mjs b/scripts/ci-attempt-receipts.mjs index a8997d83fb..545e09d978 100644 --- a/scripts/ci-attempt-receipts.mjs +++ b/scripts/ci-attempt-receipts.mjs @@ -2,6 +2,8 @@ import { execFileSync } from "node:child_process"; import { readFileSync, readdirSync } from "node:fs"; import path from "node:path"; +const MAX_ATTEMPTS = 5; + export function readAttemptJobs({ repository, runId, runAttempt }, prefix = "CI_INTEGRATION") { if ( !/^[\w.-]+\/[\w.-]+$/u.test(repository ?? "") || @@ -10,7 +12,7 @@ export function readAttemptJobs({ repository, runId, runAttempt }, prefix = "CI_ ) throw new Error(`${prefix}_RUN_IDENTITY: repository, run and attempt are required`); let response; - for (let attempt = 0; attempt < 3; attempt++) { + for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) { try { response = execFileSync( "gh", @@ -26,15 +28,15 @@ export function readAttemptJobs({ repository, runId, runAttempt }, prefix = "CI_ } catch (error) { const stderr = String(error.stderr ?? ""); if ( - attempt === 2 || + attempt === MAX_ATTEMPTS - 1 || error.status !== 1 || error.signal || !/error connecting to api\.github\.com/iu.test(stderr) || /HTTP (?:401|403|404|429)\b/iu.test(stderr) ) throw error; - // Repeat only the same read endpoint; no stale page, receipt or permission fallback. - Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 200 * (attempt + 1)); + // Retry only the same read endpoint after 1, 2, 4, 8 seconds; no stale page, receipt or permission fallback. + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 1000 * 2 ** attempt); } } const pages = JSON.parse(response); diff --git a/scripts/ci-change-scope.mjs b/scripts/ci-change-scope.mjs index a438054399..7d29692c58 100644 --- a/scripts/ci-change-scope.mjs +++ b/scripts/ci-change-scope.mjs @@ -353,8 +353,13 @@ export function validateEventPlan(value, { eventName, baseRef, forceFull = false const event = eventName; const target = (baseRef ?? "").replace(/^refs\/heads\//u, ""); const review = event === "pull_request" && target === "develop"; - const warm = event === "push" && target === "develop" && plan.selection === "warm"; - if ((!review && !warm) || forceFull) { + // PRD-550: the queue reviews the exact tree a merge would produce on the base branch, so it earns + // the same narrowed plan a pull request into develop earns. A queue entry against any other target + // still qualifies its tree here. + const queue = + event === "merge_group" && + validationProfile({ eventName: event, baseRef: baseRef ?? "" }).target === "develop"; + if ((!review && !queue) || forceFull) { if ( !plan.qualification || plan.selection !== "full" || @@ -423,6 +428,22 @@ function treeOf(root, sha) { return result.status === 0 && /^[0-9a-f]{40}$/u.test(tree) ? tree : null; } +/** A pull-request candidate must be the exact base/head merge, so a changed input needs fresh verification. */ +export function assertMergeParentCandidate(root, candidateSha, base, head) { + const commit = git(root, ["cat-file", "-p", candidateSha]); + const headers = commit.stdout?.split("\n\n", 1)[0] ?? ""; + const parents = [...headers.matchAll(/^parent ([0-9a-f]{40})$/gmu)].map((match) => match[1]); + if ( + commit.status !== 0 || + !/^[0-9a-f]{40}$/u.test(base ?? "") || + !/^[0-9a-f]{40}$/u.test(head ?? "") || + JSON.stringify(parents) !== JSON.stringify([base, head]) + ) + throw new Error( + "CI_REQUIRED_PR_CANDIDATE_MISMATCH: expected the exact proposed base/head merge; changed inputs require fresh verification", + ); +} + function ghApi(pathname) { const repository = process.env.GITHUB_REPOSITORY; if (repository === undefined || !/^[^/\s]+\/[^/\s]+$/u.test(repository)) { @@ -647,17 +668,16 @@ export function coverageMiss(current, source) { * The validation profile a run has to prove. `main` is a promotion or a main push, `develop` an * ordinary pull request or the merge queue on develop, `other` any run whose target the API does not * report — which ranks lowest, because an unproven target never stands in for a proven one. The - * native tier follows the target: an ordinary pull request owes the reduced matrix PRD-380 gives it, - * everything else the full one, and a run owing no lane owes none. + * native tier follows the target: a pull request or the merge queue on develop owes the reduced + * matrix PRD-380 gives it, everything else the full one, and a run owing no lane owes none. */ export function validationProfile({ eventName, baseRef = "", nativeRequired = false }) { - const target = - baseRef === "main" - ? "main" - : baseRef === "develop" || eventName === "merge_group" - ? "develop" - : "other"; - const review = target === "develop" && (eventName === "pull_request" || eventName === undefined); + // Only an explicit target can earn reduced validation; GitHub also reports refs/heads/. + const ref = baseRef.replace(/^refs\/heads\//u, ""); + const target = ref === "main" ? "main" : ref === "develop" ? "develop" : "other"; + const review = + target === "develop" && + (eventName === "pull_request" || eventName === "merge_group" || eventName === undefined); return { target, native: nativeRequired ? (review ? "reduced" : "full") : "none" }; } @@ -852,26 +872,16 @@ export function classify(input) { candidateSha, ); if (options.full) return full("explicit full verification requested"); - // PRD-481 phase 2. A pull request's caches are scoped to `refs/pull/N/merge`, which no other pull - // request reads, so only a run on the base branch publishes something every pull request into - // develop reads — and before this lane nothing ran on develop at all, so every pull request paid - // a full native rebuild. The lane exists only to publish those caches; it verifies nothing. - if (options.eventName === "push" && options.target === "develop") { - return selectionPlan( - "warm", - "a develop push publishes the base-branch caches every pull request into develop reads, and verifies nothing", - [], - candidateSha, - false, - 0, - target, - ); - } - // Every event except a pull request stays full, and so does every target but develop. A merge - // group is the queue testing the exact tree a merge would produce, so it runs the whole board - // before anything lands — the narrowings below are for reviewing a change, never for qualifying a - // tree, which is why the queue, a push, the nightly, an explicit audit and a promotion into main - // are all excluded here rather than narrowed further downstream. + // PRD-550. The exhaustive board left the queue, so a develop push is the promotion of an exact + // tree and qualifies that tree with every supported system, every kit and every native row. A + // full selection still runs `build-artifacts` and `test-native`, so the base-branch caches every + // pull request into develop reads keep being published here. + if (options.eventName === "push" && options.target === "develop") + return full("a develop push qualifies the exact tree with the full board"); + // Every event except a pull request and a merge group stays full, and so does every target but + // develop. The narrowings below review a change on develop; the exhaustive board belongs to the + // tree-qualifying events — a push, the nightly, an explicit audit and a promotion into main — which + // are excluded here rather than narrowed further downstream. if (!["pull_request", "merge_group"].includes(options.eventName)) return full(`event ${JSON.stringify(options.eventName)} requires complete verification`); if (options.target !== "develop") @@ -898,8 +908,6 @@ export function classify(input) { } const parsed = changedPaths(options); if ("error" in parsed) return full(parsed.error); - if (options.eventName === "merge_group") - return full("merge groups qualify the complete exact candidate", parsed.paths); // `native-platforms.yml` is the one workflow the CI-configuration rule would otherwise narrow, and // it is the matrix's own definition: exempting it would waive the very lane that has to prove the // change. Any native path therefore keeps `full`, not merely `native: true`. Computed over the @@ -1027,10 +1035,19 @@ function output(result, format) { if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { try { const options = parseArgs(process.argv.slice(2)); - output( - options.plan !== undefined ? validatePlan(JSON.parse(options.plan)) : classify(options), - options.format, - ); + if (options.plan !== undefined) { + output(validatePlan(JSON.parse(options.plan)), options.format); + } else { + // Only the live GitHub projection owes the parent assertion; local and offline keep classify(). + if (options.format === "github" && options.eventName === "pull_request" && !options.local) + assertMergeParentCandidate( + options.root, + options.candidateSha ?? git(options.root, ["rev-parse", "HEAD"]).stdout.trim(), + options.base, + options.head, + ); + output(classify(options), options.format); + } } catch (error) { console.error(error instanceof Error ? error.message : String(error)); process.exitCode = 2; diff --git a/scripts/ci-merge-latency.mjs b/scripts/ci-merge-latency.mjs new file mode 100644 index 0000000000..53b2b935ca --- /dev/null +++ b/scripts/ci-merge-latency.mjs @@ -0,0 +1,251 @@ +#!/usr/bin/env node +// PRD-550: how long a change takes to merge, measured from the GitHub API with `gh` and no new dependency. +// +// node scripts/ci-merge-latency.mjs --since 2026-10-09 [--json] [--base develop] +// +// Per merged PR: last push -> merge, enqueue -> merge, merge-queue attempts, first failing job. +// Summary: median/p90 of those, merge-group success, board wall time and runner queue wait. +// A value the API cannot supply is null, never 0. +import { execFileSync } from "node:child_process"; +import { pathToFileURL } from "node:url"; + +const MS_PER_MIN = 60_000; +const minutes = (from, to) => + from && to ? (Date.parse(to) - Date.parse(from)) / MS_PER_MIN : null; +const round = (value) => (value === null ? null : Math.round(value * 10) / 10); + +/** Linear-interpolated quantile of the finite numbers in `values`; null when there are none. */ +export function quantile(values, q) { + const sorted = values.filter((value) => Number.isFinite(value)).sort((a, b) => a - b); + if (sorted.length === 0) return null; + const position = (sorted.length - 1) * q; + const low = Math.floor(position); + const high = Math.ceil(position); + return sorted[low] + (sorted[high] - sorted[low]) * (position - low); +} + +/** The PR number a merge-queue run belongs to: `gh-readonly-queue/develop/pr-461-`. */ +export function queuePrNumber(headBranch) { + const match = /^gh-readonly-queue\/[^/]+\/pr-(\d+)-/u.exec(headBranch ?? ""); + return match === null ? null : Number(match[1]); +} + +/** The job that failed first in a run. Joins such as `ci-required` finish last, so they never win. */ +export function firstFailingJob(jobs = []) { + const failed = jobs + .filter((job) => job.conclusion === "failure" && job.completedAt) + .sort((a, b) => Date.parse(a.completedAt) - Date.parse(b.completedAt)); + return failed[0]?.name ?? null; +} + +/** + * Pure summary of one window. + * prs: [{ number, title, mergedAt, lastCommitAt }] + * runs: merge_group runs [{ id, headBranch, createdAt, updatedAt, conclusion }] + * jobsByRun: { [runId]: [{ name, conclusion, createdAt, startedAt, completedAt }] } + */ +export function summarize({ prs, runs: allRuns, jobsByRun = {} }) { + // A merge_group run whose branch is not a queue branch belongs to no PR and measures nothing here. + const runs = allRuns.filter((run) => queuePrNumber(run.headBranch) !== null); + const byPr = new Map(); + for (const run of runs) { + const number = queuePrNumber(run.headBranch); + if (number === null) continue; + byPr.set(number, [...(byPr.get(number) ?? []), run]); + } + const rows = prs.map((pr) => { + const attempts = (byPr.get(pr.number) ?? []).sort( + (a, b) => Date.parse(a.createdAt) - Date.parse(b.createdAt), + ); + const first = attempts[0]; + const failedRun = attempts.find((run) => run.conclusion === "failure"); + return { + number: pr.number, + title: pr.title, + lastPushToMergeMin: round(minutes(pr.lastCommitAt, pr.mergedAt)), + enqueueToMergeMin: round(minutes(first?.createdAt, pr.mergedAt)), + attempts: attempts.length, + firstAttempt: first === undefined ? null : (first.conclusion ?? "running"), + firstFailingJob: failedRun === undefined ? null : firstFailingJob(jobsByRun[failedRun.id]), + }; + }); + const queued = rows.filter((row) => row.firstAttempt !== null); + const done = runs.filter((run) => run.conclusion === "success" || run.conclusion === "failure"); + const failures = new Map(); + for (const run of runs.filter((r) => r.conclusion === "failure")) { + const name = firstFailingJob(jobsByRun[run.id]) ?? "unknown"; + failures.set(name, (failures.get(name) ?? 0) + 1); + } + const waits = Object.values(jobsByRun) + .flat() + .filter((job) => job.startedAt && job.conclusion !== "skipped") + .map((job) => minutes(job.createdAt, job.startedAt)) + .filter((wait) => wait !== null && wait >= 0); + const wall = runs + .filter((run) => run.conclusion === "success") + .map((run) => minutes(run.createdAt, run.updatedAt)); + const pct = (part, whole) => (whole === 0 ? null : round((100 * part) / whole)); + return { + prs: rows, + summary: { + prCount: rows.length, + lastPushToMergeMedianMin: round( + quantile( + rows.map((r) => r.lastPushToMergeMin), + 0.5, + ), + ), + lastPushToMergeP90Min: round( + quantile( + rows.map((r) => r.lastPushToMergeMin), + 0.9, + ), + ), + enqueueToMergeMedianMin: round( + quantile( + rows.map((r) => r.enqueueToMergeMin), + 0.5, + ), + ), + firstAttemptSuccessPct: pct( + queued.filter((r) => r.firstAttempt === "success").length, + queued.length, + ), + mergeGroupRuns: runs.length, + mergeGroupRunSuccessPct: pct( + done.filter((r) => r.conclusion === "success").length, + done.length, + ), + boardWallMedianMin: round(quantile(wall, 0.5)), + queueWaitP50Min: round(quantile(waits, 0.5)), + queueWaitP90Min: round(quantile(waits, 0.9)), + queueWaitJobs: waits.length, + }, + firstFailingJobs: [...failures] + .map(([job, count]) => ({ job, count })) + .sort((a, b) => b.count - a.count || a.job.localeCompare(b.job)), + }; +} + +function gh(args) { + return execFileSync("gh", args, { encoding: "utf8", maxBuffer: 256 * 1024 * 1024 }); +} + +const ndjson = (text) => + text + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line)); + +/** Fetch one window from GitHub and summarize it. */ +export function measure({ since, base = "develop", repo = process.env.GITHUB_REPOSITORY }) { + const repository = + repo ?? gh(["repo", "view", "--json", "nameWithOwner", "-q", ".nameWithOwner"]).trim(); + const prs = JSON.parse( + gh([ + "pr", + "list", + "--repo", + repository, + "--state", + "merged", + "--base", + base, + "--search", + `merged:>=${since}`, + "--limit", + "300", + "--json", + "number,title,mergedAt,headRefOid", + ]), + ).map((pr) => ({ + number: pr.number, + title: pr.title, + mergedAt: pr.mergedAt, + // `commits` in the GraphQL list exceeds GitHub's node limit, so the head commit is read on its own. + lastCommitAt: gh([ + "api", + `repos/${repository}/commits/${pr.headRefOid}`, + "-q", + ".commit.committer.date", + ]).trim(), + })); + const runs = ndjson( + gh([ + "api", + "--paginate", + "--jq", + ".workflow_runs[]", + `repos/${repository}/actions/workflows/ci.yml/runs?event=merge_group&created=%3E%3D${since}&per_page=100`, + ]), + ).map((run) => ({ + id: run.id, + headBranch: run.head_branch, + createdAt: run.created_at, + updatedAt: run.updated_at, + conclusion: run.conclusion, + })); + const jobsByRun = {}; + for (const run of runs) { + jobsByRun[run.id] = ndjson( + gh([ + "api", + "--paginate", + "--jq", + ".jobs[]", + `repos/${repository}/actions/runs/${run.id}/jobs?per_page=100`, + ]), + ).map((job) => ({ + name: job.name, + conclusion: job.conclusion, + createdAt: job.created_at, + startedAt: job.started_at, + completedAt: job.completed_at, + })); + } + return { since, base, at: new Date().toISOString(), ...summarize({ prs, runs, jobsByRun }) }; +} + +const fmt = (value, unit = "") => (value === null ? "n/a" : `${value}${unit}`); + +export function formatText(result) { + const { summary: s } = result; + const lines = [ + `merged into ${result.base} since ${result.since}: ${s.prCount} PRs, ${s.mergeGroupRuns} merge-group runs`, + `last push -> merge median ${fmt(s.lastPushToMergeMedianMin, " min")} p90 ${fmt(s.lastPushToMergeP90Min, " min")} (goal 30 / 45)`, + `enqueue -> merge median ${fmt(s.enqueueToMergeMedianMin, " min")}`, + `first-attempt merge ${fmt(s.firstAttemptSuccessPct, "%")} run success ${fmt(s.mergeGroupRunSuccessPct, "%")} (goal 90%)`, + `merge-group board median wall ${fmt(s.boardWallMedianMin, " min")} (goal 15)`, + `runner queue wait p50 ${fmt(s.queueWaitP50Min, " min")} p90 ${fmt(s.queueWaitP90Min, " min")} over ${s.queueWaitJobs} jobs (goal p90 3)`, + "", + "PR push->merge enqueue->merge attempts first first failing job", + ]; + for (const row of result.prs) { + lines.push( + `#${String(row.number).padEnd(5)} ${fmt(row.lastPushToMergeMin).padStart(10)} ${fmt(row.enqueueToMergeMin).padStart(13)} ${String(row.attempts).padStart(8)} ${String(row.firstAttempt ?? "n/a").padEnd(11)} ${row.firstFailingJob ?? ""}`, + ); + } + if (result.firstFailingJobs.length > 0) { + lines.push("", "first failing job per red merge group:"); + for (const { job, count } of result.firstFailingJobs) + lines.push(` ${String(count).padStart(3)} ${job}`); + } + return lines.join("\n"); +} + +function main(argv) { + const get = (flag) => (argv.includes(flag) ? argv[argv.indexOf(flag) + 1] : undefined); + const since = get("--since"); + if (since === undefined || !/^\d{4}-\d{2}-\d{2}(?:T[\d:.]+Z)?$/u.test(since)) { + console.error( + "usage: node scripts/ci-merge-latency.mjs --since YYYY-MM-DD [--base develop] [--json]", + ); + process.exit(2); + } + const result = measure({ since, base: get("--base") ?? "develop" }); + console.log(argv.includes("--json") ? JSON.stringify(result, null, 2) : formatText(result)); +} + +if (process.argv[1] !== undefined && import.meta.url === pathToFileURL(process.argv[1]).href) { + main(process.argv.slice(2)); +} diff --git a/scripts/ci-required.mjs b/scripts/ci-required.mjs index 6b31be88e3..b66fc110b2 100644 --- a/scripts/ci-required.mjs +++ b/scripts/ci-required.mjs @@ -3,7 +3,13 @@ import { spawnSync } from "node:child_process"; import { appendFileSync, readFileSync } from "node:fs"; import { readAttemptJobs } from "./ci-attempt-receipts.mjs"; -import { currentRun, sourceVerdict, validateEventPlan, validatePlan } from "./ci-change-scope.mjs"; +import { + assertMergeParentCandidate, + currentRun, + sourceVerdict, + validateEventPlan, + validatePlan, +} from "./ci-change-scope.mjs"; import { readIntegrationJobs } from "./ci-integration-receipts.mjs"; import { integrationCandidatePreflight, @@ -33,7 +39,6 @@ try { forceFull: process.env.TN_CI_FORCE_FULL === "true", }); if (process.env.TN_CI_EVENT === "pull_request") { - const { TN_CI_BASE_SHA: base, TN_CI_HEAD_SHA: head } = process.env; // The candidate is frozen by the exact base/head parent assertion below, not by the head // branch's name. A `promotion/` ref restated a SHA that check already verifies. if ( @@ -42,19 +47,12 @@ try { plan.selection !== "full" ) throw new Error("CI_REQUIRED_MAIN_FULL: main requires complete verification"); - const commit = spawnSync("git", ["cat-file", "-p", "HEAD"], { encoding: "utf8" }); - const headers = commit.stdout?.split("\n\n", 1)[0] ?? ""; - const parents = [...headers.matchAll(/^parent ([0-9a-f]{40})$/gmu)].map((match) => match[1]); - if ( - commit.status !== 0 || - !/^[0-9a-f]{40}$/u.test(base ?? "") || - !/^[0-9a-f]{40}$/u.test(head ?? "") || - JSON.stringify(parents) !== JSON.stringify([base, head]) - ) { - throw new Error( - "CI_REQUIRED_PR_CANDIDATE_MISMATCH: expected the exact proposed base/head merge; changed inputs require fresh verification", - ); - } + assertMergeParentCandidate( + process.cwd(), + "HEAD", + process.env.TN_CI_BASE_SHA, + process.env.TN_CI_HEAD_SHA, + ); } if (process.env.TN_CI_EVENT === "merge_group") { const base = process.env.TN_CI_BASE_SHA ?? ""; diff --git a/scripts/ci-runners.sh b/scripts/ci-runners.sh index d8dc9e86ab..24e7f7d205 100755 --- a/scripts/ci-runners.sh +++ b/scripts/ci-runners.sh @@ -239,12 +239,30 @@ up() { # advertised while it has an idle runner and withdrawn the moment it has none. balance() { need gh - local repo want_heavy want_light have_heavy="" have_light="" idle seen_heavy=0 seen_light=0 + local repo want_heavy want_light have_heavy have_light idle present token + local seen_heavy=0 seen_light=0 repo="$(repo_name)" while [ ! -e "$STATE_DIR/stop" ]; do idle="$(gh api "repos/$repo/actions/runners" --paginate --jq \ '[.runners[] | select(.status == "online" and .busy == false) | .labels[].name] | join(" ")' \ - 2>/dev/null)" || { sleep 20; continue; } + 2>/dev/null)" || { seen_heavy=0; seen_light=0; sleep 20; continue; } + # Read the variables, not a cache: a cached value misses a change made outside this loop. Report + # the name alone for the exact expected label and NAME:invalid otherwise, so a wrong label is + # still visible without ever printing a value into a token. + present="$(gh api "repos/$repo/actions/variables" --paginate --jq \ + ".variables[] | select(.name == \"$VARIABLE\" or .name == \"$LIGHT_VARIABLE\") | if (.name == \"$VARIABLE\" and .value == \"$LABEL\") or (.name == \"$LIGHT_VARIABLE\" and .value == \"$LIGHT_LABEL\") then .name else .name + \":invalid\" end" \ + 2>/dev/null)" || { seen_heavy=0; seen_light=0; sleep 20; continue; } + # A teardown that began during the reads must never be followed by an advertise. + if [ -e "$STATE_DIR/stop" ]; then break; fi + have_heavy=off; have_light=off + while IFS= read -r token; do + case "$token" in + "$VARIABLE") have_heavy=on ;; + "$VARIABLE:invalid") have_heavy=invalid ;; + "$LIGHT_VARIABLE") have_light=on ;; + "$LIGHT_VARIABLE:invalid") have_light=invalid ;; + esac + done <<< "$present" # Off at once when no runner is idle; on only after two polls in a row (~40 s) found one. A runner # that just re-registered in a saturated pool is idle for seconds before it takes a queued job, and # that blip must not route more work here. @@ -256,12 +274,12 @@ balance() { if [ "$want_heavy" != "$have_heavy" ]; then if [ "$want_heavy" = on ]; then gh variable set "$VARIABLE" --body "$LABEL" --repo "$repo" else gh variable delete "$VARIABLE" --repo "$repo" 2>/dev/null || true; fi - have_heavy="$want_heavy"; printf '%s %s=%s\n' "$(date -u +%H:%M:%S)" "$VARIABLE" "$want_heavy" + printf '%s %s=%s\n' "$(date -u +%H:%M:%S)" "$VARIABLE" "$want_heavy" fi if [ "$want_light" != "$have_light" ]; then if [ "$want_light" = on ]; then gh variable set "$LIGHT_VARIABLE" --body "$LIGHT_LABEL" --repo "$repo" else gh variable delete "$LIGHT_VARIABLE" --repo "$repo" 2>/dev/null || true; fi - have_light="$want_light"; printf '%s %s=%s\n' "$(date -u +%H:%M:%S)" "$LIGHT_VARIABLE" "$want_light" + printf '%s %s=%s\n' "$(date -u +%H:%M:%S)" "$LIGHT_VARIABLE" "$want_light" fi sleep 20 done diff --git a/scripts/ci-speed-loop.mjs b/scripts/ci-speed-loop.mjs new file mode 100644 index 0000000000..b000b16466 --- /dev/null +++ b/scripts/ci-speed-loop.mjs @@ -0,0 +1,365 @@ +#!/usr/bin/env node +// PRD-550: the CI speed loop's ledger and dashboard. +// +// node scripts/ci-speed-loop.mjs record --since 2026-10-09 --label "gating board" [--notes ...] [--decision keep] +// node scripts/ci-speed-loop.mjs show +// node scripts/ci-speed-loop.mjs discard --iteration 3 --reason "queue outage" +// node scripts/ci-speed-loop.mjs report +// +// docs/ci-speed/ledger.json is the durable record, one entry per iteration. docs/ci-speed/index.html is +// generated from it and opens with file:// and no script. Neither is edited by hand. A superseded +// iteration stays in the ledger and the table but leaves the charts and the progress strip. +import { execFileSync } from "node:child_process"; +import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { measure } from "./ci-merge-latency.mjs"; + +const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const LEDGER = resolve(ROOT, "docs/ci-speed/ledger.json"); +const HTML = resolve(ROOT, "docs/ci-speed/index.html"); +const DECISIONS = ["baseline", "keep", "reject", "provisional", "inconclusive", "invalid"]; + +/** The PRD-550 targets. `better` says which side of the goal is a win. */ +export const GOALS = [ + { + key: "lastPushToMergeMedianMin", + short: "Push→merge p50", + label: "Last push to merge, median", + unit: "min", + goal: 30, + better: "lower", + }, + { + key: "lastPushToMergeP90Min", + short: "Push→merge p90", + label: "Last push to merge, p90", + unit: "min", + goal: 45, + better: "lower", + }, + { + key: "firstAttemptSuccessPct", + short: "First attempt", + label: "Merge queue passes on the first attempt", + unit: "%", + goal: 90, + better: "higher", + }, + { + key: "boardWallMedianMin", + short: "Board wall", + label: "Merge-group board wall time, median", + unit: "min", + goal: 15, + better: "lower", + }, + { + key: "queueWaitP90Min", + short: "Queue wait p90", + label: "Runner queue wait, p90", + unit: "min", + goal: 3, + better: "lower", + }, +]; + +export const emptyLedger = () => ({ schemaVersion: 1, goals: GOALS, iterations: [] }); +/** Live = counts toward charts and the strip. Invalid and superseded runs are history only. */ +export const isLive = (iteration) => + iteration.superseded === undefined && iteration.decision !== "invalid"; + +export function readLedger(path = LEDGER) { + if (!existsSync(path)) return emptyLedger(); + const ledger = JSON.parse(readFileSync(path, "utf8")); + if (ledger.schemaVersion !== 1 || !Array.isArray(ledger.iterations)) { + throw new Error(`${path} is not a schemaVersion 1 CI speed ledger`); + } + return ledger; +} + +function writeAtomic(path, text) { + mkdirSync(dirname(path), { recursive: true }); + const temporary = `${path}.${process.pid}.tmp`; + writeFileSync(temporary, text); + renameSync(temporary, path); +} + +/** Add one measured window to the ledger. Returns a new ledger; the input is not changed. */ +export function appendIteration( + ledger, + { measurement, label, notes = "", decision, commit = "", source = "measured" }, +) { + const id = ledger.iterations.reduce((top, item) => Math.max(top, item.id), 0) + 1; + const chosen = decision ?? (ledger.iterations.length === 0 ? "baseline" : "provisional"); + if (!DECISIONS.includes(chosen)) + throw new Error(`decision must be one of ${DECISIONS.join(", ")}`); + const metrics = Object.fromEntries( + GOALS.map(({ key }) => [key, measurement.summary[key] ?? null]), + ); + const iteration = { + id, + at: measurement.at ?? new Date().toISOString(), + commit, + label, + notes, + decision: chosen, + source, + window: { since: measurement.since, base: measurement.base }, + metrics, + extra: { + prCount: measurement.summary.prCount, + mergeGroupRuns: measurement.summary.mergeGroupRuns, + mergeGroupRunSuccessPct: measurement.summary.mergeGroupRunSuccessPct, + enqueueToMergeMedianMin: measurement.summary.enqueueToMergeMedianMin, + queueWaitJobs: measurement.summary.queueWaitJobs, + }, + prs: measurement.prs, + firstFailingJobs: measurement.firstFailingJobs, + }; + return { ...ledger, goals: GOALS, iterations: [...ledger.iterations, iteration] }; +} + +export function discardIteration(ledger, id, reason) { + if (!reason) throw new Error("discard needs --reason"); + if (!ledger.iterations.some((item) => item.id === id)) + throw new Error(`no iteration ${String(id)}`); + return { + ...ledger, + iterations: ledger.iterations.map((item) => + item.id === id ? { ...item, superseded: { reason, at: new Date().toISOString() } } : item, + ), + }; +} + +/** 0..1 of the way from the baseline value to the goal. Null when either end is unknown. */ +export function progressToward(goal, baseline, current) { + if (baseline === null || baseline === undefined || current === null || current === undefined) + return null; + const span = goal.better === "lower" ? baseline - goal.goal : goal.goal - baseline; + if (span <= 0) return 1; + const done = goal.better === "lower" ? baseline - current : current - baseline; + return Math.min(1, Math.max(0, done / span)); +} + +export const meetsGoal = (goal, value) => + value !== null && + value !== undefined && + (goal.better === "lower" ? value <= goal.goal : value >= goal.goal); + +const esc = (value) => + String(value ?? "").replace( + /[&<>"']/gu, + (c) => ({ "&": "&", "<": "<", ">": ">", '"': """, "'": "'" })[c], + ); +const num = (value, unit = "") => + value === null || value === undefined + ? "n/a" + : `${value}${unit === "%" ? "%" : ` ${unit}`}`.trim(); + +/** One goal's trend: a line per live iteration, a dashed goal line, a break where a value is missing. */ +function lineChart(goal, live) { + const width = 380; + const height = 190; + const pad = { l: 44, r: 14, t: 14, b: 28 }; + const values = live.map((item) => item.metrics[goal.key]); + const known = values.filter((v) => v !== null); + const lo = Math.min(goal.goal, ...known, goal.better === "higher" ? 0 : goal.goal); + const hi = Math.max(goal.goal, ...known) * 1.1 || 1; + const x = (index) => + live.length === 1 + ? (pad.l + width - pad.r) / 2 + : pad.l + ((width - pad.l - pad.r) * index) / (live.length - 1); + const y = (value) => height - pad.b - ((height - pad.t - pad.b) * (value - lo)) / (hi - lo); + const segments = []; + let run = []; + values.forEach((value, index) => { + if (value === null) { + if (run.length > 0) segments.push(run); + run = []; + } else run.push([x(index), y(value)]); + }); + if (run.length > 0) segments.push(run); + const lines = segments + .filter((s) => s.length > 1) + .map( + (s) => + ``, + ) + .join(""); + const dots = values + .map((value, index) => + value === null + ? "" + : `#${live[index].id}: ${esc(num(value, goal.unit))}#${live[index].id}`, + ) + .join(""); + const gy = y(goal.goal).toFixed(1); + const empty = + known.length === 0 + ? `baseline pending` + : ""; + return `
${esc(goal.label)} (${goal.better === "lower" ? "lower" : "higher"} is better)
+ + + +${goal.goal} +goal ${goal.better === "lower" ? "≤" : "≥"} ${goal.goal} ${esc(goal.unit)} +${Math.abs(y(hi / 1.1) - y(goal.goal)) > 14 ? `${Math.round(hi / 1.1)}` : ""} +${lines}${dots}${empty}
`; +} + +function barChart(rows, title) { + const max = Math.max(1, ...rows.map((row) => row.count)); + const body = rows + .slice(0, 8) + .map( + (row) => + `
${esc(row.job)}${row.count}
`, + ) + .join(""); + return `
${esc(title)}
${body || '

no red merge groups in this window

'}
`; +} + +export function renderHtml(ledger, now = new Date()) { + const live = ledger.iterations.filter(isLive); + const latest = live.at(-1); + const base = live[0]; + const strip = GOALS.map((goal) => { + const current = latest?.metrics[goal.key] ?? null; + const progress = progressToward(goal, base?.metrics[goal.key], current); + const percent = progress === null ? null : Math.round(progress * 100); + return `${esc(goal.label)}${esc(num(base?.metrics[goal.key], goal.unit))}${esc(num(current, goal.unit))}${goal.better === "lower" ? "≤" : "≥"} ${goal.goal} ${esc(goal.unit === "%" ? "%" : goal.unit)} + ${percent === null ? "n/a" : `${percent}%`}${meetsGoal(goal, current) ? 'met' : 'open'}`; + }).join(""); + const history = ledger.iterations + .map((item) => { + const cells = GOALS.map( + (goal) => + `${esc(num(item.metrics[goal.key], goal.unit === "%" ? "%" : ""))}`, + ).join(""); + const out = item.superseded === undefined ? "" : ` class="out"`; + const flag = + item.superseded === undefined + ? "" + : `
superseded: ${esc(item.superseded.reason)}`; + return `#${item.id}${esc(item.at.slice(0, 10))}${esc(item.commit)}${esc(item.label)}${flag}
${esc(item.notes)}${esc(item.decision)}${esc(item.window.since)}${cells}`; + }) + .join(""); + // A window that merged no PR keeps the last window that did, so the audit table never goes blank. + const prSource = [...live].reverse().find((item) => (item.prs ?? []).length > 0); + const prTitle = + prSource === undefined ? "Merged PRs" : `Merged PRs, iteration #${String(prSource.id)} window`; + const prs = (prSource?.prs ?? []) + .map( + (row) => + `#${row.number}${esc(row.title)}${esc(num(row.lastPushToMergeMin, "min"))}${esc(num(row.enqueueToMergeMin, "min"))}${row.attempts}${esc(row.firstAttempt ?? "n/a")}${esc(row.firstFailingJob ?? "")}`, + ) + .join(""); + const charts = GOALS.map((goal) => lineChart(goal, live)).join(""); + const header = + latest === undefined + ? "Baseline pending: no iteration recorded yet." + : `Latest live iteration #${latest.id} of ${ledger.iterations.length}: ${esc(latest.label)}.`; + return ` + +CI speed loop +
+

CI speed loop

+

${header} Snapshot ${esc(now.toISOString())}. Regenerate with node scripts/ci-speed-loop.mjs report, then reload; the page is a static render of docs/ci-speed/ledger.json. Goal: a change merges within 30 minutes of its last push (PRD-550).

+

Progress toward the goal

+
${strip}
MetricBaselineNowGoalProgress
+

Trend per iteration

+
${charts}${barChart(latest?.firstFailingJobs ?? [], "First failing job per red merge group, latest iteration")}
+

Iterations

+
${GOALS.map((g) => ``).join("")}${history || ''}
#DateCommitChangeDecisionWindow from${esc(g.short)}
baseline pending
+

${prTitle}

+
${prs || ''}
PRTitlePush to mergeEnqueue to mergeAttemptsFirst attemptFirst failing job
no merged PR in this ledger
+
+`; +} + +function show(ledger) { + for (const item of ledger.iterations) { + const cells = GOALS.map((g) => `${g.key}=${item.metrics[g.key] ?? "n/a"}`).join(" "); + console.log( + `#${item.id} ${item.at.slice(0, 10)} ${item.decision}${item.superseded ? " (superseded)" : ""} ${item.label}\n ${cells}`, + ); + } + if (ledger.iterations.length === 0) console.log("baseline pending: no iterations recorded"); +} + +function main(argv) { + const [command] = argv; + const get = (flag) => (argv.includes(flag) ? argv[argv.indexOf(flag) + 1] : undefined); + let ledger = readLedger(); + if (command === "record") { + const since = get("--since"); + const label = get("--label"); + if (since === undefined || label === undefined) + throw new Error("record needs --since YYYY-MM-DD and --label"); + const from = get("--from"); + const measurement = + from === undefined + ? measure({ since, base: get("--base") ?? "develop" }) + : JSON.parse(readFileSync(from, "utf8")); + const commit = + get("--commit") ?? + execFileSync("git", ["rev-parse", "--short", "HEAD"], { cwd: ROOT, encoding: "utf8" }).trim(); + ledger = appendIteration(ledger, { + measurement, + label, + notes: get("--notes") ?? "", + decision: get("--decision"), + commit, + }); + writeAtomic(LEDGER, `${JSON.stringify(ledger, null, 2)}\n`); + } else if (command === "discard") { + ledger = discardIteration(ledger, Number(get("--iteration")), get("--reason")); + writeAtomic(LEDGER, `${JSON.stringify(ledger, null, 2)}\n`); + } else if (command === "show") { + return show(ledger); + } else if (command !== "report") { + console.error("usage: ci-speed-loop.mjs record|show|discard|report"); + process.exit(2); + } + writeAtomic(HTML, renderHtml(ledger)); + console.log(`wrote ${HTML} (${String(ledger.iterations.length)} iterations)`); +} + +if (process.argv[1] !== undefined && import.meta.url === pathToFileURL(process.argv[1]).href) { + try { + main(process.argv.slice(2)); + } catch (error) { + console.error(error instanceof Error ? error.message : error); + process.exit(1); + } +} diff --git a/scripts/exposure-ab.ts b/scripts/exposure-ab.ts index 8ebc5cd99e..e7cfffa890 100644 --- a/scripts/exposure-ab.ts +++ b/scripts/exposure-ab.ts @@ -23,12 +23,6 @@ import { type IncomingMessage, type ServerResponse, createServer } from "node:ht import path from "node:path"; import { fileURLToPath } from "node:url"; import { promisify } from "node:util"; -import { chromium } from "@playwright/test"; -import { - WEBGPU_BROWSER_ARGS, - softwareAdapterName, -} from "../packages/playtest/src/runner/browser.js"; -import { compareCaptures } from "../packages/runtime-native/conformance/metrics.mjs"; const execFileAsync = promisify(execFile); const REPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); @@ -101,6 +95,11 @@ await startVisualScene(canvas, ${JSON.stringify(DIMENSIONS)}, "exposure-ab", ({ } async function main(): Promise { + const { chromium } = await import("@playwright/test"); + const { WEBGPU_BROWSER_ARGS, softwareAdapterName } = await import( + "../packages/playtest/src/runner/browser.js" + ); + const { compareCaptures } = await import("../packages/runtime-native/conformance/metrics.mjs"); const out = process.argv[2] ?? DEFAULT_OUT; const directory = await mkdtemp(path.join(REPO_ROOT, "artifacts/exposure-ab-")); const captures: { exposure: number; image: Buffer; adapter: Record }[] = []; diff --git a/scripts/verify-velocity-history.ts b/scripts/verify-velocity-history.ts index 41849ee5fa..0571e3aca1 100644 --- a/scripts/verify-velocity-history.ts +++ b/scripts/verify-velocity-history.ts @@ -21,6 +21,22 @@ await writeFile( path.join(output, "attempt.json"), `${JSON.stringify({ sourceSha, status: "started" }, null, 2)}\n`, ); +// PRD-550 phase 1. The temporal-off cost arm prices actual frame submission against split-control +// noise, so a runner other jobs share reports their load as render cost and `p95NoiseExcessMs` goes +// red on unrelated work. That arm runs on the performance-regression lane; the history arm, which is +// the correctness gate, stays on the merge path. +const requested = + process.argv + .slice(2) + .find((value) => value.startsWith("--variant=")) + ?.slice("--variant=".length) ?? "both"; +if (!["history", "cost", "both"].includes(requested)) { + throw new Error( + `usage: verify-velocity-history.ts [--variant=], got ${requested}`, + ); +} +const runHistory = requested !== "cost"; +const runCost = requested !== "history"; try { execFileSync("pnpm", ["exec", "vite", "build", "--config", "velocity.vite.config.ts"], { cwd: path.join(root, "examples/abyss-framework"), @@ -89,7 +105,8 @@ try { failures: ["movingPixels", "oracleMaxErrorPixels", "footprintMaxErrorPixels"], }, ]; - for (const { variant, query, kind, failures } of variants) { + const historyVariants = runHistory ? variants : []; + for (const { variant, query, kind, failures } of historyVariants) { const artifactDirectory = path.join(output, variant); const report = await runStandalonePlaytest({ allowSoftwareAdapter: true, @@ -156,14 +173,17 @@ try { ); } const temporalOffCost = []; - for (const costVariant of [ - { name: "temporal-off-cost", query: "", failures: [] }, - { - name: "temporal-off-after-consumer-diagnostic", - query: "?from-temporal", - failures: [], - }, - ]) { + const costVariants = runCost + ? [ + { name: "temporal-off-cost", query: "", failures: [] }, + { + name: "temporal-off-after-consumer-diagnostic", + query: "?from-temporal", + failures: [], + }, + ] + : []; + for (const costVariant of costVariants) { const costDirectory = path.join(output, costVariant.name); const costReport = await runStandalonePlaytest({ allowSoftwareAdapter: true, @@ -221,18 +241,28 @@ try { diagnostics: costReport.diagnostics, }); } - assert.ok( - !(await readFile(path.join(output, "without-history/after.png"))).equals( - await readFile(path.join(output, "tracked/after.png")), - ), - "Actual velocity visualization must change when history is removed.", - ); + if (runHistory) { + assert.ok( + !(await readFile(path.join(output, "without-history/after.png"))).equals( + await readFile(path.join(output, "tracked/after.png")), + ), + "Actual velocity visualization must change when history is removed.", + ); + } + const qualification = [ + runHistory && "actual WebGPU velocity MRT readback and screenshots", + runCost && + "software-only temporal-off CPU submission cost, including three temporal-consumer on/off transitions", + ] + .filter(Boolean) + .join(" plus ") + .concat("; no native, ghosting or hardware-performance claim"); await writeFile( path.join(output, "summary.json"), - `${JSON.stringify({ sourceSha, pass: true, qualification: "actual WebGPU velocity MRT readback and screenshots plus software-only temporal-off CPU submission cost, including three temporal-consumer on/off transitions; no native, ghosting or hardware-performance claim", temporalOffCost, variants: results.map(({ variant, report }) => ({ variant, pass: report.pass, capture: report.capture, motion: report.observations?.resources.motion, diagnostics: report.diagnostics })) }, null, 2)}\n`, + `${JSON.stringify({ sourceSha, variant: requested, pass: true, qualification, temporalOffCost, variants: results.map(({ variant, report }) => ({ variant, pass: report.pass, capture: report.capture, motion: report.observations?.resources.motion, diagnostics: report.diagnostics })) }, null, 2)}\n`, ); console.log( - `Velocity history: expected motion/control outcomes observed, including exact skinned coverage and the original world-history control. Artifacts: ${output}`, + `Velocity ${requested}: expected outcomes observed for ${String(runHistory ? "the authored motion/control variants, including exact skinned coverage and the original world-history control" : "the temporal-off submission cost arm")}. Artifacts: ${output}`, ); } catch (error) { await writeFile(