diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4b6eb0680e9..94f2420c233 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2714,9 +2714,10 @@ jobs: # # Scope: `turbo run build` steps only, i.e. `build` tasks. Since #22077 # that is this step plus the build steps of the Test Core shards, the - # dogfood shards, Temporal Conformance and lint.yml's three Type Check - # build lanes, each carrying the four lines below verbatim; the PINS - # paragraph at the end says what keeps that set honest. `test` / + # dogfood shards, Temporal Conformance, Build UI Deps and lint.yml's + # three Type Check build lanes, each carrying the four lines below + # verbatim; the PINS paragraph at the end says what keeps that set + # honest. `test` / # `test:repo` stay off the remote: their cross-package inputs are # hand-declared, and a wrong hash replayed from a remote reaches every # run and cloud too. Build Docs forces execution and carries no @@ -2756,8 +2757,8 @@ jobs: # `remote:rw` branch above) when the cache is off; a read-only run only # warns. Neither flag enters a task hash (83 of 83 identical with and # without, `--dry=json`, turbo 2.11.5). The carriers that assert are this - # step, Test Core's closure and lint.yml's three Type Check lanes; the - # verdict table is the script header's. + # step, Test Core's closure, Build UI Deps and lint.yml's three Type + # Check lanes; the verdict table is the script header's. # # PINS, for every step that carries these four lines (#22077). Each # carrier points back here. @@ -3069,9 +3070,10 @@ jobs: # What the port changed, everywhere, and why: # - objectui's `.objectstack-pin` plumbing (restore / build / save the # pinned @objectstack build) is gone. The seven packages objectui installs - # are workspace packages here, so each job builds them from this tree with - # turbo instead (its "Build the @objectstack packages objectui consumes" - # step; the seven names match the `ui_consumers` filter). + # are workspace packages here, so they are built from this tree with + # turbo instead: once per run by `ui-deps` below, whose cache entries each + # job's "Build the @objectstack packages objectui consumes" step replays + # (the seven names match the `ui_consumers` filter). # - objectui's per-job "Decide whether this change needs a full run" step is # replaced by the `filter` job's `ui` / `ui_consumers` outputs. # - objectui's own commands are spelled `pnpm --dir ui run ` @@ -3091,10 +3093,188 @@ jobs: # not exist yet; turbo refuses the whole run over one). # - governance steps (the cross-repo closer contract) are not ported; they # belong to the governance-workflow audit (sub-issue 4 of the epic). + # + # ── Cache bookkeeping in the ui jobs (objectui#6577, objectui#7048) ────── + # Every cache SAVE in these five jobs is a main-phase `actions/cache/save` + # step after the job's last checking step, with `timeout-minutes` and + # `continue-on-error: true`: a combined `actions/cache` saves from a + # runner-generated post step, which no workflow key can bound, and an upload + # that stalls there runs the job into its ceiling and turns a verdict the job + # had already recorded into `cancelled`, which the merge queue cannot tell + # from `failure`. A failed save costs the next run time and says nothing about + # the code. The restores stay unbounded: a restore stall fails before any + # verdict exists. Each bound is 5 min, ~14x the slowest healthy save measured + # on the card (the ui turbo seeds 1-19 s over 22 saves, the pnpm store's + # combined post step 5-22 s over 734 jobs), and every job ceiling is at + # least twice its bound. + # `ui/scripts/__tests__/turbo-cache-save-bound.test.ts` pins all of it. + # + # The pnpm store is RESTORED by every ui job and SAVED by `ui-deps` alone, on + # a miss of its exact key: twenty jobs racing to upload one store under one + # key reserve it once and fail nineteen times. + + # ── Build UI Deps: the seven packages, built once per run (#25869) ─────── + # Each ui job used to build the seven @objectstack packages itself. Measured + # over the 51 CI runs that scheduled ui jobs between the ui/ cutover and + # 2026-10-26T05:30Z: the step replayed in 1-2 s from a job's own turbo seed + # on 24 runs, and built cold in EVERY ui job on the other 27 (86-288 s each, + # median 201 s) -- 1191 runner-minutes in all, 33-77 per full-lane run. A + # cold seed is the common case, not the exception: the seeds are saved per + # job on `main` pushes, and a pull request or queue entry whose closure + # differs from the newest seed's (any spec change landed since, or one ahead + # in the queue) misses in all twenty jobs at once. + # + # So this job builds the closure ONCE and uploads the turbo cache entries of + # that build (36 tasks, ~27 MB, already zstd), and each ui job downloads them + # into `.turbo/cache` before its own unchanged build step, which then + # replays every task in about a second. Entries, not `dist/` trees: the + # Type Check UI and Lint UI jobs run further turbo tasks that depend on + # `^build`, and only a cache HIT stops turbo rebuilding the seven there. + # Turbo still hashes every task, so a seed that does not match this tree is + # rebuilt, never trusted. + # + # FAILS OPEN. The download is `continue-on-error` and every ui job keeps its + # build step: with no artifact (this job red, cancelled or skipped, or the + # artifact service down) a job builds cold exactly as before, and a closure + # that does not build reds the same step it always did. That is also why the + # ui jobs' `if:` lines are unchanged: their `!cancelled()` already runs them + # whatever this job's result. + # + # The cost is wall clock: a ui job now starts after this one, ~1 min of + # setup and upload later than it used to start building. The build carries + # the remote-cache lines like every other build carrier, so a queue or + # `main` run whose closure was built before replays it from the remote. + # ⛔ A pull_request run still gets no remote credentials (Build Core's + # "Turbo remote cache (#21186)" comment says why), so a PR is served by its + # own run's build and the seed below. + ui-deps: + name: Build UI Deps + needs: filter + # The union of the four ui jobs' conditions: Type Check UI and Build & E2E + # UI's, which Test UI and Lint UI's imply. + if: ${{ !cancelled() && needs.filter.outputs.ui_tree != 'false' && (needs.filter.outputs.ui != 'false' || needs.filter.outputs.ui_consumers != 'false') }} + runs-on: ubuntu-latest + # The build step's population above: max 288 s cold; with setup, upload + # and the saves ~7 min. The smallest round number >= 3x that and >= that + # + 15 min (objectui's lint.yml ceiling rule) is 25. + timeout-minutes: 25 + permissions: + contents: read + + steps: + - name: Checkout repository + uses: actions/checkout@v7 + + - name: Setup Node.js + uses: actions/setup-node@v7 + with: + node-version: '22' + + - name: Setup pnpm + uses: ./.github/actions/setup-pnpm + + - name: Get pnpm store directory + shell: bash + run: | + echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV + + - name: Restore pnpm cache + id: pnpm-cache + uses: actions/cache/restore@v6 + with: + path: ${{ env.STORE_PATH }} + key: ${{ runner.os }}-pnpm-store-v3-${{ hashFiles('**/pnpm-lock.yaml') }} + restore-keys: | + ${{ runner.os }}-pnpm-store-v3- + + - name: Restore Turbo cache + id: turbo-cache + uses: actions/cache/restore@v6 + with: + path: .turbo/cache + key: ${{ runner.os }}-turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-turbo-${{ github.job }}-${{ github.ref_name }}- + ${{ runner.os }}-turbo-${{ github.job }}- + ${{ runner.os }}-turbo-build-core-${{ github.ref_name }}- + ${{ runner.os }}-turbo-build-core- + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + # The same command as the step of this name in the four ui jobs, so it + # schedules the same tasks with the same hashes; the four TURBO_* lines + # are Build Core's, verbatim (its "Turbo remote cache (#21186)" comment + # and the PINS paragraph there). + - name: Build the @objectstack packages objectui consumes + env: + TURBO_CACHE: ${{ (github.event_name == 'merge_group' || ((github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.ref == 'refs/heads/main')) && 'local:rw,remote:rw' || 'local:rw,remote:r' }} + TURBO_TOKEN: ${{ (github.event_name == 'merge_group' || github.ref == 'refs/heads/main') && secrets.TURBO_REMOTE_CACHE_SIGNATURE_KEY != '' && secrets.TURBO_TOKEN || '' }} + TURBO_TEAM: ${{ vars.TURBO_TEAM }} + TURBO_REMOTE_CACHE_SIGNATURE_KEY: ${{ (github.event_name == 'merge_group' || github.ref == 'refs/heads/main') && secrets.TURBO_REMOTE_CACHE_SIGNATURE_KEY || '' }} + run: | + pnpm exec turbo run build --concurrency=4 \ + --filter=@objectstack/spec... \ + --filter=@objectstack/client... \ + --filter=@objectstack/core... \ + --filter=@objectstack/formula... \ + --filter=@objectstack/lint... \ + --filter=@objectstack/sdui-parser... \ + --filter=@objectstack/types... \ + --summarize --log-file=.turbo/remote-cache/ui-deps.json + + - name: The Turbo remote cache was on for the build + run: | + node scripts/ci/assert-turbo-remote-cache.mjs --site 'Build UI Deps' \ + --log .turbo/remote-cache/ui-deps.json \ + --move-summary-to "$RUNNER_TEMP/turbo-remote-cache" + + # Keeps exactly the entries the build's run summary names, so the + # artifact and the seed carry this closure and nothing the restored seed + # brought with it. `save=false` (no summary) uploads and saves nothing. + - name: Prune the Turbo cache to this build's tasks + id: prune-turbo-cache + run: | + node scripts/ci/prune-turbo-cache.mjs --site 'Build UI Deps' \ + --cache .turbo/cache --run-summaries "$RUNNER_TEMP/turbo-remote-cache" + + # The entries are zstd tarballs already, so the artifact is stored, not + # recompressed. One day: only this run's ui jobs read it. + - name: Hand the build to the ui jobs + if: steps.prune-turbo-cache.outputs.save == 'true' + uses: actions/upload-artifact@v7 + with: + name: ui-deps-turbo-cache + path: .turbo/cache/ + if-no-files-found: error + compression-level: 0 + retention-days: 1 + overwrite: true + + - name: Save Turbo cache (main only) + if: >- + github.event_name == 'push' + && steps.turbo-cache.outputs.cache-hit != 'true' + && steps.prune-turbo-cache.outputs.save == 'true' + uses: actions/cache/save@v6 + timeout-minutes: 5 + continue-on-error: true + with: + path: .turbo/cache + key: ${{ runner.os }}-turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }} + + - name: Save pnpm cache + if: steps.pnpm-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@v6 + timeout-minutes: 5 + continue-on-error: true + with: + path: ${{ env.STORE_PATH }} + key: ${{ runner.os }}-pnpm-store-v3-${{ hashFiles('**/pnpm-lock.yaml') }} type-check-ui: name: Type Check UI - needs: filter + needs: [filter, ui-deps] if: ${{ !cancelled() && needs.filter.outputs.ui_tree != 'false' && (needs.filter.outputs.ui != 'false' || needs.filter.outputs.ui_consumers != 'false') }} runs-on: ubuntu-latest timeout-minutes: 40 @@ -3104,6 +3284,9 @@ jobs: # @object-ui/app-shell's and the Console's tsc need ~2.3 GB and OOM at # Node's default heap (the import spike, objectstack#25678 finding 2). NODE_OPTIONS: --max-old-space-size=4096 + # Every turbo run writes its summary to `.turbo/runs/`, which the prune + # before the seed save reads (neither enters a task hash). + TURBO_RUN_SUMMARY: 'true' steps: - name: Checkout repository @@ -3126,8 +3309,9 @@ jobs: run: | echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV - - name: Setup pnpm cache - uses: actions/cache@v6 + # Restore only: Build UI Deps saves the store (see the ui jobs' header). + - name: Restore pnpm cache + uses: actions/cache/restore@v6 with: path: ${{ env.STORE_PATH }} key: ${{ runner.os }}-pnpm-store-v3-${{ hashFiles('**/pnpm-lock.yaml') }} @@ -3135,6 +3319,7 @@ jobs: ${{ runner.os }}-pnpm-store-v3- - name: Restore Turbo cache + id: turbo-cache uses: actions/cache/restore@v6 with: path: .turbo/cache @@ -3145,6 +3330,16 @@ jobs: ${{ runner.os }}-turbo-build-core-${{ github.ref_name }}- ${{ runner.os }}-turbo-build-core- + # Build UI Deps' turbo cache entries for the seven packages, which the + # build step below replays (that job's comment). Fails open: without + # them the build step builds cold, as it did before that job existed. + - name: Download the @objectstack packages objectui consumes + continue-on-error: true + uses: actions/download-artifact@v8 + with: + name: ui-deps-turbo-cache + path: .turbo/cache + # objectui runs this first because it is instant and catches the # structural mistake: turbo skips a package with no `type-check` script, # so without it a package that lacks one reads as passing. @@ -3244,7 +3439,6 @@ jobs: EVENT: ${{ github.event_name }} PR_BASE: ${{ github.event.pull_request.base.sha }} QUEUE_BASE: ${{ github.event.merge_group.base_sha }} - TURBO_RUN_SUMMARY: 'true' run: | case "$EVENT" in pull_request) base="$PR_BASE" ;; @@ -3285,9 +3479,23 @@ jobs: - name: Type-check ui repo-root vitest setup files run: pnpm --dir ui run type-check:vitest-setup - - name: Save Turbo cache (main only) + # Bookkeeping after the last checking step (the ui jobs' header): the + # seed keeps only the entries this job's turbo runs used. + - name: Prune the Turbo cache to this run's tasks (main only) + id: prune-turbo-cache if: always() && github.event_name == 'push' + run: | + node scripts/ci/prune-turbo-cache.mjs --site 'Type Check UI' \ + --cache .turbo/cache --run-summaries .turbo/runs + + - name: Save Turbo cache (main only) + if: >- + always() && github.event_name == 'push' + && steps.turbo-cache.outputs.cache-hit != 'true' + && steps.prune-turbo-cache.outputs.save == 'true' uses: actions/cache/save@v6 + timeout-minutes: 5 + continue-on-error: true with: path: .turbo/cache key: ${{ runner.os }}-turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }} @@ -3327,7 +3535,7 @@ jobs: # what it does not see; the full battery on `main` is the backstop. test-ui: name: Test UI (${{ matrix.shard }}/16) - needs: filter + needs: [filter, ui-deps] if: ${{ !cancelled() && needs.filter.outputs.ui_tree != 'false' && needs.filter.outputs.ui != 'false' }} runs-on: ubuntu-latest timeout-minutes: 30 @@ -3359,24 +3567,25 @@ jobs: run: | echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV - - name: Setup pnpm cache - uses: actions/cache@v6 + # Restore only: Build UI Deps saves the store (see the ui jobs' header). + - name: Restore pnpm cache + uses: actions/cache/restore@v6 with: path: ${{ env.STORE_PATH }} key: ${{ runner.os }}-pnpm-store-v3-${{ hashFiles('**/pnpm-lock.yaml') }} restore-keys: | ${{ runner.os }}-pnpm-store-v3- - - name: Restore Turbo cache - uses: actions/cache/restore@v6 + # Build UI Deps' turbo cache entries for the seven packages, which the + # build step below replays (that job's comment). Fails open: without + # them the build step builds cold, as it did before that job existed. + # No turbo seed of this job's own: that build is its only turbo run. + - name: Download the @objectstack packages objectui consumes + continue-on-error: true + uses: actions/download-artifact@v8 with: + name: ui-deps-turbo-cache path: .turbo/cache - key: ${{ runner.os }}-turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-turbo-${{ github.job }}-${{ github.ref_name }}- - ${{ runner.os }}-turbo-${{ github.job }}- - ${{ runner.os }}-turbo-build-core-${{ github.ref_name }}- - ${{ runner.os }}-turbo-build-core- - name: Install dependencies run: pnpm install --frozen-lockfile @@ -3475,21 +3684,35 @@ jobs: pnpm --dir ui run test --shard=${{ matrix.shard }}/16 --reporter=default --reporter=github-actions fi - - name: Save Turbo cache (main only) - if: always() && github.event_name == 'push' && matrix.shard == 1 - uses: actions/cache/save@v6 - with: - path: .turbo/cache - key: ${{ runner.os }}-turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }} - + # ── Lint UI and Build & E2E UI: the ceilings (#25869 row F13) ────────── + # objectui's rule (its lint.yml `Lint` ceiling): the smallest round number + # that is both >= 3x the slowest successful run and >= that run + 15 min, so + # a healthy-but-slow runner cannot hit it and a wedge dies in minutes rather + # than hours. ⛔ Raising one is the ruled-out non-fix (objectui#6577, + # objectui#7048): the hang runs longer and the check still reads `cancelled`. + # Population: every successful job of each name in the CI runs completed + # between the ui/ cutover and 2026-10-26T05:30Z, `completed_at` minus + # `started_at` from the Actions jobs endpoint, before Build UI Deps existed + # (so each run includes a cold seven-package build of up to 288 s): + # - Lint UI: n=42, min 71 s, median 112 s, p95 524 s, max 558 s (9.3 min) + # => max(27.9, 24.3) => 30, the value it was ported with. + # - Build & E2E UI: n=51, min 106 s, median 170 s, p95 423 s, max 457 s + # (7.6 min) => max(22.9, 22.6) => 25, down from objectui's 30 plus the + # 15 the port added for the in-job build, which no derivation backed. + # Re-derive when a job's work changes; ui/scripts/__tests__/ + # turbo-cache-save-bound.test.ts holds both values. lint-ui: name: Lint UI - needs: filter + needs: [filter, ui-deps] if: ${{ !cancelled() && needs.filter.outputs.ui_tree != 'false' && needs.filter.outputs.ui != 'false' }} runs-on: ubuntu-latest timeout-minutes: 30 permissions: contents: read + env: + # Every turbo run writes its summary to `.turbo/runs/`, which the prune + # before the seed save reads (neither enters a task hash). + TURBO_RUN_SUMMARY: 'true' steps: - name: Checkout repository @@ -3508,8 +3731,9 @@ jobs: run: | echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV - - name: Setup pnpm cache - uses: actions/cache@v6 + # Restore only: Build UI Deps saves the store (see the ui jobs' header). + - name: Restore pnpm cache + uses: actions/cache/restore@v6 with: path: ${{ env.STORE_PATH }} key: ${{ runner.os }}-pnpm-store-v3-${{ hashFiles('**/pnpm-lock.yaml') }} @@ -3517,6 +3741,7 @@ jobs: ${{ runner.os }}-pnpm-store-v3- - name: Restore Turbo cache + id: turbo-cache uses: actions/cache/restore@v6 with: path: .turbo/cache @@ -3527,6 +3752,16 @@ jobs: ${{ runner.os }}-turbo-build-core-${{ github.ref_name }}- ${{ runner.os }}-turbo-build-core- + # Build UI Deps' turbo cache entries for the seven packages, which the + # build step below replays (that job's comment). Fails open: without + # them the build step builds cold, as it did before that job existed. + - name: Download the @objectstack packages objectui consumes + continue-on-error: true + uses: actions/download-artifact@v8 + with: + name: ui-deps-turbo-cache + path: .turbo/cache + - name: Verify ui lint coverage run: pnpm --dir ui run lint:coverage @@ -3560,8 +3795,6 @@ jobs: run: pnpm --dir ui run lint:root - name: Lint the ui packages - env: - TURBO_RUN_SUMMARY: 'true' run: | pnpm exec turbo run lint --concurrency=4 \ --filter='./ui/packages/*' \ @@ -3580,19 +3813,34 @@ jobs: - name: Verify the ui CLI's own check command passes on the ui tree run: pnpm --dir ui run check - - name: Save Turbo cache (main only) + # Bookkeeping after the last checking step (the ui jobs' header): the + # seed keeps only the entries this job's turbo runs used. + - name: Prune the Turbo cache to this run's tasks (main only) + id: prune-turbo-cache if: always() && github.event_name == 'push' + run: | + node scripts/ci/prune-turbo-cache.mjs --site 'Lint UI' \ + --cache .turbo/cache --run-summaries .turbo/runs + + - name: Save Turbo cache (main only) + if: >- + always() && github.event_name == 'push' + && steps.turbo-cache.outputs.cache-hit != 'true' + && steps.prune-turbo-cache.outputs.save == 'true' uses: actions/cache/save@v6 + timeout-minutes: 5 + continue-on-error: true with: path: .turbo/cache key: ${{ runner.os }}-turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }} build-e2e-ui: name: Build & E2E UI - needs: filter + needs: [filter, ui-deps] if: ${{ !cancelled() && needs.filter.outputs.ui_tree != 'false' && (needs.filter.outputs.ui != 'false' || needs.filter.outputs.ui_consumers != 'false') }} runs-on: ubuntu-latest - timeout-minutes: 45 + # Derived above Lint UI. + timeout-minutes: 25 permissions: contents: read env: @@ -3617,24 +3865,25 @@ jobs: run: | echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV - - name: Setup pnpm cache - uses: actions/cache@v6 + # Restore only: Build UI Deps saves the store (see the ui jobs' header). + - name: Restore pnpm cache + uses: actions/cache/restore@v6 with: path: ${{ env.STORE_PATH }} key: ${{ runner.os }}-pnpm-store-v3-${{ hashFiles('**/pnpm-lock.yaml') }} restore-keys: | ${{ runner.os }}-pnpm-store-v3- - - name: Restore Turbo cache - uses: actions/cache/restore@v6 + # Build UI Deps' turbo cache entries for the seven packages, which the + # build step below replays (that job's comment). Fails open: without + # them the build step builds cold, as it did before that job existed. + # No turbo seed of this job's own: that build is its only turbo run. + - name: Download the @objectstack packages objectui consumes + continue-on-error: true + uses: actions/download-artifact@v8 with: + name: ui-deps-turbo-cache path: .turbo/cache - key: ${{ runner.os }}-turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-turbo-${{ github.job }}-${{ github.ref_name }}- - ${{ runner.os }}-turbo-${{ github.job }}- - ${{ runner.os }}-turbo-build-core-${{ github.ref_name }}- - ${{ runner.os }}-turbo-build-core- - name: Install dependencies run: pnpm install --frozen-lockfile @@ -3715,10 +3964,3 @@ jobs: with: path: ~/.cache/ms-playwright key: playwright-${{ runner.os }}-${{ steps.playwright-version.outputs.version }} - - - name: Save Turbo cache (main only) - if: always() && github.event_name == 'push' - uses: actions/cache/save@v6 - with: - path: .turbo/cache - key: ${{ runner.os }}-turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }} diff --git a/ui/scripts/__tests__/report-turbo-cache.test.ts b/ui/scripts/__tests__/report-turbo-cache.test.ts index 9fdce593f88..fcfe14390f6 100644 --- a/ui/scripts/__tests__/report-turbo-cache.test.ts +++ b/ui/scripts/__tests__/report-turbo-cache.test.ts @@ -206,6 +206,12 @@ describe('report-turbo-cache (objectui#12222)', () => { // (objectstack#25697, row 7 and row 16 of its workflow roster) carry // objectstack's own Turbo-cache steps instead, on purpose: a // `-turbo---` key that restores from `build-core`'s cache -// when the job has none, a `Save Turbo cache (main only)` on push, and no -// prune or report step. Running this reporter there is a follow-up port; it -// brings these pins back against those jobs (`helpers/root-ci.ts`). +// when the job has none, and a `Save Turbo cache (main only)` on push. +// objectstack#25869 (row F2) settled the follow-up port: the root jobs do NOT +// run this script. The prune before each seed save is objectstack's +// `scripts/ci/prune-turbo-cache.mjs`, the same summary-driven prune, already +// the root's for every other seed, and the hit/miss report is +// `scripts/ci/assert-turbo-remote-cache.mjs`'s table on `Build UI Deps`, the +// one job that builds what the others replay. `turbo-cache-save-bound.test.ts` +// pins those saves. This script retires with its last callers, the nested +// workflows under `ui/.github/workflows/` (objectstack#25712). diff --git a/ui/scripts/__tests__/turbo-cache-save-bound.test.ts b/ui/scripts/__tests__/turbo-cache-save-bound.test.ts new file mode 100644 index 00000000000..166100954a0 --- /dev/null +++ b/ui/scripts/__tests__/turbo-cache-save-bound.test.ts @@ -0,0 +1,266 @@ +import { describe, expect, it } from 'vitest'; +import fs from 'node:fs'; +import { parse as parseYaml } from 'yaml'; + +import { rootCiPath, UI_JOBS } from './helpers/root-ci'; + +/** + * objectui#6577 — cache bookkeeping voided a verdict the gate had already + * recorded; objectstack#25869 — the root ui jobs that objectui's CI became. + * + * On 2026-08-26 the merge-group build of objectui#6571 was ejected from the + * queue with `CI_FAILURE` while every substantive check passed. All 21 real + * steps of objectui's `Type Check` job succeeded, and then `Post Turbo Cache`, + * the save phase of `actions/cache@v6`, spent 13m09s inside the upload and was + * still there when the job's ceiling fired. The job went `cancelled`, the merge + * queue cannot tell `cancelled` from `failure`, and a pull request that had + * passed was dequeued. The same job on the same head thirteen minutes earlier + * saved its cache in ONE SECOND: a transient, not a property of the workflow. + * + * The ordering is the whole point: a job's verdict is recorded by its command + * steps, and everything after them is bookkeeping, which must be bounded and + * must not be able to speak for the code. objectui pinned that on its own + * `type-check` job. The port to the objectstack monorepo's root `ci.yml` + * (objectstack#25697) carried the jobs over on combined `actions/cache` steps + * and unbounded saves, and objectstack#25879 retired this file with the nested + * workflow it read; objectstack#25869 bounded the root jobs' saves and + * restores it here, by path, over every ui job and `ui-deps`, the job that + * builds the seven @objectstack packages once per run for the other four. + * + * Why this needs a pin at all — the fix is invisible when reverted. Folding a + * split back into one `actions/cache` step, or dropping either key from a + * save, restores the defect with nothing going red: the cache still works, + * every run still passes, and the next transient upload stall cancels the next + * green check (objectui#3009, #3181, #3494: "looks like enforcement, isn't"). + * + * Deliberately NOT asserted: that a restore is bounded. A restore stall fails + * BEFORE any verdict exists — a gate that did not run, which is honest. + */ + +interface Step { + name?: string; + uses?: string; + run?: string; + id?: string; + if?: string; + 'timeout-minutes'?: number; + 'continue-on-error'?: boolean; + with?: Record; +} +interface Job { + name?: string; + needs?: string | string[]; + if?: string; + 'timeout-minutes'?: number; + steps: Step[]; +} + +const workflow = parseYaml(fs.readFileSync(rootCiPath, 'utf8')) as { jobs: Record }; + +/** The job that builds the seven packages once and hands them to the four. */ +const DEPS_JOB = 'ui-deps'; +const CACHE_JOBS = [DEPS_JOB, ...Object.values(UI_JOBS)]; + +/** + * Each job's ceiling as derived beside it in ci.yml. A pin, not a restatement + * of a derivation: moving one is a decision that re-derives it there and moves + * it here in the same commit. + * - type-check-ui: the value it was ported with; objectui's own rule here + * was "does not rise" (#6577's ruled-out non-fix). + * - lint-ui, build-e2e-ui: #25869 row F13, the ceiling comment above lint-ui. + * - ui-deps: the derivation on that job. + */ +const CEILINGS: Record = { + [DEPS_JOB]: 25, + [UI_JOBS.typeCheck]: 40, + [UI_JOBS.lint]: 30, + [UI_JOBS.buildE2e]: 25, +}; + +function job(key: string): Job { + const found = workflow.jobs[key]; + expect(found, `ci.yml must still define a \`${key}:\` job`).toBeDefined(); + // Non-vacuity: every assertion below scans this step list, so a parse that + // collapsed it to nothing would make all of them pass while reading nothing. + expect(found.steps.length, `the \`${key}\` job parsed to implausibly few steps`).toBeGreaterThan(5); + return found; +} + +const isSave = (s: Step) => (s.uses ?? '').startsWith('actions/cache/save@'); +const isRestore = (s: Step) => (s.uses ?? '').startsWith('actions/cache/restore@'); +const BUILD_STEP = 'Build the @objectstack packages objectui consumes'; +const DOWNLOAD_STEP = 'Download the @objectstack packages objectui consumes'; + +describe('root ci.yml — the ui jobs\' cache saves are bounded and non-fatal (objectui#6577)', () => { + it('reads the saves it is about', () => { + // Non-vacuity for the per-save cases below: ui-deps saves the turbo seed + // and the pnpm store; type-check-ui and lint-ui save their turbo seeds. + const saves = CACHE_JOBS.flatMap((key) => job(key).steps.filter(isSave)); + expect(saves.length, 'the five ui jobs declare implausibly few cache saves').toBeGreaterThanOrEqual(4); + expect( + job(UI_JOBS.typeCheck).steps.filter((s) => isSave(s) && s.with?.path === '.turbo/cache').length, + 'the `type-check-ui` job must save `.turbo/cache` through exactly one `actions/cache/save` step', + ).toBe(1); + }); + + it.each(CACHE_JOBS)('%s saves through MAIN-phase steps, never the combined action', (key) => { + // Combined `actions/cache` declares `post: dist/save/index.js`: its save is + // a step the RUNNER generates at job end, and no workflow syntax attaches + // `timeout-minutes` or `continue-on-error` to a generated post step. + // Splitting is the only way the keys below can exist at all. + const combined = job(key) + .steps.filter((s) => /^actions\/cache@/.test(s.uses ?? '')) + .map((s) => s.name ?? '(unnamed)'); + expect( + combined, + `the \`${key}\` job is back on the combined \`actions/cache\` action:\n` + + combined.map((n) => ` - ${n}`).join('\n') + + '\n\nIts save runs as a runner-generated POST step, which cannot carry `timeout-minutes` or ' + + '`continue-on-error`, so an upload stall runs the job into its ceiling and cancels a check ' + + 'that had already passed (objectui#6577). Use `actions/cache/restore` plus a bounded ' + + '`actions/cache/save`.', + ).toEqual([]); + }); + + it.each(CACHE_JOBS)('%s bounds every save, and no failed save speaks for the code', (key) => { + for (const save of job(key).steps.filter(isSave)) { + expect( + typeof save['timeout-minutes'], + `\`${key}\` › "${save.name}" must declare \`timeout-minutes\`: it is the only thing between a ` + + 'stalled upload and the job ceiling, and the ceiling reports `cancelled` (objectui#6577).', + ).toBe('number'); + expect( + save['continue-on-error'], + `\`${key}\` › "${save.name}" must declare \`continue-on-error: true\`. Without it the bound only ` + + 'trades a `cancelled` check for a red one, and a cache that failed to upload says nothing ' + + 'about the code under test.', + ).toBe(true); + } + }); + + it.each(CACHE_JOBS)('%s leaves the verdict at least as much ceiling as bookkeeping can take', (key) => { + const j = job(key); + const ceiling = j['timeout-minutes']; + expect(typeof ceiling, `the \`${key}\` job must declare \`timeout-minutes\``).toBe('number'); + for (const save of j.steps.filter(isSave)) { + expect( + (save['timeout-minutes'] as number) * 2, + `\`${key}\` › "${save.name}" may run for ${save['timeout-minutes']} of the job's ${ceiling} ` + + 'minutes; a bound that large can still starve the checking steps and cancel the job.', + ).toBeLessThanOrEqual(ceiling as number); + } + }); + + it.each(CACHE_JOBS)('%s saves AFTER every command step — bookkeeping follows the verdict', (key) => { + const steps = job(key).steps; + // The prune before a seed save is bookkeeping too, and a `run:` step; the + // verdict is the last command that is neither. + const lastCommand = steps + .map((s) => s.run !== undefined && !/prune-turbo-cache\.mjs/.test(s.run)) + .lastIndexOf(true); + expect(lastCommand, `the \`${key}\` job must still run commands`).toBeGreaterThan(-1); + for (const save of steps.filter(isSave)) { + expect( + steps.indexOf(save), + `\`${key}\` › "${save.name}" must come after the job's last command step, ` + + `"${steps[lastCommand].name}": everything after it is bookkeeping, and that ordering is ` + + 'what makes the save safe to bound and safe to lose (objectui#6577).', + ).toBeGreaterThan(lastCommand); + } + }); + + it.each(CACHE_JOBS)('%s still caches what the combined step did', (key) => { + // Same path, same key, and the save skipped on an exact hit of the + // restore's primary key — which is how the combined action decided. + const steps = job(key).steps; + for (const save of steps.filter(isSave)) { + const restores = steps.filter((s) => isRestore(s) && s.with?.path === save.with?.path); + expect(restores.length, `\`${key}\` › "${save.name}" saves a path no step restores`).toBe(1); + const [restore] = restores; + expect( + save.with?.key, + `\`${key}\` › "${save.name}" must write the key "${restore.name}" asks for, or every run misses`, + ).toBe(restore.with?.key); + expect(restore.id, `\`${key}\` › "${restore.name}" needs an \`id\` for the save's guard`).toBeTruthy(); + expect( + save.if ?? '', + `\`${key}\` › "${save.name}" must skip on an exact hit of the primary key — ` + + `\`steps.${restore.id}.outputs.cache-hit != 'true'\`; without it every re-run re-uploads a ` + + 'cache that already exists.', + ).toContain(`steps.${restore.id}.outputs.cache-hit != 'true'`); + } + }); + + it.each(Object.entries(CEILINGS))('%s keeps its derived ceiling of %i minutes', (key, ceiling) => { + // Raising a ceiling does not fix a hung save: the hang runs longer and the + // check still reports `cancelled` (objectui#6577, objectui#7048). Moving + // one is a deliberate act: re-derive it beside the job, move it here. + expect(job(key)['timeout-minutes'], `the \`${key}\` job ceiling moved`).toBe(ceiling); + }); +}); + +describe('root ci.yml — ui-deps builds the seven packages once for the ui jobs (objectstack#25869)', () => { + const deps = () => job(DEPS_JOB); + const step = (key: string, name: string) => { + const found = job(key).steps.filter((s) => s.name === name); + expect(found.length, `the \`${key}\` job must carry exactly one "${name}" step`).toBe(1); + return found[0]; + }; + /** The build command with the carrier-only flags taken off. */ + const buildCommand = (run: string) => + run + .replace(/\s*\\?\n?\s*--summarize --log-file=\S+/, '') + .replace(/\s+/g, ' ') + .trim(); + const upload = () => { + const found = deps().steps.filter((s) => (s.uses ?? '').startsWith('actions/upload-artifact@')); + expect(found.length, '`ui-deps` must upload exactly one artifact').toBe(1); + return found[0]; + }; + + it('runs whenever any ui job runs', () => { + // The union of the four conditions is Type Check UI's (and Build & E2E + // UI's): Test UI and Lint UI run on a subset of it. + expect(deps().if).toBe(job(UI_JOBS.typeCheck).if); + expect(job(UI_JOBS.buildE2e).if).toBe(job(UI_JOBS.typeCheck).if); + }); + + it('hands over the pruned turbo cache, only when the prune found the build', () => { + const up = upload(); + expect(up.with?.path?.replace(/\/$/, '')).toBe('.turbo/cache'); + expect(up.if ?? '').toContain("steps.prune-turbo-cache.outputs.save == 'true'"); + const steps = deps().steps; + expect(steps.indexOf(up), 'the upload must follow the build').toBeGreaterThan( + steps.indexOf(step(DEPS_JOB, BUILD_STEP)), + ); + }); + + it.each(Object.values(UI_JOBS))('%s waits for ui-deps without letting its result skip the job', (key) => { + const j = job(key); + const needs = Array.isArray(j.needs) ? j.needs : [j.needs]; + expect(needs).toContain(DEPS_JOB); + // A job whose `if:` names no status function is SKIPPED when a job it + // needs did not succeed. `!cancelled()` is what keeps a red or skipped + // ui-deps from silently skipping a required ui check. + expect(j.if ?? '', `the \`${key}\` job's \`if:\` must start with \`!cancelled()\``).toMatch(/^\$\{\{ !cancelled\(\) && /); + }); + + it.each(Object.values(UI_JOBS))('%s downloads the build fail-open, before the step that replays it', (key) => { + const download = step(key, DOWNLOAD_STEP); + expect(download.uses ?? '').toMatch(/^actions\/download-artifact@/); + expect(download.with?.name, 'the download must name the artifact ui-deps uploads').toBe(upload().with?.name); + expect(download.with?.path).toBe('.turbo/cache'); + expect( + download['continue-on-error'], + 'no artifact must mean "build cold", never a red or skipped ui check', + ).toBe(true); + const steps = job(key).steps; + expect(steps.indexOf(download)).toBeLessThan(steps.indexOf(step(key, BUILD_STEP))); + }); + + it.each(Object.values(UI_JOBS))('%s builds the closure ui-deps built, so every task replays', (key) => { + // Filters do not enter a task hash, but a filter one side lacks is a task + // the other never built: it would build cold in every ui job again. + expect(buildCommand(step(key, BUILD_STEP).run ?? '')).toBe(buildCommand(step(DEPS_JOB, BUILD_STEP).run ?? '')); + }); +});