diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml index b640b7106..c347dbd8e 100644 --- a/.github/actionlint.yaml +++ b/.github/actionlint.yaml @@ -6,3 +6,5 @@ self-hosted-runner: - blacksmith-6vcpu-macos-26 - blacksmith-8vcpu-ubuntu-2404 - blacksmith-32vcpu-ubuntu-2404 + # GitHub-hosted larger runner (org runner group "Maple"); actionlint only knows standard labels. + - maple-windows-2025-16core diff --git a/.github/workflows/agent-desktop-build.yml b/.github/workflows/agent-desktop-build.yml index 87f6d3790..2ea6bd402 100644 --- a/.github/workflows/agent-desktop-build.yml +++ b/.github/workflows/agent-desktop-build.yml @@ -189,7 +189,8 @@ jobs: name: Build Windows binary (${{ matrix.variant }}) needs: changes if: ${{ always() && !cancelled() && (needs.changes.result != 'success' || needs.changes.outputs.agent != 'false') && github.repository == 'MaplePrivacyLabs/Maple' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') }} - runs-on: blacksmith-8vcpu-windows-2025 + # GitHub-hosted, like the signing and verify jobs that consume its binary. + runs-on: maple-windows-2025-16core permissions: contents: read timeout-minutes: 120 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ea727600d..bbda69781 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -65,6 +65,7 @@ jobs: CI_LINUX_HEAVY_RUNNER: ${{ vars.CI_LINUX_HEAVY_RUNNER }} CI_MACOS_RUNNER: ${{ vars.CI_MACOS_RUNNER }} CI_WINDOWS_RUNNER: ${{ vars.CI_WINDOWS_RUNNER }} + CI_WINDOWS_GITHUB_RUNNER: ${{ vars.CI_WINDOWS_GITHUB_RUNNER }} run: python3 scripts/ci/plan.py github nix: @@ -101,7 +102,7 @@ jobs: ~/.cargo/registry/index ~/.cargo/registry/cache ~/.cargo/git/db - - name: Rust cache (restore everywhere, save only on master and on Blacksmith) + - name: Rust cache (restore everywhere, save only on master and in the lane's home store) if: matrix.cache != '' uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 with: @@ -113,8 +114,8 @@ jobs: add-rust-environment-hash-key: "false" # The action runs cargo from the workspace directory: use an absolute flake path. cmd-format: nix develop --no-update-lock-file ${{ github.workspace }}/${{ matrix.flake }} -c {0} - # A failover to a GitHub-hosted runner must not fill GitHub's small cache store. - save-if: ${{ github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'schedule') && startsWith(matrix.runner, 'blacksmith-') }} + # cache_save (plan.py): a failover runner never saves into another vendor's store. + save-if: ${{ github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'schedule') && matrix.cache_save }} - name: Run lane env: LANE: ${{ matrix.id }} @@ -138,7 +139,7 @@ jobs: with: persist-credentials: false - uses: ./.github/actions/setup-agent-windows - - name: Rust cache (restore everywhere, save only on master and on Blacksmith) + - name: Rust cache (restore everywhere, save only on master and in the lane's home store) if: matrix.cache != '' uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 with: @@ -146,22 +147,22 @@ jobs: shared-key: ${{ matrix.cache }}-${{ matrix.cache_key }} workspaces: ${{ matrix.workspaces }} add-rust-environment-hash-key: "false" - # A failover to a GitHub-hosted runner must not fill GitHub's small cache store. - save-if: ${{ github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'schedule') && startsWith(matrix.runner, 'blacksmith-') }} + # cache_save (plan.py): a failover runner never saves into another vendor's store. + save-if: ${{ github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'schedule') && matrix.cache_save }} - name: Run lane env: LANE: ${{ matrix.id }} run: python3 scripts/ci/plan.py run "$LANE" - # Native Windows updater tests and the app typecheck, compiled from source with - # no cache, credential or environment. Their receipts and Cargo logs are kept - # as an artifact even when a script fails. + # Native Windows updater tests and the app typecheck, with no credential or + # environment. Their receipts and Cargo logs are kept as an artifact even when + # a script fails. agent-windows-updater: name: agent-windows-updater needs: plan if: fromJSON(needs.plan.outputs.lanes)['agent-windows-updater'] # GitHub's image, as the native updater tests require (scripts/ci/lanes.py GITHUB_WINDOWS). - runs-on: windows-2025 + runs-on: ${{ vars.CI_WINDOWS_GITHUB_RUNNER || 'maple-windows-2025-16core' }} timeout-minutes: 60 defaults: run: @@ -171,6 +172,17 @@ jobs: with: persist-credentials: false - uses: ./.github/actions/setup-agent-windows + - name: Rust cache (restore everywhere, save only on master and on GitHub's runners) + uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 + with: + prefix-key: ci-v1 + # Its own key: these checks compile other units than agent-full-windows' + # workspace build (check-mode metadata, one package's features). + shared-key: agent-windows-updater-${{ hashFiles('apps/maple-agent/flake.lock', 'apps/maple-agent/Cargo.lock') }} + workspaces: apps/maple-agent -> target + add-rust-environment-hash-key: "false" + # A failover to a Blacksmith runner must not fill Blacksmith's store. + save-if: ${{ github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'schedule') && !startsWith(runner.name, 'blacksmith') }} - name: Typecheck the Windows app and run the native updater tests run: python3 scripts/ci/plan.py run agent-windows-updater - name: Keep the receipts and Cargo logs @@ -279,7 +291,7 @@ jobs: needs: plan if: fromJSON(needs.plan.outputs.lanes)['agent-package-windows'] # GitHub's image, where master's install smoke runs (scripts/ci/lanes.py GITHUB_WINDOWS). - runs-on: windows-2025 + runs-on: ${{ vars.CI_WINDOWS_GITHUB_RUNNER || 'maple-windows-2025-16core' }} timeout-minutes: 90 defaults: run: @@ -291,8 +303,30 @@ jobs: - uses: ./.github/actions/setup-agent-windows with: release: "true" + - name: Restore the packaging rehearsal cache + id: package-cache + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + with: + # Thin-LTO release artifacts; master's fat-LTO cache cannot serve them. + key: ci-v1-agent-package-${{ runner.os }}-${{ hashFiles('apps/maple-agent/flake.lock', 'apps/maple-agent/Cargo.lock') }} + restore-keys: ci-v1-agent-package-${{ runner.os }}- + path: | + ~/.cargo/registry/index + ~/.cargo/registry/cache + ~/.cargo/git/db + apps/maple-agent/target - name: Build, package and verify an unsigned Dev installer run: python3 scripts/ci/plan.py run agent-package-windows + - name: Save the packaging rehearsal cache (nightly, GitHub's runners only) + if: ${{ github.ref == 'refs/heads/master' && github.event_name == 'schedule' && !startsWith(runner.name, 'blacksmith') && steps.package-cache.outputs.cache-hit != 'true' }} + uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + with: + key: ${{ steps.package-cache.outputs.cache-primary-key }} + path: | + ~/.cargo/registry/index + ~/.cargo/registry/cache + ~/.cargo/git/db + apps/maple-agent/target - name: Upload the unsigned package uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: diff --git a/.github/workflows/desktop-build.yml b/.github/workflows/desktop-build.yml index 0aa30777a..d6e5a48d1 100644 --- a/.github/workflows/desktop-build.yml +++ b/.github/workflows/desktop-build.yml @@ -244,7 +244,7 @@ jobs: build-windows: needs: changes if: ${{ always() && !cancelled() && (needs.changes.result != 'success' || needs.changes.outputs.frontend != 'false' || needs.changes.outputs.windows != 'false') }} - runs-on: windows-2025 + runs-on: maple-windows-2025-16core timeout-minutes: 150 environment: windows-signing permissions: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 812c8b2a6..2ca953e2f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -249,7 +249,7 @@ jobs: build-windows: needs: classify-app-release - runs-on: windows-2025 + runs-on: maple-windows-2025-16core environment: windows-signing env: MAPLE_WINDOWS_ARTIFACT_SIGNING_ENDPOINT: ${{ secrets.AZURE_ARTIFACT_SIGNING_ENDPOINT }} diff --git a/apps/maple-agent/docs/desktop-builds.md b/apps/maple-agent/docs/desktop-builds.md index 1998b585b..2e2f17ff6 100644 --- a/apps/maple-agent/docs/desktop-builds.md +++ b/apps/maple-agent/docs/desktop-builds.md @@ -207,7 +207,7 @@ continues to use the macOS feed. Dev has no update feed on either platform. Both platforms retain one release version and source commit; separate feed paths preserve compatibility with existing macOS clients. -Windows builds run on `blacksmith-8vcpu-windows-2025` with the pinned Rust/MSVC +Windows builds run on the org's 16-core GitHub-hosted `maple-windows-2025-16core` with the pinned Rust/MSVC environment from `.github/actions/setup-agent-windows` and `RUSTFLAGS='-D warnings -C target-feature=+crt-static'`; signing stays on GitHub's `windows-2025`. `build-release.sh` selects the explicit x64 MSVC target diff --git a/docs/ci.md b/docs/ci.md index 6e27da7ee..083ceaf74 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -104,7 +104,9 @@ of the lanes your staged files select. - Never add `paths:` to `ci.yml`; `ci-ok` must report on every pull request. - Pull requests never save caches (one branch-scoped exception, outside `ci.yml`, is listed under [caches](repository-workflows.md#caches)). Master - and nightly runs save lane caches, on Blacksmith runners only. + and nightly runs save lane caches into the store of the lane's own runner + vendor: Blacksmith's, or GitHub's for the Agent lanes on GitHub's Windows + image. - Never cancel master, signed, evidence or nightly runs. - Never edit the release or publisher workflows, signing environments or OpenSecret approval gating to clear a check. A failing contract test means @@ -114,8 +116,13 @@ of the lanes your staged files select. ## Runners -Lanes use Blacksmith runners or free GitHub-hosted runners; the runner +Lanes use Blacksmith runners or GitHub-hosted runners; the runner constants are at the top of `scripts/ci/lanes.py`. During an outage, set the repository variable `CI_LINUX_RUNNER`, `CI_LINUX_HEAVY_RUNNER`, `CI_MACOS_RUNNER` or `CI_WINDOWS_RUNNER` to another label; no pull request is -needed, and a GitHub-hosted failover never saves a cache. +needed. The three Agent lanes that need GitHub's Windows image +(`agent-full-windows`, `agent-windows-updater`, `agent-package-windows`) run on +the org's 16-core GitHub-hosted runner `maple-windows-2025-16core` (billed by the +minute); `CI_WINDOWS_GITHUB_RUNNER=windows-2025` moves them back to the standard +runner, which uses the same cache store. A runner from the other vendor never +saves a cache. diff --git a/docs/repository-workflows.md b/docs/repository-workflows.md index c7161fc91..a26449235 100644 --- a/docs/repository-workflows.md +++ b/docs/repository-workflows.md @@ -36,11 +36,14 @@ branches in step with the lane table. goes to the run's branch-scoped Actions cache, which `master` and other pull requests cannot read. - Lane caches (`Swatinem/rust-cache`, prefix `ci-v1`, keyed by lane and - lockfile hashes) are saved by master pushes and the nightly run on Blacksmith - runners only. Heavy Rust caches therefore live in Blacksmith's store; GitHub's - own store, which is small, holds only what GitHub-hosted jobs need (the - Research Windows signed build, the release, the Nix installer and that EIF - check). + lockfile hashes) are saved by master pushes and the nightly run, into the + store of the lane's own runner vendor; a runner from the other vendor (a + failover) only restores. Heavy Rust caches therefore live in Blacksmith's + store. GitHub's holds what GitHub-hosted jobs need: the lane caches of + `agent-full-windows` and `agent-windows-updater`, the + `agent-package-windows` packaging cache (below), the unsigned Agent and signed + Research Windows builds, the release, the Nix installer and that EIF check. + `test_workflow_policy.py` lists these writers. - The signed master builds write the keys `release.yml` restores by prefix (`-sccache-*`, `xcode-cache-*`, `release-onnxruntime-ios-*`, `-bun-*`, `-gradle-*`). No pull-request job writes these prefixes; @@ -50,8 +53,10 @@ branches in step with the lane table. starts with a cold Bun cache; Bun installs in seconds. - `agent-desktop-build.yml`'s unsigned builders save the exact key `ci-v1-agent-release--` on master pushes, with - no fallback keys; the Agent packaging lanes only restore it. Signing, - attestation and verification jobs restore nothing. + no fallback keys; no `ci.yml` lane reads it. The Agent packaging lanes build + thin LTO into their own `ci-v1-agent-package--*` caches, which only the + nightly master run saves. Signing, attestation and verification jobs restore + nothing. ## Concurrency @@ -143,8 +148,10 @@ merging source. The `agent-windows-updater` lane runs on pull requests and master pushes that change the updater, the Windows bootstrap, the app's update and desktop code, the release profiles or the Windows packaging scripts, and with `ci:windows` or -`ci:full`. It compiles from source, with no cache, signing credential or -protected environment. `scripts/ci/windows-app-typecheck.ps1` typechecks +`ci:full`. It has no signing credential or protected environment. Its +dependencies come from its own cache, which master pushes and the nightly run +save; the Agent's own crates always compile from source. +`scripts/ci/windows-app-typecheck.ps1` typechecks `maple-agent-app --all-targets` with `cargo check --locked`, Windows-only test fixtures included, without code generation. `scripts/ci/windows-updater-tests.ps1` refuses signing and publication credentials, then runs the native updater, @@ -208,15 +215,16 @@ Compilation runs on Blacksmith runners (ARM64 `blacksmith-6vcpu-macos-26` for macOS); lane runners are the constants in `scripts/ci/lanes.py`. The signed Research macOS, Linux, Android and iOS builds compile and sign on Blacksmith. Artifact verifiers, TestFlight submission, and the Agent's signing and -attestation use GitHub-hosted runners. Windows signing and the Windows release -stay on GitHub's `windows-2025`; unsigned Agent and Research Windows builds use -`blacksmith-8vcpu-windows-2025`, with Rust, MSVC, Perl and the linker pinned by -`.github/actions/setup-agent-windows` for the Agent. The Agent's native Windows -test lanes (`agent-windows-updater`, `agent-full-windows`) and the Windows -packaging rehearsal also run on GitHub's `windows-2025`: Blacksmith's workspace -inherits a foreign write ACE that the updater's storage validation rejects, and -master's install smoke runs on GitHub's image. The Research Windows PR package has no -cache, because its signed build's cache lives in GitHub's store. +attestation use GitHub-hosted runners. Windows builds that compile run on the +org's 16-core GitHub-hosted runner `maple-windows-2025-16core`: the unsigned Agent +build, the signed Research build, the release's `build-windows`, and the Agent's +native Windows test lanes and packaging rehearsal ([CI runners](ci.md#runners)). +The short signing and verify jobs stay on GitHub's standard `windows-2025`. Rust, +MSVC, Perl and the linker are pinned by `.github/actions/setup-agent-windows` for +the Agent. Blacksmith's Windows workspace inherits a foreign write ACE that the +updater's storage validation rejects, so only the Agent Windows Clippy lane and +the Research Windows PR package use `blacksmith-8vcpu-windows-2025`; the latter +has no cache, because its signed build's cache lives in GitHub's store. Research desktop and iOS builds select Xcode 26.5 (`17F42`) from [`scripts/ci/apple-toolchain.json`](../scripts/ci/apple-toolchain.json) through diff --git a/scripts/ci/lanes.py b/scripts/ci/lanes.py index 8c7e82c8e..ace3ee2fa 100644 --- a/scripts/ci/lanes.py +++ b/scripts/ci/lanes.py @@ -28,6 +28,11 @@ class Runner: variable: Optional[str] = None # repository variable that overrides the label +def cache_store(label: str) -> str: + """Whose Actions cache a runner uses: Blacksmith runners have their own store, every other runner GitHub's.""" + return "blacksmith" if label.startswith("blacksmith-") else "github" + + # The override variables are unset; set one to fail over during an outage. LINUX = Runner("blacksmith-8vcpu-ubuntu-2404", "CI_LINUX_RUNNER") LINUX_HEAVY = Runner("blacksmith-32vcpu-ubuntu-2404", "CI_LINUX_HEAVY_RUNNER") @@ -37,7 +42,9 @@ class Runner: # The Agent updater's native Windows tests and the installer smoke test need GitHub's # runner: Blacksmith's workspace inherits a foreign write ACE the updater rejects, and # master's Windows install smoke (agent-desktop-build.yml verify-windows) runs there too. -GITHUB_WINDOWS = Runner("windows-2025") +# Its home is the org's 16-core GitHub-hosted Windows 2025 runner (the same image family, +# GitHub's cache store); CI_WINDOWS_GITHUB_RUNNER=windows-2025 falls back to the standard one. +GITHUB_WINDOWS = Runner("maple-windows-2025-16core", "CI_WINDOWS_GITHUB_RUNNER") # .github/actions/setup-agent-windows pins this toolchain (Windows has no Nix); # agent-lint checks that the Agent flake's rustc still reports it. AGENT_WINDOWS_RUST = "1.98.0" @@ -224,8 +231,8 @@ def deny(manifest: str, component: str) -> str: " && cargo clippy --workspace --all-targets --locked -- -D warnings", cache="agent-windows", timeout=45, **AGENT_WIN), # The Windows app typecheck and the native updater, filesystem, Authenticode and - # bootstrap tests, compiled without a cache. Both scripts always run; each needs the - # Actions context and leaves a receipt and its Cargo log, which the job keeps. + # bootstrap tests, from the job's own dependency cache. Both scripts always run; each + # needs the Actions context and leaves a receipt and its Cargo log, which the job keeps. Lane("agent-windows-updater", AGENT_WINDOWS_UPDATER, labels=("ci:windows", "ci:full"), job="agent-windows-updater", runner=GITHUB_WINDOWS, run="status=0; pwsh -NoProfile -File scripts/ci/windows-app-typecheck.ps1 || status=1;" @@ -261,7 +268,7 @@ def deny(manifest: str, component: str) -> str: events=PACKAGE_EVENTS, job="agent-package-macos", runner=MACOS, run="just package-rehearsal", timeout=60, gate=False, **AGENT_PACKAGE_FLAKE), # Windows is on the Agent feed, so its own packaging inputs select it too. - Lane("agent-package-windows", AGENT_WINDOWS_PACKAGING, labels=("ci:package", "ci:full"), events=PR_SIDE, + Lane("agent-package-windows", AGENT_WINDOWS_PACKAGING, labels=("ci:package", "ci:full"), events=PACKAGE_EVENTS, job="agent-package-windows", runner=GITHUB_WINDOWS, workdir="apps/maple-agent", run=WINDOWS_PACKAGE, env=(("RUSTFLAGS", "-D warnings -C target-feature=+crt-static"),) + FAST_RELEASE, timeout=90, gate=False), # Every root flake check: actionlint, workflow policy and contracts, CI script diff --git a/scripts/ci/plan.py b/scripts/ci/plan.py index 97d459e3e..ce9d1110c 100644 --- a/scripts/ci/plan.py +++ b/scripts/ci/plan.py @@ -29,7 +29,7 @@ from typing import Dict, Iterable, List, Optional, Sequence from lanes import (AGENT_WINDOWS_RUST, CI_INFRA, DISPATCH, INERT, LANES, MATRIX_JOBS, NIGHTLY, NOT_YET_TRACKED, - OPENSECRET, OPENSECRET_HOME, PR, PR_EVENTS, PUSH, QUEUE, SELECTORS, Lane) + OPENSECRET, OPENSECRET_HOME, PR, PR_EVENTS, PUSH, QUEUE, SELECTORS, Lane, cache_store) ROOT = Path(__file__).resolve().parents[2] CI_WORKFLOW = ROOT / ".github/workflows/ci.yml" @@ -209,17 +209,25 @@ def cache_key(lane: Lane) -> str: return digest.hexdigest()[:16] +def runner_label(lane: Lane) -> str: + """The lane's runner label, or the label in its override variable.""" + return os.environ.get(lane.runner.variable or "") or lane.runner.label + + def outputs(selected: Selection) -> Dict[str, str]: """`lanes` maps every active lane to whether it runs; each matrix job gets its entries.""" result = {"lanes": {lane.id: lane.id in selected for lane in LANES}} for job in MATRIX_JOBS: result[job] = [{ "id": lane.id, - "runner": os.environ.get(lane.runner.variable or "") or lane.runner.label, + "runner": runner_label(lane), "timeout": lane.timeout, "flake": lane.flake or "", "cache": lane.cache, "cache_key": cache_key(lane) if lane.cache else "", + # Master and nightly runs may save the cache, unless a failover runner + # would write it into another vendor's store. + "cache_save": cache_store(runner_label(lane)) == cache_store(lane.runner.label), "workspaces": lane.workspaces, "submodules": "recursive" if lane.submodules else "false", "fetch_depth": 0 if lane.submodules else 1, @@ -432,6 +440,10 @@ def lint_problems(files: Sequence[str]) -> List[str]: problems.append(f"ci.yml {lane.job}: runs-on is not {runner}") if not re.search(rf"^ timeout-minutes: {lane.timeout}\b", blocks[lane.job], re.M): problems.append(f"ci.yml {lane.job}: timeout-minutes is not {lane.timeout}") + # The planner resolves matrix runners from the override variables the plan step passes. + for variable in sorted({lane.runner.variable for lane in LANES if lane.job in MATRIX_JOBS} - {None}): + if not re.search(rf"^ +{variable}: \$\{{\{{ vars\.{variable} \}}\}}$", blocks.get("plan", ""), re.M): + problems.append(f"ci.yml plan: {variable} is not passed to plan.py") if f"toolchain: {AGENT_WINDOWS_RUST}\n" not in WINDOWS_SETUP.read_text(): problems.append(f"setup-agent-windows does not pin Rust {AGENT_WINDOWS_RUST}") declared = set(re.findall(r"^\s+- (\S+)$", (ROOT / ".github/actionlint.yaml").read_text(), re.M)) diff --git a/scripts/ci/test_contracts.py b/scripts/ci/test_contracts.py index 64585dd98..ab70f204b 100644 --- a/scripts/ci/test_contracts.py +++ b/scripts/ci/test_contracts.py @@ -46,9 +46,9 @@ def module(name: str, path: str): # The unsigned Agent builders that write the exact-key release cache; no ci.yml lane reads it. AGENT_RELEASE_WRITERS = frozenset(("agent-desktop-build.yml", job) for job in ("macos-build", "windows-build", "linux")) AGENT_RELEASE_READERS: frozenset = frozenset() -# The Linux and macOS packaging rehearsals build thin LTO into their own cache, saved -# only by ci.yml's nightly master run. -AGENT_PACKAGE_JOBS = frozenset(("ci.yml", f"agent-package-{os}") for os in ("linux", "macos")) +# The packaging rehearsals build thin LTO into their own cache, saved only by +# ci.yml's nightly master run. +AGENT_PACKAGE_JOBS = frozenset(("ci.yml", f"agent-package-{os}") for os in ("linux", "macos", "windows")) # Jobs that may mint an OIDC token: attestation, signing identity, FlakeHub, registries. OIDC_JOBS = frozenset({ ("agent-desktop-build.yml", "macos"), ("agent-desktop-build.yml", "windows"), @@ -391,6 +391,7 @@ def test_c4_ci_restores_only_what_a_master_workflow_saves(self): self.assertIn("refs/heads/master' && github.event_name == 'schedule'", step["if"]) else: self.assertTrue(step["with"]["key"].startswith("ci-v1-agent-package-${{ runner.os }}-")) + self.assertEqual(step["with"]["restore-keys"], "ci-v1-agent-package-${{ runner.os }}-") elif name == "ci.yml": with self.subTest(job=job_id): self.assertTrue(step["uses"].startswith("actions/cache/restore@")) @@ -404,7 +405,12 @@ def test_c4_agent_release_cache_is_exact_and_written_only_by_master_builders(sel AGENT_RELEASE_WRITERS | AGENT_RELEASE_READERS) self.assertEqual(len({(cache["key"], cache["path"]) for _, _, cache in restores}), 1) self.assertFalse([cache for _, _, cache in restores if "restore-keys" in cache]) # exact keys only - writers = [(name, job_id, step) for name, job_id, step in cache_steps() if cache_writer(step) + # No other job falls back into the family by prefix. + fallbacks = {(name, job_id) for name, job_id, step in cache_steps() + for line in str(step["with"].get("restore-keys", "")).splitlines() + if line.strip().startswith("ci-v1-agent-release-")} + self.assertEqual(fallbacks, set()) + writers =[(name, job_id, step) for name, job_id, step in cache_steps() if cache_writer(step) and (name, job_id) in AGENT_RELEASE_WRITERS | AGENT_RELEASE_READERS] self.assertEqual({(name, job_id) for name, job_id, _ in writers}, AGENT_RELEASE_WRITERS) for _, job_id, step in writers: diff --git a/scripts/ci/test_plan.py b/scripts/ci/test_plan.py index 33f08928a..c1fec31f3 100644 --- a/scripts/ci/test_plan.py +++ b/scripts/ci/test_plan.py @@ -214,6 +214,14 @@ def test_ci_infrastructure_selects_every_active_gate_lane(self): def test_every_tracked_file_is_classified_and_ci_yml_matches(self): self.assertEqual(plan.lint_problems(plan.tracked_files()), []) self.assertIn("unclassified: newroot/x", plan.lint_problems(["newroot/x"])) + # Without the plan step's variable, the matrix would ignore the override. + with tempfile.TemporaryDirectory() as tmp: + workflow = Path(tmp) / "ci.yml" + workflow.write_text(plan.CI_WORKFLOW.read_text().replace( + "CI_WINDOWS_GITHUB_RUNNER: ${{ vars.CI_WINDOWS_GITHUB_RUNNER }}\n", "")) + with mock.patch.object(plan, "CI_WORKFLOW", workflow): + self.assertIn("ci.yml plan: CI_WINDOWS_GITHUB_RUNNER is not passed to plan.py", + plan.lint_problems(plan.tracked_files())) class HookTests(unittest.TestCase): @@ -394,7 +402,8 @@ def test_full_suites_run_on_master_nightly_and_ci_full_only(self): self.assertLessEqual(full, selected(["docs/x.md"], labels=["ci:full"])) self.assertLessEqual(full | {"agent-windows"}, active(NIGHTLY)) self.assertFalse({"agent-provider"} & active(NIGHTLY)) # nightly.yml runs it on three OSes - self.assertLessEqual(AGENT_PACKAGE, active(NIGHTLY)) # refreshes the rehearsal's own cache + # Nightly refreshes the rehearsals' own caches. + self.assertLessEqual(AGENT_PACKAGE | {"agent-package-windows"}, active(NIGHTLY)) self.assertFalse(full & selected([".github/workflows/ci.yml"])) # CI edits prove the gate lanes only def test_agent_desktop_selector_covers_the_signed_builds_inputs(self): @@ -607,6 +616,30 @@ def test_outputs_empty_matrix_and_runner_overrides(self): entries = [entry for job in lanes.MATRIX_JOBS for entry in json.loads(result[job])] self.assertEqual({len(entry["cache_key"]) for entry in entries if entry["cache"]}, {16}) + def test_a_cache_is_saved_only_into_the_home_runners_store(self): + def saves(**overrides): + with mock.patch.dict(os.environ, overrides): + result = plan.outputs(dict.fromkeys(ACTIVE, ["x"])) + return {entry["id"]: entry["cache_save"] for job in lanes.MATRIX_JOBS for entry in json.loads(result[job])} + + cached = {"agent-test", "agent-windows", "agent-full-windows"} + self.assertTrue(cached <= {lane.id for lane in lanes.LANES if lane.cache}) + with mock.patch.dict(os.environ): + for runner in lanes.LINUX, lanes.LINUX_HEAVY, lanes.MACOS, lanes.WINDOWS, lanes.GITHUB_WINDOWS: + os.environ.pop(runner.variable, None) + self.assertTrue(all(saves().values())) + # Another Blacksmith size, or a larger GitHub runner, keeps its vendor's store. + on = saves(CI_LINUX_HEAVY_RUNNER="blacksmith-16vcpu-ubuntu-2404", + CI_WINDOWS_GITHUB_RUNNER="maple-windows-2025-16core") + self.assertEqual({lane: on[lane] for lane in cached}, dict.fromkeys(cached, True)) + # A failover to the other vendor restores what it can and saves nothing. + off = saves(CI_LINUX_HEAVY_RUNNER="ubuntu-latest", CI_WINDOWS_RUNNER="windows-2025", + CI_WINDOWS_GITHUB_RUNNER="blacksmith-8vcpu-windows-2025") + self.assertEqual({lane: off[lane] for lane in cached}, dict.fromkeys(cached, False)) + self.assertEqual(lanes.GITHUB_WINDOWS.variable, "CI_WINDOWS_GITHUB_RUNNER") + self.assertEqual({lane.id for lane in lanes.LANES if lane.runner == lanes.GITHUB_WINDOWS}, + {"agent-windows-updater", "agent-full-windows", "agent-package-windows"}) + def test_run_executes_the_lane_command_in_its_flake(self): with mock.patch.object(plan.subprocess, "run") as run, contextlib.redirect_stdout(io.StringIO()), \ contextlib.redirect_stderr(io.StringIO()): diff --git a/scripts/ci/test_windows_updater_workflow.py b/scripts/ci/test_windows_updater_workflow.py index f80a51366..10ff49aaa 100644 --- a/scripts/ci/test_windows_updater_workflow.py +++ b/scripts/ci/test_windows_updater_workflow.py @@ -56,7 +56,7 @@ def test_native_tests_have_no_release_or_signing_authority(self): self.assertIn("windows", LANE.runner.label) self.assertLessEqual(LANE.timeout, 60) - def test_native_toolchain_matches_the_windows_pin_and_avoids_artifact_caches(self): + def test_native_toolchain_matches_the_windows_pin_and_compiles_the_agent_from_source(self): steps = job()["steps"] self.assertIn("./.github/actions/setup-agent-windows", [step.get("uses") for step in steps]) setup = load(".github/actions/setup-agent-windows/action.yml")["runs"]["steps"] @@ -69,9 +69,16 @@ def test_native_toolchain_matches_the_windows_pin_and_avoids_artifact_caches(sel commands = "\n".join(step.get("run", "") for step in setup) for needed in ("OPENSSL_SRC_PERL", "MSWin32", "agent_windows.py"): self.assertIn(needed, commands) - # Compiled from source: no build cache or other run's artifact is restored. + # No other run's artifact is restored, and the only cache is the lane's own + # dependency cache, which master saves: rust-cache drops the workspace's own + # crates before saving, so the Agent code under test always compiles from source. actions = [step["uses"] for step in steps if "uses" in step] - self.assertFalse([action for action in actions if "cache" in action or "download-artifact" in action]) + self.assertFalse([action for action in actions if "download-artifact" in action]) + caches = [step for step in steps if "cache" in step.get("uses", "")] + self.assertEqual([step["uses"].split("@")[0] for step in caches], ["Swatinem/rust-cache"]) + self.assertTrue(caches[0]["with"]["shared-key"].startswith("agent-windows-updater-")) + self.assertNotIn("cache-workspace-crates", caches[0]["with"]) + self.assertIn("github.ref == 'refs/heads/master'", caches[0]["with"]["save-if"]) self.assertFalse(LANE.cache or LANE.restore) self.assertEqual(dict(LANE.env).get("RUSTFLAGS"), "-D warnings") diff --git a/scripts/ci/test_workflow_policy.py b/scripts/ci/test_workflow_policy.py index 40857b9a9..a58bc6357 100644 --- a/scripts/ci/test_workflow_policy.py +++ b/scripts/ci/test_workflow_policy.py @@ -28,7 +28,8 @@ VERIFY_NIX = "bash scripts/ci/verify-nix-runtime.sh" SETUP_NIX = "./.github/actions/setup-nix" PINNED = re.compile(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+(?:/[A-Za-z0-9_./-]+)?@[0-9a-f]{40}") -GITHUB_HOSTED = re.compile(r"(?:ubuntu|macos|windows)-[A-Za-z0-9.-]+") +# Standard images, and the org's GitHub-hosted larger runners (named maple-*). +GITHUB_HOSTED = re.compile(r"(?:ubuntu|macos|windows)-[A-Za-z0-9.-]+|maple-(?:ubuntu|macos|windows)-[A-Za-z0-9.-]+") CLOUDFLARE = frozenset({"CLOUDFLARE_ACCOUNT_ID", "CLOUDFLARE_API_TOKEN"}) # Every protected environment and the secrets configured in it. ENVIRONMENTS = { @@ -64,6 +65,20 @@ # credentials, so its Nix store goes to the run's own branch-scoped Actions cache, # which master and other pull requests cannot read. BRANCH_SCOPED_NIX = frozenset({("opensecret-eif.yml", "eif")}) +# Jobs on GitHub-hosted runners that save caches into GitHub's store; every other +# cache lives in Blacksmith's. The release and the Nix installer also use GitHub's. +GITHUB_STORE_WRITERS = frozenset({ + ("desktop-build.yml", "build-windows"), # the signed Research Windows build + ("agent-desktop-build.yml", "windows-build"), # the unsigned Agent Windows build (exact key) + # The Agent lanes on GitHub's Windows image (scripts/ci/lanes.py GITHUB_WINDOWS): + # agent-full-windows in the windows matrix, the updater tests and the packaging rehearsal. + ("ci.yml", "windows"), ("ci.yml", "agent-windows-updater"), ("ci.yml", "agent-package-windows"), +}) | BRANCH_SCOPED_NIX +# The only cached matrix lanes whose home runner uses GitHub's store. +GITHUB_STORE_LANES = frozenset({"agent-full-windows"}) +# A special job whose runner a repository variable overrides saves only on its home +# runner's vendor; Blacksmith runner names start with "blacksmith". +ON_BLACKSMITH = "startsWith(runner.name, 'blacksmith')" @functools.lru_cache(maxsize=None) def load(path: Path) -> dict: @@ -256,19 +271,36 @@ def test_pull_requests_never_save_caches_or_read_other_runs_artifacts(self): self.assertNotIn("id-token", json.dumps(job)) elif pr and cache_writer(step): self.assertRegex(guard, r"refs/heads/master|github\.event_name == 'push'", (name, job_id)) + if name == "ci.yml": # master's pushes and nightly save; dispatch and queue runs only restore + events = set(re.findall(r"github\.event_name == '(\w+)'", guard)) + self.assertTrue(events and events <= {"push", "schedule"}, (job_id, guard)) self.assertEqual(scoped, BRANCH_SCOPED_NIX) # the exemption lapses with its job - def test_only_windows_signing_and_the_eif_check_fill_githubs_cache_store(self): - # GitHub's cache store is small; heavy caches live on Blacksmith, whose - # runners cannot read GitHub's. The release and the Nix installer are the other users. + def test_only_listed_jobs_fill_githubs_cache_store_and_failovers_never_save(self): + # Blacksmith runners cannot read GitHub's store, nor GitHub's runners Blacksmith's. writers = set() for name, job_id, job in jobs(): for step in job.get("steps", []): - if (name not in SACRED and cache_writer(step) - and "startsWith(matrix.runner, 'blacksmith-')" not in str(step["with"].get("save-if")) - and any(GITHUB_HOSTED.fullmatch(label) for label in runner_labels(name, job_id, job))): + if name in SACRED or not cache_writer(step): + continue + guard = " ".join(str(value) for value in (step.get("if", ""), step["with"].get("save-if", ""))) + if name == "ci.yml" and job_id in lanes.MATRIX_JOBS: + # The planner's cache_save is false when a failover leaves the lane's home vendor. + self.assertIn("matrix.cache_save", guard, job_id) + labels = {lane.runner.label for lane in lanes.LANES if lane.job == job_id and lane.cache} + else: + labels = runner_labels(name, job_id, job) + if "vars." in str(job.get("runs-on")): # the default label is the home runner + with self.subTest(workflow=name, job=job_id): + self.assertIn(ON_BLACKSMITH, guard) + self.assertEqual("!" + ON_BLACKSMITH in guard, + {lanes.cache_store(label) for label in labels} == {"github"}) + if any(GITHUB_HOSTED.fullmatch(label) for label in labels): writers.add((name, job_id)) - self.assertEqual(writers, {("desktop-build.yml", "build-windows")} | BRANCH_SCOPED_NIX) + self.assertEqual(writers, GITHUB_STORE_WRITERS) + # Within a matrix writer, pin the lanes themselves. + self.assertEqual({lane.id for lane in lanes.LANES if lane.job in lanes.MATRIX_JOBS and lane.cache + and lanes.cache_store(lane.runner.label) == "github"}, GITHUB_STORE_LANES) if __name__ == "__main__": unittest.main() diff --git a/scripts/ci/windows_signature_reference.py b/scripts/ci/windows_signature_reference.py index 3ea89dcef..dfecac894 100644 --- a/scripts/ci/windows_signature_reference.py +++ b/scripts/ci/windows_signature_reference.py @@ -19,7 +19,7 @@ "Verify downloaded Windows (prod)") # The runner each job may report. Signing and verification run on GitHub's # image; the unsigned build runs on Blacksmith's (on GitHub's before CI moved it). -RUNNERS = ({"windows-2025", "blacksmith-8vcpu-windows-2025"}, {"windows-2025"}, {"windows-2025"}) +RUNNERS = ({"windows-2025", "blacksmith-8vcpu-windows-2025", "maple-windows-2025-16core"}, {"windows-2025"}, {"windows-2025"}) MAX_ZIP = 64 * 1024 * 1024