CI: run the full e2e tier nightly, not per push (#1414)

.github/workflows/ci.yml

@@ -17,7 +17,8 @@ on:
     types: [checks_requested]
   schedule:
     # Nightly on main: the `full` tier (e2e-full, cargo-vex-matrix-full,
-    # yarn-berry-full) and e2e-docker, which pull_request runs skip.
+    # yarn-berry-full) and e2e-docker, which pull_request, merge_group and
+    # push runs skip.
     - cron: '41 5 * * *'
   workflow_dispatch:
     inputs:
@@ -68,10 +69,11 @@ concurrency:
 # MERGE-QUEUE REUSE: a push to main whose exact SHA already passed this
 # workflow in the merge queue (scripts/ci-reuse-merge-group.py, run by
 # clippy) only compiles, to refresh the main-only caches, and skips the
-# test steps and test-only jobs the queue already ran. Jobs the queue never
-# runs (test-release, e2e-full, yarn-berry-full, cargo-vex-matrix-full) and
-# hosted-e2e still run. API errors, missing evidence and direct pushes run
-# everything.
+# test steps and test-only jobs the queue already ran. hosted-e2e still
+# runs. The full tier (e2e-full, yarn-berry-full, cargo-vex-matrix-full)
+# never runs on push unless CI_SCOPE is `full`; it is nightly and on
+# dispatch. API errors, missing evidence and direct pushes run everything
+# else.
 # MERGE-MAIN REUSE: a pull_request push that only merged the base branch,
 # with the PR's own diff unchanged and the previous head green
 # (scripts/ci-reuse-pr-head.py, run by clippy), runs only clippy, the cheap
@@ -1012,14 +1014,16 @@ jobs:
           cargo test --locked -p socket-patch-cli --all-features --tests --no-run --message-format=json-render-diagnostics > target-build.json
 
       - name: Bundle the binaries the legs run
-        if: needs.clippy.outputs.reuse != 'true' || matrix.os == 'ubuntu-latest'
+        # A reused push runs no leg, except e2e-full and cargo-vex-matrix-full
+        # on ubuntu when CI_SCOPE is `full`.
+        if: needs.clippy.outputs.reuse != 'true' || (matrix.os == 'ubuntu-latest' && vars.CI_SCOPE == 'full')
         shell: bash
         env:
           BUNDLE_OS: ${{ matrix.os }}
         run: python3 scripts/ci-e2e-bundle.py --os "$BUNDLE_OS" --cargo-json target-build.json --dest target/e2e-bin
 
       - name: Compress the e2e binaries
-        if: needs.clippy.outputs.reuse != 'true' || matrix.os == 'ubuntu-latest'
+        if: needs.clippy.outputs.reuse != 'true' || (matrix.os == 'ubuntu-latest' && vars.CI_SCOPE == 'full')
         # Compress the bundle as one stream so identical Rust code across
         # test binaries shares a dictionary. Zstd is on every runner image.
         # The 128 MiB window cuts the Linux artifact from ~200 MB to ~94 MB;
@@ -1030,7 +1034,7 @@ jobs:
           tar -C target/e2e-bin -cf - . | zstd -q -f -10 --long=27 -o target/e2e-bin.tar.zst
 
       - uses: ./.github/actions/upload-artifact
-        if: needs.clippy.outputs.reuse != 'true' || matrix.os == 'ubuntu-latest'
+        if: needs.clippy.outputs.reuse != 'true' || (matrix.os == 'ubuntu-latest' && vars.CI_SCOPE == 'full')
         with:
           name: e2e-bin-${{ matrix.os }}
           path: target/e2e-bin.tar.zst
@@ -1805,10 +1809,10 @@ jobs:
 
   # The PM-version legs with no boundary of their own: the middle uv lines,
   # Pipenv releases and .NET SDK majors, and bundler 2.7. Skipped
-  # on pull_request (the `e2e` rows keep every named boundary, the oldest
-  # and newest release of each tool and every vlt era there); main pushes,
-  # v5 landings, the nightly schedule and dispatch run them with the `e2e`
-  # steps.
+  # on pull_request, merge_group and main pushes (the `e2e` rows keep every
+  # named boundary, the oldest and newest release of each tool and every vlt
+  # era there); v5 landings, the nightly schedule and dispatch run them with
+  # the `e2e` steps, and so do main pushes when CI_SCOPE is `full`.
 
   # Each OS consumes only its own build. A queued macOS runner must not
   # delay Linux or Windows, and a Windows compile must not delay Gradle.
@@ -2028,8 +2032,9 @@ jobs:
     steps: *e2e-steps
 
   e2e-full:
-    # merge_group runs the pull_request tier: the queue gates on ci-ok.
-    if: (github.event_name != 'pull_request' && github.event_name != 'merge_group') || github.head_ref == 'release/v5-prerelease'
+    # Nightly, on dispatch (release-qa) and on v5 landings only, like
+    # e2e-docker; CI_SCOPE=full also runs it on pushes to main.
+    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && vars.CI_SCOPE == 'full') || github.head_ref == 'release/v5-prerelease'
     needs: [e2e-build]
     strategy:
       fail-fast: false
@@ -2259,7 +2264,7 @@ jobs:
           YARN_BERRY_RELEASE: ${{ matrix.yarn }}
         run: scripts/yarn-berry-vex-matrix.sh "$YARN_BERRY_RELEASE"
 
-  # Skipped on pull_request (except v5 landings), like e2e-full.
+  # Nightly, on dispatch and on v5 landings only, like e2e-full.
 
   # The macOS rows of `yarn-berry-e2e`: they run on main, the merge queue and
   # nightly, not on every PR push, so PRs stop queueing on the small
@@ -2282,8 +2287,8 @@ jobs:
 
   yarn-berry-full:
     name: yarn-berry ${{ matrix.yarn }} (${{ matrix.os }})
-    # merge_group runs the pull_request tier: the queue gates on ci-ok.
-    if: (github.event_name != 'pull_request' && github.event_name != 'merge_group') || github.head_ref == 'release/v5-prerelease'
+    # Same gate as e2e-full.
+    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && vars.CI_SCOPE == 'full') || github.head_ref == 'release/v5-prerelease'
     needs: clippy
     strategy:
       fail-fast: false
@@ -2301,7 +2306,7 @@ jobs:
   # plus macOS and Windows; 1.93.1 with its own lock (on every OS) is the
   # pinned rust-toolchain.toml cell `cargo test` used to give
   # e2e_safety_cargo_build. cargo-vex-matrix-full runs the rest of the
-  # toolchain x lock cross off pull_request. Each leg also runs
+  # toolchain x lock cross nightly and on dispatch. Each leg also runs
   # e2e_safety_cargo_build (its headline test honours the knobs).
   cargo-vex-matrix:
     if: needs.e2e-build.outputs.reuse != 'true'
@@ -2422,8 +2427,8 @@ jobs:
 
   cargo-vex-matrix-full:
     name: cargo ${{ matrix.toolchain }} lock-v${{ matrix.lock || 'own' }} (${{ matrix.os }})
-    # merge_group runs the pull_request tier: the queue gates on ci-ok.
-    if: (github.event_name != 'pull_request' && github.event_name != 'merge_group') || github.head_ref == 'release/v5-prerelease'
+    # Same gate as e2e-full.
+    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && vars.CI_SCOPE == 'full') || github.head_ref == 'release/v5-prerelease'
     needs: [e2e-build]
     runs-on: ${{ vars.DISABLE_DEPOT_RUNNERS != 'true' && (matrix.os == 'ubuntu-latest' && 'depot-ubuntu-24.04-4' || matrix.os == 'ubuntu-22.04' && 'depot-ubuntu-22.04-4') || matrix.os }}
     timeout-minutes: 40

docs/testing/yarn-berry-compatibility.md

@@ -91,8 +91,9 @@ The `yarn-berry-e2e` job in `.github/workflows/ci.yml` runs
 | macos-latest | 4.12.0 |
 | windows-latest | 4.12.0 |
 
-Pull requests skip ubuntu 4.6.0 and 4.12.0 (the `yarn-berry-full` job, which
-runs on main pushes, nightly and dispatch).
+Pull requests, the merge queue and main pushes skip ubuntu 4.6.0 and 4.12.0
+(the `yarn-berry-full` job, which runs nightly, on dispatch and on the v5
+integration PR, and on main pushes only when `CI_SCOPE` is `full`).
 
 Each release drives four real-yarn suites — `e2e_redirect_yarn_berry_build`,
 `e2e_vendor_yarn_berry_build`, `e2e_yarn4_pnpm_linker_build` and

scripts/ci-reuse-merge-group.py

@@ -47,7 +47,7 @@ def main():
     with Path(os.environ["GITHUB_OUTPUT"]).open("a", encoding="utf-8") as output:
         output.write(f"reuse={'true' if reuse else 'false'}\n")
     if reuse:
-        print(f"Merge-queue CI run {run} passed this SHA; retaining cache builds and the full tier.")
+        print(f"Merge-queue CI run {run} passed this SHA; retaining cache builds only.")
     else:
         print("No verified merge-queue CI result for this push; running all checks.")
 

scripts/tests/test_ci_e2e_tiers.py

@@ -59,15 +59,17 @@ def test_full_rows_only_add_releases_to_pr_suites(self):
         self.assertFalse(set(pr_rows) & set(full_rows), "a row in both tiers runs twice")
         self.assertEqual(len(pr_rows + full_rows), len(set(pr_rows + full_rows)), "duplicate rows")
 
-    def test_full_jobs_skip_pull_requests_and_share_steps(self):
+    def test_full_jobs_run_nightly_only_and_share_steps(self):
         for job, anchor in (("e2e-full", "e2e-steps"), ("yarn-berry-full", "yarn-berry-steps"),
                             ("cargo-vex-matrix-full", "cargo-vex-steps")):
             with self.subTest(job=job):
                 text = job_text(job)
-                # The merge queue runs the pull_request tier, so the full tier
-                # skips merge_group too.
-                self.assertIn("if: (github.event_name != 'pull_request' && github.event_name != 'merge_group')",
-                              text)
+                # Nightly, dispatch (release-qa) and v5 landings, like
+                # e2e-docker: not on PRs, the merge queue or a lean push to
+                # main, where its 41 legs ran on every landing.
+                self.assertIn("    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'"
+                              " || (github.event_name == 'push' && vars.CI_SCOPE == 'full')"
+                              " || github.head_ref == 'release/v5-prerelease'\n", text)
                 self.assertIn(f"steps: *{anchor}", text)
                 self.assertIn(f"steps: &{anchor}", TEXT)
 

scripts/tests/test_ci_scheduling.py

@@ -111,13 +111,21 @@ def test_lean_queue_type_checks_windows_and_full_scope_builds_it(self):
 
     def test_reused_push_keeps_cache_writers_and_unqueued_jobs(self):
         # A push whose SHA passed the merge queue still compiles (main is the
-        # only cache writer) and still runs what the queue never ran.
+        # only cache writer) and still runs what the queue never ran. The
+        # full tier runs on a push only when CI_SCOPE is full, so only then
+        # does e2e-build still ship the ubuntu binaries it downloads.
         for job in ("clippy", "node-addon", "test", "test-release", "coverage", "e2e-build",
                     "e2e-build-windows", "e2e-build-macos", "cargo-old-toolchains",
                     "e2e-full", "cargo-vex-matrix-full", "yarn-berry-full", "hosted-e2e"):
             with self.subTest(job=job):
                 condition = next((line for line in JOBS[job] if line.startswith("    if:")), "")
                 self.assertNotIn("outputs.reuse", condition)
+        ship = [body for name, body in reader.steps(JOBS["e2e-build"])
+                if name.startswith(("Bundle ", "Compress ")) or "upload-artifact" in body]
+        self.assertEqual(len(ship), 3)
+        for body in ship:
+            self.assertIn("if: needs.clippy.outputs.reuse != 'true'"
+                          " || (matrix.os == 'ubuntu-latest' && vars.CI_SCOPE == 'full')", body)
         for job in ("docker-base", "yarn-classic-matrix", "yarn-berry-e2e", "yarn-berry-e2e-macos"):
             with self.subTest(job=job):
                 condition = next(line for line in JOBS[job] if line.startswith("    if:"))
@@ -152,8 +160,10 @@ def test_unchanged_pr_merge_skips_only_what_the_queue_re_tests(self):
                  "e2e-build-windows", "yarn-classic-matrix", "yarn-berry-e2e",
                  "cargo-old-toolchains"}
         never_on_pr = "    if: (github.event_name != 'pull_request'"
-        v5_pr_only = ("    if: (github.event_name != 'pull_request' && github.event_name != 'merge_group')"
-                      " || github.head_ref == 'release/v5-prerelease'")
+        # The full tier runs on a PR only as the v5 integration PR.
+        full_tier = ("    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'"
+                     " || (github.event_name == 'push' && vars.CI_SCOPE == 'full')"
+                     " || github.head_ref == 'release/v5-prerelease'")
         for job in JOBS:
             condition = next((line for line in JOBS[job] if line.startswith("    if:")), "")
             with self.subTest(job=job):
@@ -167,8 +177,8 @@ def test_unchanged_pr_merge_skips_only_what_the_queue_re_tests(self):
             with self.subTest(job=job):
                 # Ungated jobs wait on a gated build, never run on a PR, or run
                 # only on the v5 integration PR, which never reuses (below).
-                self.assertTrue(ancestors(job) & gated or condition.startswith(never_on_pr)
-                                and (condition == v5_pr_only or "head_ref" not in condition), job)
+                self.assertTrue(ancestors(job) & gated or condition == full_tier
+                                or condition.startswith(never_on_pr) and "head_ref" not in condition, job)
         clippy = "\n".join(JOBS["clippy"])
         self.assertIn("pr_unchanged: ${{ steps.pr-head.outputs.unchanged }}", clippy)
         self.assertIn("python3 scripts/ci-reuse-pr-head.py", clippy)