From 50f1b759f72fe816f5a24e663286cbb3b122e07b Mon Sep 17 00:00:00 2001 From: pasta Date: Thu, 29 Oct 2026 19:27:03 +0000 Subject: [PATCH 1/2] ci: pick a self-hosted runner for a linux64 fast path Adds a runner_fast output to the dynamic runner selector. When RUNNER_SELFHOSTED_FAST names a label, it uses the pre-lint rung's ladder: hosted backlog present, no blacksmith-ci override, and the label not saturated (at most 2 queued jobs, as these hold a runner for minutes, not seconds). Trust is the pre-lint allowlist plus every push to develop or master, which carry merged code and need write access. Otherwise it falls back to the amd64 runner, so nothing changes while the variable is unset. The pre-lint selection is refactored to take a precomputed "allowed" flag so both rungs share it; its decisions and reason strings are unchanged. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/select_dynamic_runner.py | 115 +++++++++++++----- .../workflows/test_select_dynamic_runner.py | 106 ++++++++++++++++ 2 files changed, 193 insertions(+), 28 deletions(-) diff --git a/.github/workflows/select_dynamic_runner.py b/.github/workflows/select_dynamic_runner.py index f6da231efa94..13f1d34323e2 100644 --- a/.github/workflows/select_dynamic_runner.py +++ b/.github/workflows/select_dynamic_runner.py @@ -31,6 +31,12 @@ # Above this many queued jobs for a self-hosted label, our own runners are the # bottleneck and the pre-lint jobs go back to the hosted ladder. DEFAULT_SELFHOSTED_MAX_QUEUED = 6 +# The fast path's build and test jobs hold a runner for 5-20 minutes rather +# than seconds, so a much shorter wait for one already loses to hosted. +DEFAULT_SELFHOSTED_FAST_MAX_QUEUED = 2 +# Pushes to these branches only ever carry merged code, so they may use the +# fast path whoever pushed them. +FAST_PATH_PUSH_REFS = ("refs/heads/develop", "refs/heads/master") def parse_next_link(link_header: str) -> Optional[str]: @@ -234,6 +240,22 @@ def is_selfhosted_allowed( return False +def is_fast_path_allowed( + event_name: str, + event: Dict, + actor: str, + allowed_authors: Set[str], +) -> bool: + """Whether this event may build and test on the self-hosted fast path. + + The pre-lint rung's trust, plus pushes to develop and master by anyone: + pushing there already requires write access, and they carry merged code. + """ + if event_name == "push" and event.get("ref") in FAST_PATH_PUSH_REFS: + return True + return is_selfhosted_allowed(event_name, event, actor, allowed_authors) + + def allows_cache_write( event_name: str, event: Dict, @@ -252,35 +274,26 @@ def allows_cache_write( ) -def select_prelint_runner( - event_name: str, - event: Dict, - actor: str, +def select_selfhosted_runner( + allowed: bool, backlog_count_value: Optional[int], measurement_error: Optional[str], selfhosted_label: str, - allowed_authors: Set[str], fallback_runner: str, label_override: bool = False, selfhosted_queued: int = 0, selfhosted_max_queued: int = DEFAULT_SELFHOSTED_MAX_QUEUED, disabled_reason: str = "selfhosted-disabled", ) -> Tuple[str, str]: - """Pick the runner for the pre-lint jobs of one arch, and the reason. - - The pre-lint jobs are the short ones that gate lint and the long builds: - lint itself, cache-sources and the container builds and manifest. They - take seconds but can wait most of an hour for a GitHub-hosted runner. + """Pick between a self-hosted label and fallback_runner, and the reason. Inserts a self-hosted rung into the hosted -> Blacksmith ladder: once any - GitHub-hosted job is queued, these jobs go to our own hardware instead of + GitHub-hosted job is queued, the jobs go to our own hardware instead of competing for the account-wide concurrency limit, unless our own runners already have more than selfhosted_max_queued jobs waiting. When the rung is not available they keep the caller's existing decision for the arch. - Every job but the manifest runs pull request head code (lint scripts, - `make -C depends download`, Dockerfile RUN steps), so all of them go - through the same allowlist as lint did on its own. + allowed says whether this event may run on our own hardware at all. """ if not selfhosted_label: return fallback_runner, disabled_reason @@ -298,7 +311,7 @@ def select_prelint_runner( backlog_count_value, SELFHOSTED_BACKLOG_THRESHOLD ) - if not is_selfhosted_allowed(event_name, event, actor, allowed_authors): + if not allowed: return fallback_runner, "actor-not-allowed" if selfhosted_queued > selfhosted_max_queued: @@ -326,6 +339,8 @@ def select_runners( actor: str = "", runner_selfhosted_arm64_var: str = "", selfhosted_max_queued: int = DEFAULT_SELFHOSTED_MAX_QUEUED, + runner_selfhosted_fast_var: str = "", + selfhosted_fast_max_queued: int = DEFAULT_SELFHOSTED_FAST_MAX_QUEUED, ) -> Dict[str, str]: label_names = [ label.get("name", "") @@ -344,7 +359,11 @@ def select_runners( backlog_count_value, selfhosted_queued = count_queued_jobs( fetch_json, repos, - non_hosted_labels=(runner_selfhosted_var, runner_selfhosted_arm64_var), + non_hosted_labels=( + runner_selfhosted_var, + runner_selfhosted_arm64_var, + runner_selfhosted_fast_var, + ), ) backlog_count = str(backlog_count_value) except Exception as exc: # noqa: BLE001 @@ -401,29 +420,52 @@ def select_runners( allowed_authors = parse_author_allowlist(selfhosted_authors) - def prelint( - selfhosted_label: str, fallback_runner: str, disabled_reason: str + def selfhosted( + allowed: bool, + selfhosted_label: str, + fallback_runner: str, + max_queued: int, + disabled_reason: str, ) -> Tuple[str, str]: - return select_prelint_runner( - event_name=event_name, - event=event, - actor=actor, + return select_selfhosted_runner( + allowed=allowed, backlog_count_value=backlog_count_value, measurement_error=measurement_error, selfhosted_label=selfhosted_label, - allowed_authors=allowed_authors, fallback_runner=fallback_runner, label_override=label_override, selfhosted_queued=selfhosted_queued.get(selfhosted_label.lower(), 0), - selfhosted_max_queued=selfhosted_max_queued, + selfhosted_max_queued=max_queued, disabled_reason=disabled_reason, ) - runner_prelint, prelint_decision_reason = prelint( - runner_selfhosted_var, runner_amd64, "selfhosted-disabled" + # The pre-lint jobs (lint, cache-sources, the container builds and + # manifest) take seconds but can wait most of an hour for a hosted runner. + # Every one but the manifest runs pull request head code (lint scripts, + # `make -C depends download`, Dockerfile RUN steps), so all of them go + # through the same allowlist as lint did on its own. + prelint_allowed = is_selfhosted_allowed(event_name, event, actor, allowed_authors) + runner_prelint, prelint_decision_reason = selfhosted( + allowed=prelint_allowed, + selfhosted_label=runner_selfhosted_var, + fallback_runner=runner_amd64, + max_queued=selfhosted_max_queued, + disabled_reason="selfhosted-disabled", + ) + runner_prelint_arm64, prelint_arm64_decision_reason = selfhosted( + allowed=prelint_allowed, + selfhosted_label=runner_selfhosted_arm64_var, + fallback_runner=runner_arm64, + max_queued=selfhosted_max_queued, + disabled_reason="selfhosted-arm64-disabled", ) - runner_prelint_arm64, prelint_arm64_decision_reason = prelint( - runner_selfhosted_arm64_var, runner_arm64, "selfhosted-arm64-disabled" + # The fast path: the linux64 depends, build and functional test jobs. + runner_fast, fast_decision_reason = selfhosted( + allowed=is_fast_path_allowed(event_name, event, actor, allowed_authors), + selfhosted_label=runner_selfhosted_fast_var, + fallback_runner=runner_amd64, + max_queued=selfhosted_fast_max_queued, + disabled_reason="selfhosted-fast-disabled", ) cache_write = allows_cache_write(event_name, event, actor, allowed_authors) @@ -434,6 +476,8 @@ def prelint( "prelint_decision_reason": prelint_decision_reason, "runner_prelint_arm64": runner_prelint_arm64, "prelint_arm64_decision_reason": prelint_arm64_decision_reason, + "runner_fast": runner_fast, + "fast_decision_reason": fast_decision_reason, "use_blacksmith": "true" if use_blacksmith_amd64 or use_blacksmith_arm64 else "false", "use_blacksmith_amd64": "true" if use_blacksmith_amd64 else "false", "use_blacksmith_arm64": "true" if use_blacksmith_arm64 else "false", @@ -481,10 +525,12 @@ def write_step_summary(path: Optional[str], outputs: Dict[str, str]) -> None: fh.write("- arm64 runner: `{}`\n".format(outputs["runner_arm64"])) fh.write("- pre-lint amd64 runner: `{}`\n".format(outputs["runner_prelint"])) fh.write("- pre-lint arm64 runner: `{}`\n".format(outputs["runner_prelint_arm64"])) + fh.write("- linux64 fast path runner: `{}`\n".format(outputs["runner_fast"])) fh.write("- Decision: `{}`\n".format(outputs["decision_reason"])) fh.write("- Pre-lint amd64 decision: `{}`\n".format(outputs["prelint_decision_reason"])) fh.write("- Pre-lint arm64 decision: `{}`\n".format( outputs["prelint_arm64_decision_reason"])) + fh.write("- Fast path decision: `{}`\n".format(outputs["fast_decision_reason"])) fh.write( "- Remote ccache writes: {}\n".format( "yes" if outputs["cache_write"] == "true" else "no" @@ -550,6 +596,17 @@ def parse_args(argv: Sequence[str]) -> argparse.Namespace: default=env_int("SELFHOSTED_MAX_QUEUED", DEFAULT_SELFHOSTED_MAX_QUEUED), help="Use a self-hosted label only while at most this many jobs are queued for it", ) + parser.add_argument( + "--runner-selfhosted-fast-var", + default=os.environ.get("RUNNER_SELFHOSTED_FAST_VAR", ""), + help="Self-hosted amd64 runner label for the linux64 build and test fast path; empty disables it", + ) + parser.add_argument( + "--selfhosted-fast-max-queued", + type=int, + default=env_int("SELFHOSTED_FAST_MAX_QUEUED", DEFAULT_SELFHOSTED_FAST_MAX_QUEUED), + help="Use the fast path label only while at most this many jobs are queued for it", + ) parser.add_argument( "--selfhosted-authors", default=os.environ.get("SELFHOSTED_LINT_AUTHORS", ""), @@ -607,6 +664,8 @@ def main(argv: Sequence[str]) -> int: actor=args.actor, runner_selfhosted_arm64_var=args.runner_selfhosted_arm64_var, selfhosted_max_queued=args.selfhosted_max_queued, + runner_selfhosted_fast_var=args.runner_selfhosted_fast_var, + selfhosted_fast_max_queued=args.selfhosted_fast_max_queued, ) write_github_output(os.environ.get("GITHUB_OUTPUT"), outputs) diff --git a/.github/workflows/test_select_dynamic_runner.py b/.github/workflows/test_select_dynamic_runner.py index 5d71db24e6a7..b1b90239f150 100644 --- a/.github/workflows/test_select_dynamic_runner.py +++ b/.github/workflows/test_select_dynamic_runner.py @@ -523,6 +523,112 @@ def test_lint_inherits_blacksmith_when_rung_unavailable(self): self.assertEqual(outputs["runner_amd64"], "blacksmith-amd64") self.assertEqual(outputs["runner_prelint"], "blacksmith-amd64") + # --- linux64 build and test fast path ------------------------------------ + + def _select_fast(self, **overrides): + overrides.setdefault("runner_selfhosted_fast_var", "dash-x64-build") + return self._select(**overrides) + + def test_fast_path_is_off_until_its_label_is_set(self): + outputs = self._select() + + self.assertEqual(outputs["runner_fast"], MODULE.DEFAULT_RUNNER_AMD64) + self.assertEqual(outputs["fast_decision_reason"], "selfhosted-fast-disabled") + + def test_fast_path_for_trusted_pull_requests(self): + outputs = self._select_fast() + + self.assertEqual(outputs["runner_fast"], "dash-x64-build") + self.assertEqual( + outputs["fast_decision_reason"], "selfhosted:backlog:3>0;queued:0<=2" + ) + # Independent of the pre-lint rung's label. + self.assertEqual(outputs["runner_prelint"], "ubuntu-core") + + def test_fast_path_falls_back_for_untrusted_pull_requests(self): + for author, actor, head_owner in ( + ("mallory", "PastaPastaPasta", "mallory"), + ("PastaPastaPasta", "mallory", "PastaPastaPasta"), + ("PastaPastaPasta", "PastaPastaPasta", "mallory"), + ): + with self.subTest(author=author, actor=actor, head_owner=head_owner): + outputs = self._select_fast( + event=pull_request_event(author, head_owner=head_owner), + actor=actor, + ) + self.assertEqual(outputs["runner_fast"], MODULE.DEFAULT_RUNNER_AMD64) + self.assertEqual(outputs["fast_decision_reason"], "actor-not-allowed") + + def test_fast_path_for_develop_and_master_pushes_by_anyone(self): + for ref in ("refs/heads/develop", "refs/heads/master"): + with self.subTest(ref=ref): + outputs = self._select_fast( + event_name="push", event={"ref": ref}, actor="mallory" + ) + self.assertEqual(outputs["runner_fast"], "dash-x64-build") + + def test_fast_path_for_other_pushes_needs_the_allowlist(self): + for ref in ("refs/heads/feature", "refs/tags/v24.0.0", "refs/heads/developer"): + with self.subTest(ref=ref): + outputs = self._select_fast( + event_name="push", event={"ref": ref}, actor="mallory" + ) + self.assertEqual(outputs["runner_fast"], MODULE.DEFAULT_RUNNER_AMD64) + self.assertEqual(outputs["fast_decision_reason"], "actor-not-allowed") + + outputs = self._select_fast( + event_name="push", event={"ref": ref}, actor="knst" + ) + self.assertEqual(outputs["runner_fast"], "dash-x64-build") + + def test_fast_path_falls_back_without_backlog(self): + outputs = self._select_fast( + event_name="push", event={"ref": "refs/heads/develop"}, queued_jobs=0 + ) + + self.assertEqual(outputs["runner_fast"], MODULE.DEFAULT_RUNNER_AMD64) + self.assertEqual(outputs["fast_decision_reason"], "backlog:0<=0") + + def test_fast_path_falls_back_on_measurement_error(self): + def fetch_json(_url): + raise RuntimeError("boom") + + outputs = self._select_fast(fetch_json=fetch_json) + + self.assertEqual(outputs["runner_fast"], MODULE.DEFAULT_RUNNER_AMD64) + self.assertEqual(outputs["fast_decision_reason"], "metric-unavailable") + + def test_fast_path_has_its_own_saturation_cap(self): + for queued, expected in ((2, "dash-x64-build"), (3, MODULE.DEFAULT_RUNNER_AMD64)): + with self.subTest(queued=queued): + responses = add_queued_jobs(backlog_responses(3), "dash-x64-build", queued) + + outputs = self._select_fast(fetch_json=lambda url: responses[url]) + + # Jobs waiting for our own runners are not hosted backlog. + self.assertEqual(outputs["backlog_count"], "3") + self.assertEqual(outputs["runner_fast"], expected) + # The pre-lint rung keeps its own, larger cap. + self.assertEqual(outputs["runner_prelint"], "ubuntu-core") + if queued > 2: + self.assertEqual( + outputs["fast_decision_reason"], "selfhosted-saturated:queued:3>2" + ) + + def test_fast_path_inherits_the_amd64_decision_when_unavailable(self): + outputs = self._select_fast( + event=pull_request_event("PastaPastaPasta", labels=["blacksmith-ci"]), + ) + self.assertEqual(outputs["runner_fast"], "blacksmith-amd64") + self.assertEqual(outputs["fast_decision_reason"], "label:blacksmith-ci") + + outputs = self._select_fast( + queued_jobs=11, + event=pull_request_event("mallory"), + actor="mallory", + ) + self.assertEqual(outputs["runner_fast"], "blacksmith-amd64") + # --- remote ccache write gate -------------------------------------------- def test_cache_write_on_any_push(self): From 29cbd9d7be76ce1d583eba3e6727e580d14504eb Mon Sep 17 00:00:00 2001 From: pasta Date: Thu, 29 Oct 2026 19:27:03 +0000 Subject: [PATCH 2/2] ci: run linux64 build and functional tests on the fast path runner Routes the linux64 depends build job (cache misses only), linux64-build and linux64-test through check-skip's runner-fast output. These are the jobs on the critical path that wait longest for hosted runners: over the last week linux64-build waited p50 7 / p90 43 min, the depends build p90 84 min, and linux64-test p90 28 min, against 6, 17 and 16 min of work. The jobs replace their hosted runs instead of running in addition, so required check names stay the same and no hosted capacity is spent twice. build-depends.yml gains runs-on-build for the build job only; runs-on still forms the cache key, so a fast-path build saves the entry the hosted asan, fuzz and sqlite builds restore. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/build-depends.yml | 7 +++++- .github/workflows/build.yml | 13 +++++++++-- ci/README.md | 36 +++++++++++++++++++++++++++++ 3 files changed, 53 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-depends.yml b/.github/workflows/build-depends.yml index 211958857ae4..7afb5663bf23 100644 --- a/.github/workflows/build-depends.yml +++ b/.github/workflows/build-depends.yml @@ -28,6 +28,11 @@ on: required: false default: x86_64 type: string + runs-on-build: + description: "Runner label for the build job if not runs-on; runs-on still names the cache key, so either one saves the entry every consumer of this target restores" + required: false + default: "" + type: string outputs: key: description: "Key needed for restoring depends cache" @@ -103,7 +108,7 @@ jobs: name: Build depends needs: [check-cache] if: needs.check-cache.outputs.cache-hit != 'true' - runs-on: ${{ inputs.runs-on }} + runs-on: ${{ inputs.runs-on-build || inputs.runs-on }} outputs: built-artifact: depends-built-${{ inputs.build-target }} container: diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ecd017fe24fa..c0e2f0f44979 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -38,6 +38,7 @@ jobs: runner-arm64: ${{ steps.select-runner.outputs.runner_arm64 }} runner-prelint: ${{ steps.select-runner.outputs.runner_prelint }} runner-prelint-arm64: ${{ steps.select-runner.outputs.runner_prelint_arm64 }} + runner-fast: ${{ steps.select-runner.outputs.runner_fast }} use-blacksmith: ${{ steps.select-runner.outputs.use_blacksmith }} use-blacksmith-amd64: ${{ steps.select-runner.outputs.use_blacksmith_amd64 }} use-blacksmith-arm64: ${{ steps.select-runner.outputs.use_blacksmith_arm64 }} @@ -119,6 +120,13 @@ jobs: SELFHOSTED_LINT_AUTHORS: ${{ vars.SELFHOSTED_LINT_AUTHORS }} # Fall back to hosted once more jobs than this wait for a self-hosted label SELFHOSTED_MAX_QUEUED: "6" + # Fast path for the linux64 depends, build and functional test jobs, + # which replace their hosted runs rather than add to them. Same + # allowlist as the pre-lint rung, plus every push to develop or + # master. Unset RUNNER_SELFHOSTED_FAST to keep them on the hosted + # ladder; see ci/README.md. + RUNNER_SELFHOSTED_FAST_VAR: ${{ vars.RUNNER_SELFHOSTED_FAST }} + SELFHOSTED_FAST_MAX_QUEUED: "2" run: | python3 .github/workflows/select_dynamic_runner.py @@ -234,6 +242,7 @@ jobs: container-path: ${{ needs.container.outputs.path }} base-image-digest: ${{ needs.check-skip.outputs.base-image-digest }} runs-on: ${{ needs.check-skip.outputs['runner-amd64'] }} + runs-on-build: ${{ needs.check-skip.outputs['runner-fast'] }} runs-on-check: ${{ needs.check-skip.outputs['runner-check-skip'] }} depends-linux64_multiprocess: @@ -322,7 +331,7 @@ jobs: depends-host: ${{ needs.depends-linux64.outputs.host }} depends-dep-opts: ${{ needs.depends-linux64.outputs.dep-opts }} depends-artifact: ${{ needs.depends-linux64.outputs.built-artifact }} - runs-on: ${{ needs.check-skip.outputs['runner-amd64'] }} + runs-on: ${{ needs.check-skip.outputs['runner-fast'] }} secrets: *remote-ccache-auth src-linux64_asan: @@ -451,7 +460,7 @@ jobs: bundle-key: ${{ needs.src-linux64.outputs.key }} build-target: linux64 container-path: ${{ needs.container-slim.outputs.path }} - runs-on: ${{ needs.check-skip.outputs['runner-amd64'] }} + runs-on: ${{ needs.check-skip.outputs['runner-fast'] }} cancel-stale: ${{ needs.src-linux64.outputs.freshness == 'current' && needs.lint.outputs.freshness == 'current' }} test-linux64_asan: diff --git a/ci/README.md b/ci/README.md index 433440af1298..bed691dabb7e 100644 --- a/ci/README.md +++ b/ci/README.md @@ -24,3 +24,39 @@ with fallback values. In order to avoid rebuilding all dependencies for each build, the binaries are cached and reused when possible. Changes in the dependency-generator will trigger cache-invalidation and rebuilds as necessary. + +### Self-hosted fast path + +When the repository variable `RUNNER_SELFHOSTED_FAST` names a runner label, +`.github/workflows/select_dynamic_runner.py` may move three jobs of the +`linux64` target from the hosted runner ladder (GitHub-hosted or Blacksmith) +to that label: the depends build (only on a depends cache miss), +`linux64-build` and `linux64-test`. They keep their names and steps, so +required checks are unchanged, and the depends cache key still names the +hosted runner, so the entry they save is the one the other linux64 builds +restore. Every other job stays where it was. + +The decision is made once per run, by the `Check skip conditions` job. A run +takes the fast path only when all of these hold: + +- it is a push to `develop` or `master`, or an event the pre-lint rung trusts: + a push whose actor is listed in `SELFHOSTED_LINT_AUTHORS`, or a pull request + whose author and actor are listed and whose head repository is this one or + is owned by a listed login. Everything else, including fork pull requests + from other contributors, stays on the hosted ladder; +- some GitHub-hosted job is queued (otherwise hosted starts at once); +- no `blacksmith-ci` label overrides the choice; +- at most `SELFHOSTED_FAST_MAX_QUEUED` (2, set in `build.yml`) jobs are queued + for the label at that moment. Jobs already running there, or due to queue + later in runs that already chose it, are not counted, so a burst of runs + started together can still overfill a small pool. + +Leaving the variable unset turns the fast path off. Jobs already queued for +the label must be cancelled and re-run after unsetting it. + +Runners registered for the label must provide x86_64 Linux with a Docker +daemon (the jobs run in the CI container with `--cap-add SYS_PTRACE`), about 8 +CPUs and 16 GiB of memory or more, roughly 40 GiB of free disk, and access to +GitHub's cache service and the remote ccache (`CCACHE_REMOTE_URL`). These runs +execute pull request code with access to that Docker daemon, so give each job +a fresh, single-use machine.