diff --git a/.github/workflows/build-depends.yml b/.github/workflows/build-depends.yml index 211958857ae4..7afb5663bf23 100644 --- a/.github/workflows/build-depends.yml +++ b/.github/workflows/build-depends.yml @@ -28,6 +28,11 @@ on: required: false default: x86_64 type: string + runs-on-build: + description: "Runner label for the build job if not runs-on; runs-on still names the cache key, so either one saves the entry every consumer of this target restores" + required: false + default: "" + type: string outputs: key: description: "Key needed for restoring depends cache" @@ -103,7 +108,7 @@ jobs: name: Build depends needs: [check-cache] if: needs.check-cache.outputs.cache-hit != 'true' - runs-on: ${{ inputs.runs-on }} + runs-on: ${{ inputs.runs-on-build || inputs.runs-on }} outputs: built-artifact: depends-built-${{ inputs.build-target }} container: diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ecd017fe24fa..c0e2f0f44979 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -38,6 +38,7 @@ jobs: runner-arm64: ${{ steps.select-runner.outputs.runner_arm64 }} runner-prelint: ${{ steps.select-runner.outputs.runner_prelint }} runner-prelint-arm64: ${{ steps.select-runner.outputs.runner_prelint_arm64 }} + runner-fast: ${{ steps.select-runner.outputs.runner_fast }} use-blacksmith: ${{ steps.select-runner.outputs.use_blacksmith }} use-blacksmith-amd64: ${{ steps.select-runner.outputs.use_blacksmith_amd64 }} use-blacksmith-arm64: ${{ steps.select-runner.outputs.use_blacksmith_arm64 }} @@ -119,6 +120,13 @@ jobs: SELFHOSTED_LINT_AUTHORS: ${{ vars.SELFHOSTED_LINT_AUTHORS }} # Fall back to hosted once more jobs than this wait for a self-hosted label SELFHOSTED_MAX_QUEUED: "6" + # Fast path for the linux64 depends, build and functional test jobs, + # which replace their hosted runs rather than add to them. Same + # allowlist as the pre-lint rung, plus every push to develop or + # master. Unset RUNNER_SELFHOSTED_FAST to keep them on the hosted + # ladder; see ci/README.md. + RUNNER_SELFHOSTED_FAST_VAR: ${{ vars.RUNNER_SELFHOSTED_FAST }} + SELFHOSTED_FAST_MAX_QUEUED: "2" run: | python3 .github/workflows/select_dynamic_runner.py @@ -234,6 +242,7 @@ jobs: container-path: ${{ needs.container.outputs.path }} base-image-digest: ${{ needs.check-skip.outputs.base-image-digest }} runs-on: ${{ needs.check-skip.outputs['runner-amd64'] }} + runs-on-build: ${{ needs.check-skip.outputs['runner-fast'] }} runs-on-check: ${{ needs.check-skip.outputs['runner-check-skip'] }} depends-linux64_multiprocess: @@ -322,7 +331,7 @@ jobs: depends-host: ${{ needs.depends-linux64.outputs.host }} depends-dep-opts: ${{ needs.depends-linux64.outputs.dep-opts }} depends-artifact: ${{ needs.depends-linux64.outputs.built-artifact }} - runs-on: ${{ needs.check-skip.outputs['runner-amd64'] }} + runs-on: ${{ needs.check-skip.outputs['runner-fast'] }} secrets: *remote-ccache-auth src-linux64_asan: @@ -451,7 +460,7 @@ jobs: bundle-key: ${{ needs.src-linux64.outputs.key }} build-target: linux64 container-path: ${{ needs.container-slim.outputs.path }} - runs-on: ${{ needs.check-skip.outputs['runner-amd64'] }} + runs-on: ${{ needs.check-skip.outputs['runner-fast'] }} cancel-stale: ${{ needs.src-linux64.outputs.freshness == 'current' && needs.lint.outputs.freshness == 'current' }} test-linux64_asan: diff --git a/.github/workflows/select_dynamic_runner.py b/.github/workflows/select_dynamic_runner.py index f6da231efa94..13f1d34323e2 100644 --- a/.github/workflows/select_dynamic_runner.py +++ b/.github/workflows/select_dynamic_runner.py @@ -31,6 +31,12 @@ # Above this many queued jobs for a self-hosted label, our own runners are the # bottleneck and the pre-lint jobs go back to the hosted ladder. DEFAULT_SELFHOSTED_MAX_QUEUED = 6 +# The fast path's build and test jobs hold a runner for 5-20 minutes rather +# than seconds, so a much shorter wait for one already loses to hosted. +DEFAULT_SELFHOSTED_FAST_MAX_QUEUED = 2 +# Pushes to these branches only ever carry merged code, so they may use the +# fast path whoever pushed them. +FAST_PATH_PUSH_REFS = ("refs/heads/develop", "refs/heads/master") def parse_next_link(link_header: str) -> Optional[str]: @@ -234,6 +240,22 @@ def is_selfhosted_allowed( return False +def is_fast_path_allowed( + event_name: str, + event: Dict, + actor: str, + allowed_authors: Set[str], +) -> bool: + """Whether this event may build and test on the self-hosted fast path. + + The pre-lint rung's trust, plus pushes to develop and master by anyone: + pushing there already requires write access, and they carry merged code. + """ + if event_name == "push" and event.get("ref") in FAST_PATH_PUSH_REFS: + return True + return is_selfhosted_allowed(event_name, event, actor, allowed_authors) + + def allows_cache_write( event_name: str, event: Dict, @@ -252,35 +274,26 @@ def allows_cache_write( ) -def select_prelint_runner( - event_name: str, - event: Dict, - actor: str, +def select_selfhosted_runner( + allowed: bool, backlog_count_value: Optional[int], measurement_error: Optional[str], selfhosted_label: str, - allowed_authors: Set[str], fallback_runner: str, label_override: bool = False, selfhosted_queued: int = 0, selfhosted_max_queued: int = DEFAULT_SELFHOSTED_MAX_QUEUED, disabled_reason: str = "selfhosted-disabled", ) -> Tuple[str, str]: - """Pick the runner for the pre-lint jobs of one arch, and the reason. - - The pre-lint jobs are the short ones that gate lint and the long builds: - lint itself, cache-sources and the container builds and manifest. They - take seconds but can wait most of an hour for a GitHub-hosted runner. + """Pick between a self-hosted label and fallback_runner, and the reason. Inserts a self-hosted rung into the hosted -> Blacksmith ladder: once any - GitHub-hosted job is queued, these jobs go to our own hardware instead of + GitHub-hosted job is queued, the jobs go to our own hardware instead of competing for the account-wide concurrency limit, unless our own runners already have more than selfhosted_max_queued jobs waiting. When the rung is not available they keep the caller's existing decision for the arch. - Every job but the manifest runs pull request head code (lint scripts, - `make -C depends download`, Dockerfile RUN steps), so all of them go - through the same allowlist as lint did on its own. + allowed says whether this event may run on our own hardware at all. """ if not selfhosted_label: return fallback_runner, disabled_reason @@ -298,7 +311,7 @@ def select_prelint_runner( backlog_count_value, SELFHOSTED_BACKLOG_THRESHOLD ) - if not is_selfhosted_allowed(event_name, event, actor, allowed_authors): + if not allowed: return fallback_runner, "actor-not-allowed" if selfhosted_queued > selfhosted_max_queued: @@ -326,6 +339,8 @@ def select_runners( actor: str = "", runner_selfhosted_arm64_var: str = "", selfhosted_max_queued: int = DEFAULT_SELFHOSTED_MAX_QUEUED, + runner_selfhosted_fast_var: str = "", + selfhosted_fast_max_queued: int = DEFAULT_SELFHOSTED_FAST_MAX_QUEUED, ) -> Dict[str, str]: label_names = [ label.get("name", "") @@ -344,7 +359,11 @@ def select_runners( backlog_count_value, selfhosted_queued = count_queued_jobs( fetch_json, repos, - non_hosted_labels=(runner_selfhosted_var, runner_selfhosted_arm64_var), + non_hosted_labels=( + runner_selfhosted_var, + runner_selfhosted_arm64_var, + runner_selfhosted_fast_var, + ), ) backlog_count = str(backlog_count_value) except Exception as exc: # noqa: BLE001 @@ -401,29 +420,52 @@ def select_runners( allowed_authors = parse_author_allowlist(selfhosted_authors) - def prelint( - selfhosted_label: str, fallback_runner: str, disabled_reason: str + def selfhosted( + allowed: bool, + selfhosted_label: str, + fallback_runner: str, + max_queued: int, + disabled_reason: str, ) -> Tuple[str, str]: - return select_prelint_runner( - event_name=event_name, - event=event, - actor=actor, + return select_selfhosted_runner( + allowed=allowed, backlog_count_value=backlog_count_value, measurement_error=measurement_error, selfhosted_label=selfhosted_label, - allowed_authors=allowed_authors, fallback_runner=fallback_runner, label_override=label_override, selfhosted_queued=selfhosted_queued.get(selfhosted_label.lower(), 0), - selfhosted_max_queued=selfhosted_max_queued, + selfhosted_max_queued=max_queued, disabled_reason=disabled_reason, ) - runner_prelint, prelint_decision_reason = prelint( - runner_selfhosted_var, runner_amd64, "selfhosted-disabled" + # The pre-lint jobs (lint, cache-sources, the container builds and + # manifest) take seconds but can wait most of an hour for a hosted runner. + # Every one but the manifest runs pull request head code (lint scripts, + # `make -C depends download`, Dockerfile RUN steps), so all of them go + # through the same allowlist as lint did on its own. + prelint_allowed = is_selfhosted_allowed(event_name, event, actor, allowed_authors) + runner_prelint, prelint_decision_reason = selfhosted( + allowed=prelint_allowed, + selfhosted_label=runner_selfhosted_var, + fallback_runner=runner_amd64, + max_queued=selfhosted_max_queued, + disabled_reason="selfhosted-disabled", + ) + runner_prelint_arm64, prelint_arm64_decision_reason = selfhosted( + allowed=prelint_allowed, + selfhosted_label=runner_selfhosted_arm64_var, + fallback_runner=runner_arm64, + max_queued=selfhosted_max_queued, + disabled_reason="selfhosted-arm64-disabled", ) - runner_prelint_arm64, prelint_arm64_decision_reason = prelint( - runner_selfhosted_arm64_var, runner_arm64, "selfhosted-arm64-disabled" + # The fast path: the linux64 depends, build and functional test jobs. + runner_fast, fast_decision_reason = selfhosted( + allowed=is_fast_path_allowed(event_name, event, actor, allowed_authors), + selfhosted_label=runner_selfhosted_fast_var, + fallback_runner=runner_amd64, + max_queued=selfhosted_fast_max_queued, + disabled_reason="selfhosted-fast-disabled", ) cache_write = allows_cache_write(event_name, event, actor, allowed_authors) @@ -434,6 +476,8 @@ def prelint( "prelint_decision_reason": prelint_decision_reason, "runner_prelint_arm64": runner_prelint_arm64, "prelint_arm64_decision_reason": prelint_arm64_decision_reason, + "runner_fast": runner_fast, + "fast_decision_reason": fast_decision_reason, "use_blacksmith": "true" if use_blacksmith_amd64 or use_blacksmith_arm64 else "false", "use_blacksmith_amd64": "true" if use_blacksmith_amd64 else "false", "use_blacksmith_arm64": "true" if use_blacksmith_arm64 else "false", @@ -481,10 +525,12 @@ def write_step_summary(path: Optional[str], outputs: Dict[str, str]) -> None: fh.write("- arm64 runner: `{}`\n".format(outputs["runner_arm64"])) fh.write("- pre-lint amd64 runner: `{}`\n".format(outputs["runner_prelint"])) fh.write("- pre-lint arm64 runner: `{}`\n".format(outputs["runner_prelint_arm64"])) + fh.write("- linux64 fast path runner: `{}`\n".format(outputs["runner_fast"])) fh.write("- Decision: `{}`\n".format(outputs["decision_reason"])) fh.write("- Pre-lint amd64 decision: `{}`\n".format(outputs["prelint_decision_reason"])) fh.write("- Pre-lint arm64 decision: `{}`\n".format( outputs["prelint_arm64_decision_reason"])) + fh.write("- Fast path decision: `{}`\n".format(outputs["fast_decision_reason"])) fh.write( "- Remote ccache writes: {}\n".format( "yes" if outputs["cache_write"] == "true" else "no" @@ -550,6 +596,17 @@ def parse_args(argv: Sequence[str]) -> argparse.Namespace: default=env_int("SELFHOSTED_MAX_QUEUED", DEFAULT_SELFHOSTED_MAX_QUEUED), help="Use a self-hosted label only while at most this many jobs are queued for it", ) + parser.add_argument( + "--runner-selfhosted-fast-var", + default=os.environ.get("RUNNER_SELFHOSTED_FAST_VAR", ""), + help="Self-hosted amd64 runner label for the linux64 build and test fast path; empty disables it", + ) + parser.add_argument( + "--selfhosted-fast-max-queued", + type=int, + default=env_int("SELFHOSTED_FAST_MAX_QUEUED", DEFAULT_SELFHOSTED_FAST_MAX_QUEUED), + help="Use the fast path label only while at most this many jobs are queued for it", + ) parser.add_argument( "--selfhosted-authors", default=os.environ.get("SELFHOSTED_LINT_AUTHORS", ""), @@ -607,6 +664,8 @@ def main(argv: Sequence[str]) -> int: actor=args.actor, runner_selfhosted_arm64_var=args.runner_selfhosted_arm64_var, selfhosted_max_queued=args.selfhosted_max_queued, + runner_selfhosted_fast_var=args.runner_selfhosted_fast_var, + selfhosted_fast_max_queued=args.selfhosted_fast_max_queued, ) write_github_output(os.environ.get("GITHUB_OUTPUT"), outputs) diff --git a/.github/workflows/test_select_dynamic_runner.py b/.github/workflows/test_select_dynamic_runner.py index 5d71db24e6a7..b1b90239f150 100644 --- a/.github/workflows/test_select_dynamic_runner.py +++ b/.github/workflows/test_select_dynamic_runner.py @@ -523,6 +523,112 @@ def test_lint_inherits_blacksmith_when_rung_unavailable(self): self.assertEqual(outputs["runner_amd64"], "blacksmith-amd64") self.assertEqual(outputs["runner_prelint"], "blacksmith-amd64") + # --- linux64 build and test fast path ------------------------------------ + + def _select_fast(self, **overrides): + overrides.setdefault("runner_selfhosted_fast_var", "dash-x64-build") + return self._select(**overrides) + + def test_fast_path_is_off_until_its_label_is_set(self): + outputs = self._select() + + self.assertEqual(outputs["runner_fast"], MODULE.DEFAULT_RUNNER_AMD64) + self.assertEqual(outputs["fast_decision_reason"], "selfhosted-fast-disabled") + + def test_fast_path_for_trusted_pull_requests(self): + outputs = self._select_fast() + + self.assertEqual(outputs["runner_fast"], "dash-x64-build") + self.assertEqual( + outputs["fast_decision_reason"], "selfhosted:backlog:3>0;queued:0<=2" + ) + # Independent of the pre-lint rung's label. + self.assertEqual(outputs["runner_prelint"], "ubuntu-core") + + def test_fast_path_falls_back_for_untrusted_pull_requests(self): + for author, actor, head_owner in ( + ("mallory", "PastaPastaPasta", "mallory"), + ("PastaPastaPasta", "mallory", "PastaPastaPasta"), + ("PastaPastaPasta", "PastaPastaPasta", "mallory"), + ): + with self.subTest(author=author, actor=actor, head_owner=head_owner): + outputs = self._select_fast( + event=pull_request_event(author, head_owner=head_owner), + actor=actor, + ) + self.assertEqual(outputs["runner_fast"], MODULE.DEFAULT_RUNNER_AMD64) + self.assertEqual(outputs["fast_decision_reason"], "actor-not-allowed") + + def test_fast_path_for_develop_and_master_pushes_by_anyone(self): + for ref in ("refs/heads/develop", "refs/heads/master"): + with self.subTest(ref=ref): + outputs = self._select_fast( + event_name="push", event={"ref": ref}, actor="mallory" + ) + self.assertEqual(outputs["runner_fast"], "dash-x64-build") + + def test_fast_path_for_other_pushes_needs_the_allowlist(self): + for ref in ("refs/heads/feature", "refs/tags/v24.0.0", "refs/heads/developer"): + with self.subTest(ref=ref): + outputs = self._select_fast( + event_name="push", event={"ref": ref}, actor="mallory" + ) + self.assertEqual(outputs["runner_fast"], MODULE.DEFAULT_RUNNER_AMD64) + self.assertEqual(outputs["fast_decision_reason"], "actor-not-allowed") + + outputs = self._select_fast( + event_name="push", event={"ref": ref}, actor="knst" + ) + self.assertEqual(outputs["runner_fast"], "dash-x64-build") + + def test_fast_path_falls_back_without_backlog(self): + outputs = self._select_fast( + event_name="push", event={"ref": "refs/heads/develop"}, queued_jobs=0 + ) + + self.assertEqual(outputs["runner_fast"], MODULE.DEFAULT_RUNNER_AMD64) + self.assertEqual(outputs["fast_decision_reason"], "backlog:0<=0") + + def test_fast_path_falls_back_on_measurement_error(self): + def fetch_json(_url): + raise RuntimeError("boom") + + outputs = self._select_fast(fetch_json=fetch_json) + + self.assertEqual(outputs["runner_fast"], MODULE.DEFAULT_RUNNER_AMD64) + self.assertEqual(outputs["fast_decision_reason"], "metric-unavailable") + + def test_fast_path_has_its_own_saturation_cap(self): + for queued, expected in ((2, "dash-x64-build"), (3, MODULE.DEFAULT_RUNNER_AMD64)): + with self.subTest(queued=queued): + responses = add_queued_jobs(backlog_responses(3), "dash-x64-build", queued) + + outputs = self._select_fast(fetch_json=lambda url: responses[url]) + + # Jobs waiting for our own runners are not hosted backlog. + self.assertEqual(outputs["backlog_count"], "3") + self.assertEqual(outputs["runner_fast"], expected) + # The pre-lint rung keeps its own, larger cap. + self.assertEqual(outputs["runner_prelint"], "ubuntu-core") + if queued > 2: + self.assertEqual( + outputs["fast_decision_reason"], "selfhosted-saturated:queued:3>2" + ) + + def test_fast_path_inherits_the_amd64_decision_when_unavailable(self): + outputs = self._select_fast( + event=pull_request_event("PastaPastaPasta", labels=["blacksmith-ci"]), + ) + self.assertEqual(outputs["runner_fast"], "blacksmith-amd64") + self.assertEqual(outputs["fast_decision_reason"], "label:blacksmith-ci") + + outputs = self._select_fast( + queued_jobs=11, + event=pull_request_event("mallory"), + actor="mallory", + ) + self.assertEqual(outputs["runner_fast"], "blacksmith-amd64") + # --- remote ccache write gate -------------------------------------------- def test_cache_write_on_any_push(self): diff --git a/ci/README.md b/ci/README.md index 433440af1298..bed691dabb7e 100644 --- a/ci/README.md +++ b/ci/README.md @@ -24,3 +24,39 @@ with fallback values. In order to avoid rebuilding all dependencies for each build, the binaries are cached and reused when possible. Changes in the dependency-generator will trigger cache-invalidation and rebuilds as necessary. + +### Self-hosted fast path + +When the repository variable `RUNNER_SELFHOSTED_FAST` names a runner label, +`.github/workflows/select_dynamic_runner.py` may move three jobs of the +`linux64` target from the hosted runner ladder (GitHub-hosted or Blacksmith) +to that label: the depends build (only on a depends cache miss), +`linux64-build` and `linux64-test`. They keep their names and steps, so +required checks are unchanged, and the depends cache key still names the +hosted runner, so the entry they save is the one the other linux64 builds +restore. Every other job stays where it was. + +The decision is made once per run, by the `Check skip conditions` job. A run +takes the fast path only when all of these hold: + +- it is a push to `develop` or `master`, or an event the pre-lint rung trusts: + a push whose actor is listed in `SELFHOSTED_LINT_AUTHORS`, or a pull request + whose author and actor are listed and whose head repository is this one or + is owned by a listed login. Everything else, including fork pull requests + from other contributors, stays on the hosted ladder; +- some GitHub-hosted job is queued (otherwise hosted starts at once); +- no `blacksmith-ci` label overrides the choice; +- at most `SELFHOSTED_FAST_MAX_QUEUED` (2, set in `build.yml`) jobs are queued + for the label at that moment. Jobs already running there, or due to queue + later in runs that already chose it, are not counted, so a burst of runs + started together can still overfill a small pool. + +Leaving the variable unset turns the fast path off. Jobs already queued for +the label must be cancelled and re-run after unsetting it. + +Runners registered for the label must provide x86_64 Linux with a Docker +daemon (the jobs run in the CI container with `--cap-add SYS_PTRACE`), about 8 +CPUs and 16 GiB of memory or more, roughly 40 GiB of free disk, and access to +GitHub's cache service and the remote ccache (`CCACHE_REMOTE_URL`). These runs +execute pull request code with access to that Docker daemon, so give each job +a fresh, single-use machine.