Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion .github/workflows/build-depends.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,11 @@ on:
required: false
default: x86_64
type: string
runs-on-build:
description: "Runner label for the build job if not runs-on; runs-on still names the cache key, so either one saves the entry every consumer of this target restores"
required: false
default: ""
type: string
outputs:
key:
description: "Key needed for restoring depends cache"
Expand Down Expand Up @@ -103,7 +108,7 @@ jobs:
name: Build depends
needs: [check-cache]
if: needs.check-cache.outputs.cache-hit != 'true'
runs-on: ${{ inputs.runs-on }}
runs-on: ${{ inputs.runs-on-build || inputs.runs-on }}
outputs:
built-artifact: depends-built-${{ inputs.build-target }}
container:
Expand Down
13 changes: 11 additions & 2 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ jobs:
runner-arm64: ${{ steps.select-runner.outputs.runner_arm64 }}
runner-prelint: ${{ steps.select-runner.outputs.runner_prelint }}
runner-prelint-arm64: ${{ steps.select-runner.outputs.runner_prelint_arm64 }}
runner-fast: ${{ steps.select-runner.outputs.runner_fast }}
use-blacksmith: ${{ steps.select-runner.outputs.use_blacksmith }}
use-blacksmith-amd64: ${{ steps.select-runner.outputs.use_blacksmith_amd64 }}
use-blacksmith-arm64: ${{ steps.select-runner.outputs.use_blacksmith_arm64 }}
Expand Down Expand Up @@ -119,6 +120,13 @@ jobs:
SELFHOSTED_LINT_AUTHORS: ${{ vars.SELFHOSTED_LINT_AUTHORS }}
# Fall back to hosted once more jobs than this wait for a self-hosted label
SELFHOSTED_MAX_QUEUED: "6"
# Fast path for the linux64 depends, build and functional test jobs,
# which replace their hosted runs rather than add to them. Same
# allowlist as the pre-lint rung, plus every push to develop or
# master. Unset RUNNER_SELFHOSTED_FAST to keep them on the hosted
# ladder; see ci/README.md.
RUNNER_SELFHOSTED_FAST_VAR: ${{ vars.RUNNER_SELFHOSTED_FAST }}
SELFHOSTED_FAST_MAX_QUEUED: "2"
run: |
python3 .github/workflows/select_dynamic_runner.py

Expand Down Expand Up @@ -234,6 +242,7 @@ jobs:
container-path: ${{ needs.container.outputs.path }}
base-image-digest: ${{ needs.check-skip.outputs.base-image-digest }}
runs-on: ${{ needs.check-skip.outputs['runner-amd64'] }}
runs-on-build: ${{ needs.check-skip.outputs['runner-fast'] }}
runs-on-check: ${{ needs.check-skip.outputs['runner-check-skip'] }}

depends-linux64_multiprocess:
Expand Down Expand Up @@ -322,7 +331,7 @@ jobs:
depends-host: ${{ needs.depends-linux64.outputs.host }}
depends-dep-opts: ${{ needs.depends-linux64.outputs.dep-opts }}
depends-artifact: ${{ needs.depends-linux64.outputs.built-artifact }}
runs-on: ${{ needs.check-skip.outputs['runner-amd64'] }}
runs-on: ${{ needs.check-skip.outputs['runner-fast'] }}
secrets: *remote-ccache-auth

src-linux64_asan:
Expand Down Expand Up @@ -451,7 +460,7 @@ jobs:
bundle-key: ${{ needs.src-linux64.outputs.key }}
build-target: linux64
container-path: ${{ needs.container-slim.outputs.path }}
runs-on: ${{ needs.check-skip.outputs['runner-amd64'] }}
runs-on: ${{ needs.check-skip.outputs['runner-fast'] }}
cancel-stale: ${{ needs.src-linux64.outputs.freshness == 'current' && needs.lint.outputs.freshness == 'current' }}

test-linux64_asan:
Expand Down
115 changes: 87 additions & 28 deletions .github/workflows/select_dynamic_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,12 @@
# Above this many queued jobs for a self-hosted label, our own runners are the
# bottleneck and the pre-lint jobs go back to the hosted ladder.
DEFAULT_SELFHOSTED_MAX_QUEUED = 6
# The fast path's build and test jobs hold a runner for 5-20 minutes rather
# than seconds, so a much shorter wait for one already loses to hosted.
DEFAULT_SELFHOSTED_FAST_MAX_QUEUED = 2
# Pushes to these branches only ever carry merged code, so they may use the
# fast path whoever pushed them.
FAST_PATH_PUSH_REFS = ("refs/heads/develop", "refs/heads/master")


def parse_next_link(link_header: str) -> Optional[str]:
Expand Down Expand Up @@ -234,6 +240,22 @@ def is_selfhosted_allowed(
return False


def is_fast_path_allowed(
event_name: str,
event: Dict,
actor: str,
allowed_authors: Set[str],
) -> bool:
"""Whether this event may build and test on the self-hosted fast path.

The pre-lint rung's trust, plus pushes to develop and master by anyone:
pushing there already requires write access, and they carry merged code.
"""
if event_name == "push" and event.get("ref") in FAST_PATH_PUSH_REFS:
return True
return is_selfhosted_allowed(event_name, event, actor, allowed_authors)


def allows_cache_write(
event_name: str,
event: Dict,
Expand All @@ -252,35 +274,26 @@ def allows_cache_write(
)


def select_prelint_runner(
event_name: str,
event: Dict,
actor: str,
def select_selfhosted_runner(
allowed: bool,
backlog_count_value: Optional[int],
measurement_error: Optional[str],
selfhosted_label: str,
allowed_authors: Set[str],
fallback_runner: str,
label_override: bool = False,
selfhosted_queued: int = 0,
selfhosted_max_queued: int = DEFAULT_SELFHOSTED_MAX_QUEUED,
disabled_reason: str = "selfhosted-disabled",
) -> Tuple[str, str]:
"""Pick the runner for the pre-lint jobs of one arch, and the reason.

The pre-lint jobs are the short ones that gate lint and the long builds:
lint itself, cache-sources and the container builds and manifest. They
take seconds but can wait most of an hour for a GitHub-hosted runner.
"""Pick between a self-hosted label and fallback_runner, and the reason.

Inserts a self-hosted rung into the hosted -> Blacksmith ladder: once any
GitHub-hosted job is queued, these jobs go to our own hardware instead of
GitHub-hosted job is queued, the jobs go to our own hardware instead of
competing for the account-wide concurrency limit, unless our own runners
already have more than selfhosted_max_queued jobs waiting. When the rung
is not available they keep the caller's existing decision for the arch.

Every job but the manifest runs pull request head code (lint scripts,
`make -C depends download`, Dockerfile RUN steps), so all of them go
through the same allowlist as lint did on its own.
allowed says whether this event may run on our own hardware at all.
"""
if not selfhosted_label:
return fallback_runner, disabled_reason
Expand All @@ -298,7 +311,7 @@ def select_prelint_runner(
backlog_count_value, SELFHOSTED_BACKLOG_THRESHOLD
)

if not is_selfhosted_allowed(event_name, event, actor, allowed_authors):
if not allowed:
return fallback_runner, "actor-not-allowed"

if selfhosted_queued > selfhosted_max_queued:
Expand Down Expand Up @@ -326,6 +339,8 @@ def select_runners(
actor: str = "",
runner_selfhosted_arm64_var: str = "",
selfhosted_max_queued: int = DEFAULT_SELFHOSTED_MAX_QUEUED,
runner_selfhosted_fast_var: str = "",
selfhosted_fast_max_queued: int = DEFAULT_SELFHOSTED_FAST_MAX_QUEUED,
) -> Dict[str, str]:
label_names = [
label.get("name", "")
Expand All @@ -344,7 +359,11 @@ def select_runners(
backlog_count_value, selfhosted_queued = count_queued_jobs(
fetch_json,
repos,
non_hosted_labels=(runner_selfhosted_var, runner_selfhosted_arm64_var),
non_hosted_labels=(
runner_selfhosted_var,
runner_selfhosted_arm64_var,
runner_selfhosted_fast_var,
),
)
backlog_count = str(backlog_count_value)
except Exception as exc: # noqa: BLE001
Expand Down Expand Up @@ -401,29 +420,52 @@ def select_runners(

allowed_authors = parse_author_allowlist(selfhosted_authors)

def prelint(
selfhosted_label: str, fallback_runner: str, disabled_reason: str
def selfhosted(
allowed: bool,
selfhosted_label: str,
fallback_runner: str,
max_queued: int,
disabled_reason: str,
) -> Tuple[str, str]:
return select_prelint_runner(
event_name=event_name,
event=event,
actor=actor,
return select_selfhosted_runner(
allowed=allowed,
backlog_count_value=backlog_count_value,
measurement_error=measurement_error,
selfhosted_label=selfhosted_label,
allowed_authors=allowed_authors,
fallback_runner=fallback_runner,
label_override=label_override,
selfhosted_queued=selfhosted_queued.get(selfhosted_label.lower(), 0),
selfhosted_max_queued=selfhosted_max_queued,
selfhosted_max_queued=max_queued,
disabled_reason=disabled_reason,
)

runner_prelint, prelint_decision_reason = prelint(
runner_selfhosted_var, runner_amd64, "selfhosted-disabled"
# The pre-lint jobs (lint, cache-sources, the container builds and
# manifest) take seconds but can wait most of an hour for a hosted runner.
# Every one but the manifest runs pull request head code (lint scripts,
# `make -C depends download`, Dockerfile RUN steps), so all of them go
# through the same allowlist as lint did on its own.
prelint_allowed = is_selfhosted_allowed(event_name, event, actor, allowed_authors)
runner_prelint, prelint_decision_reason = selfhosted(
allowed=prelint_allowed,
selfhosted_label=runner_selfhosted_var,
fallback_runner=runner_amd64,
max_queued=selfhosted_max_queued,
disabled_reason="selfhosted-disabled",
)
runner_prelint_arm64, prelint_arm64_decision_reason = selfhosted(
allowed=prelint_allowed,
selfhosted_label=runner_selfhosted_arm64_var,
fallback_runner=runner_arm64,
max_queued=selfhosted_max_queued,
disabled_reason="selfhosted-arm64-disabled",
)
runner_prelint_arm64, prelint_arm64_decision_reason = prelint(
runner_selfhosted_arm64_var, runner_arm64, "selfhosted-arm64-disabled"
# The fast path: the linux64 depends, build and functional test jobs.
runner_fast, fast_decision_reason = selfhosted(
allowed=is_fast_path_allowed(event_name, event, actor, allowed_authors),
selfhosted_label=runner_selfhosted_fast_var,
fallback_runner=runner_amd64,
max_queued=selfhosted_fast_max_queued,
disabled_reason="selfhosted-fast-disabled",
)
cache_write = allows_cache_write(event_name, event, actor, allowed_authors)

Expand All @@ -434,6 +476,8 @@ def prelint(
"prelint_decision_reason": prelint_decision_reason,
"runner_prelint_arm64": runner_prelint_arm64,
"prelint_arm64_decision_reason": prelint_arm64_decision_reason,
"runner_fast": runner_fast,
"fast_decision_reason": fast_decision_reason,
"use_blacksmith": "true" if use_blacksmith_amd64 or use_blacksmith_arm64 else "false",
"use_blacksmith_amd64": "true" if use_blacksmith_amd64 else "false",
"use_blacksmith_arm64": "true" if use_blacksmith_arm64 else "false",
Expand Down Expand Up @@ -481,10 +525,12 @@ def write_step_summary(path: Optional[str], outputs: Dict[str, str]) -> None:
fh.write("- arm64 runner: `{}`\n".format(outputs["runner_arm64"]))
fh.write("- pre-lint amd64 runner: `{}`\n".format(outputs["runner_prelint"]))
fh.write("- pre-lint arm64 runner: `{}`\n".format(outputs["runner_prelint_arm64"]))
fh.write("- linux64 fast path runner: `{}`\n".format(outputs["runner_fast"]))
fh.write("- Decision: `{}`\n".format(outputs["decision_reason"]))
fh.write("- Pre-lint amd64 decision: `{}`\n".format(outputs["prelint_decision_reason"]))
fh.write("- Pre-lint arm64 decision: `{}`\n".format(
outputs["prelint_arm64_decision_reason"]))
fh.write("- Fast path decision: `{}`\n".format(outputs["fast_decision_reason"]))
fh.write(
"- Remote ccache writes: {}\n".format(
"yes" if outputs["cache_write"] == "true" else "no"
Expand Down Expand Up @@ -550,6 +596,17 @@ def parse_args(argv: Sequence[str]) -> argparse.Namespace:
default=env_int("SELFHOSTED_MAX_QUEUED", DEFAULT_SELFHOSTED_MAX_QUEUED),
help="Use a self-hosted label only while at most this many jobs are queued for it",
)
parser.add_argument(
"--runner-selfhosted-fast-var",
default=os.environ.get("RUNNER_SELFHOSTED_FAST_VAR", ""),
help="Self-hosted amd64 runner label for the linux64 build and test fast path; empty disables it",
)
parser.add_argument(
"--selfhosted-fast-max-queued",
type=int,
default=env_int("SELFHOSTED_FAST_MAX_QUEUED", DEFAULT_SELFHOSTED_FAST_MAX_QUEUED),
help="Use the fast path label only while at most this many jobs are queued for it",
)
parser.add_argument(
"--selfhosted-authors",
default=os.environ.get("SELFHOSTED_LINT_AUTHORS", ""),
Expand Down Expand Up @@ -607,6 +664,8 @@ def main(argv: Sequence[str]) -> int:
actor=args.actor,
runner_selfhosted_arm64_var=args.runner_selfhosted_arm64_var,
selfhosted_max_queued=args.selfhosted_max_queued,
runner_selfhosted_fast_var=args.runner_selfhosted_fast_var,
selfhosted_fast_max_queued=args.selfhosted_fast_max_queued,
)

write_github_output(os.environ.get("GITHUB_OUTPUT"), outputs)
Expand Down
Loading
Loading