From d1c47dfdcd21b5603429bbcc73b21174544894cf Mon Sep 17 00:00:00 2001 From: Zach Vorhies Date: Thu, 19 Nov 2026 11:16:17 -0800 Subject: [PATCH 1/6] test(ci): record RED for member PR Dell eligibility and isolation (Refs #4896) --- ci/test_member_pr_routing.py | 163 +++++++++++++++++++++++++++++++++++ 1 file changed, 163 insertions(+) create mode 100644 ci/test_member_pr_routing.py diff --git a/ci/test_member_pr_routing.py b/ci/test_member_pr_routing.py new file mode 100644 index 0000000000..d921203157 --- /dev/null +++ b/ci/test_member_pr_routing.py @@ -0,0 +1,163 @@ +#!/usr/bin/env -S uv run --script +"""Member PR routing stays base-owned, fail-closed and below queue priority.""" + +import io +import json +import os +from pathlib import Path +import re +import tempfile +import unittest +from unittest.mock import patch +from urllib.error import HTTPError +from urllib.parse import parse_qs, urlparse + +import select_mq_runner as selector + +ROOT = Path(__file__).resolve().parent.parent +LABELS = '["self-hosted","linux","x64","obs-mq"]' + + +class MemberPrRoutingTest(unittest.TestCase): + def select(self, *, head="obs-rust/obs-studio", author="alice", state="active", + error=False, capacity=12, reserve="", enabled="on", event="pull_request", + peers=(), jobs=None, payload=None): + self.paths = [] + current = {"id": 20, "event": event, "status": "in_progress", + "created_at": "2026-11-19T12:00:00Z"} + runs = [current, *peers] + data = payload if payload is not None else { + "repository": {"full_name": "obs-rust/obs-studio", "id": 1}, + "pull_request": { + "user": {"login": author}, + "head": {"repo": {"full_name": head, "id": 1 if head == "obs-rust/obs-studio" else 2}}, + "base": {"repo": {"full_name": "obs-rust/obs-studio", "id": 1}}, + }, + } + + def respond(request, **kwargs): + url = urlparse(request.full_url) + self.paths.append(url.path) + if "/public_members/" in url.path: + raise HTTPError(url.geturl(), 404, "private", {}, None) + if "/memberships/" in url.path: + if error: + raise HTTPError(url.geturl(), 503, "unavailable", {}, None) + result = {"state": state, "user": {"login": author}} + elif url.path.endswith("/runners"): + entries = [{"id": i + 1, "status": "online", "busy": False, + "labels": [{"name": name} for name in json.loads(LABELS)]} + for i in range(capacity)] + result = {"total_count": len(entries), "runners": entries} + elif url.path.endswith("/runs"): + query = parse_qs(url.query) + entries = [r for r in runs if r["event"] == query["event"][0] + and r["status"] == query["status"][0]] + result = {"total_count": len(entries), "workflow_runs": entries} + else: + run_id = int(url.path.split("/runs/")[1].split("/")[0]) + entries = (jobs or {}).get(run_id, []) + result = {"total_count": len(entries), "jobs": entries} + return io.BytesIO(json.dumps(result).encode()) + + with tempfile.TemporaryDirectory() as directory: + event_path = Path(directory) / "event.json" + event_path.write_text(json.dumps(data)) + with patch.dict(os.environ, { + "GITHUB_EVENT_NAME": event, "GITHUB_EVENT_PATH": str(event_path), + "GITHUB_RUN_ID": "20", "GITHUB_TOKEN": "actions-token", + "DELL_PR_ROUTING": enabled, "MQ_PR_RESERVE": reserve, + }, clear=True), patch.object(selector, "urlopen", side_effect=respond), patch.object(selector.time, "sleep"): + return selector.select_labels(event, LABELS, "probe-token", "obs-rust/obs-studio", demand="6") + + def test_member_with_idle_capacity_uses_dell(self): + self.assertEqual(self.select(), LABELS) + self.assertIn("/orgs/obs-rust/memberships/alice", self.paths) + + def test_fork_uses_hosted_without_api(self): + self.assertEqual(self.select(head="fork/obs-studio"), "") + self.assertEqual(self.paths, []) + + def test_non_member_and_pending_members_use_hosted(self): + for state in ("pending", "inactive", None): + with self.subTest(state=state): + self.assertEqual(self.select(state=state), "") + + def test_membership_api_error_uses_hosted_without_retry(self): + self.assertEqual(self.select(error=True), "") + self.assertEqual(self.paths.count("/orgs/obs-rust/memberships/alice"), 1) + self.assertFalse(any(path.endswith("/runners") for path in self.paths)) + + def test_capacity_below_default_reserve_uses_hosted(self): + self.assertEqual(self.select(capacity=11), "") + + def test_reserve_can_be_configured(self): + self.assertEqual(self.select(capacity=8, reserve="2"), LABELS) + self.assertEqual(self.select(capacity=7, reserve="2"), "") + + def test_invalid_reserve_uses_hosted(self): + for reserve in ("-1", "oops"): + self.assertEqual(self.select(reserve=reserve), "") + + def test_kill_switch_defaults_off(self): + for enabled in ("", "off", "true", "ON"): + self.assertEqual(self.select(enabled=enabled), "") + self.assertEqual(self.paths, []) + + def test_pull_request_target_never_routes(self): + self.assertEqual(self.select(event="pull_request_target"), "") + self.assertEqual(self.paths, []) + + def test_untrusted_or_missing_payload_uses_hosted(self): + for payload in ({}, {"pull_request": None}, {"repository": {"full_name": "fork/repo"}}): + self.assertEqual(self.select(payload=payload), "") + + def test_merge_group_ignores_pr_kill_switch_and_reserve(self): + self.assertEqual(self.select(event="merge_group", enabled="off", capacity=6, reserve="bad"), LABELS) + + def test_pr_reserves_undecided_merge_queue_even_when_newer(self): + peer = {"id": 30, "event": "merge_group", "status": "queued", + "created_at": "2026-11-19T12:01:00Z"} + self.assertEqual(self.select(peers=[peer]), "") + + def test_existing_pr_dell_jobs_reduce_capacity_for_prs_and_queue(self): + peer = {"id": 10, "event": "pull_request", "status": "in_progress", + "created_at": "2026-11-19T11:59:00Z"} + jobs = {10: [{"name": "Fast Gate ⚡", "status": "queued", "conclusion": None, + "labels": json.loads(LABELS)} for _ in range(7)]} + for event in ("pull_request", "merge_group"): + self.assertEqual(self.select(event=event, peers=[peer], jobs=jobs), "") + + +class MemberPrWorkflowTest(unittest.TestCase): + def test_base_owned_selector_and_membership_helper(self): + ci = (ROOT / ".github/workflows/ci.yml").read_text() + probe = ci.split(" select-mq-runner:\n", 1)[1].split(" cancel-superseded:\n", 1)[0] + self.assertIn("ref: ${{ github.event.pull_request.base.sha || github.sha }}", probe) + self.assertIn('git show "$BASE:ci/$script"', probe) + self.assertIn("select_mq_runner.py pr_attestation.py", probe) + self.assertIn('uv run --no-project "$trusted/select_mq_runner.py"', probe) + self.assertIn("DELL_PR_ROUTING: ${{ vars.DELL_PR_ROUTING || 'off' }}", probe) + self.assertIn("MQ_PR_RESERVE: ${{ vars.MQ_PR_RESERVE }}", probe) + self.assertNotIn("if: github.event_name != 'pull_request'\n", probe) + + def test_dell_pr_jobs_do_not_receive_repository_secrets(self): + text = (ROOT / ".github/workflows/build-project.yaml").read_text() + blocks = dict(re.findall(r"^ ([\w-]+):\n(.*?)(?=^ [\w-]+:|\Z)", text, re.M | re.S)) + for name, block in blocks.items(): + if not re.search(r"runs-on: .*inputs.mq-heavy-labels", block): + continue + for line in block.splitlines(): + if "secrets." in line and "secrets.GITHUB_TOKEN" not in line: + with self.subTest(job=name, line=line): + self.assertIn("github.event_name != 'pull_request' && secrets.", line) + + def test_fast_gate_pr_routing_keeps_queue_hosted(self): + text = (ROOT / ".github/workflows/build-project.yaml").read_text() + block = text.split(" rust-tests-linux:\n", 1)[1].split("\n cross-compile:", 1)[0] + runner = re.search(r"^ runs-on: (.*)$", block, re.M)[1] + self.assertIn("github.event_name == 'pull_request' && inputs.mq-heavy-labels", runner) + + +if __name__ == "__main__": + unittest.main() From e7b88f632ff9b7003af15316d430464f6b4ec9e9 Mon Sep 17 00:00:00 2001 From: Zach Vorhies Date: Thu, 19 Nov 2026 11:20:51 -0800 Subject: [PATCH 2/6] ci: let members' same-repo PRs use idle Dell capacity (default off) Refs #4839 #4896 --- .github/workflows/build-project.yaml | 79 +++++++++++++++++++------- .github/workflows/ci.yml | 45 ++++++++++++--- ci/select_mq_runner.py | 83 +++++++++++++++++++++++----- ci/test_ci_scope.py | 17 +++--- ci/test_member_pr_routing.py | 11 +++- docs/ci-tooling.md | 47 +++++++++++++++- docs/features.md | 6 ++ 7 files changed, 234 insertions(+), 54 deletions(-) diff --git a/.github/workflows/build-project.yaml b/.github/workflows/build-project.yaml index 452e4d6f84..6b9514b6a4 100644 --- a/.github/workflows/build-project.yaml +++ b/.github/workflows/build-project.yaml @@ -65,6 +65,43 @@ on: type: boolean required: false default: false + secrets: + GPU_PRIORITY_VAL: + required: false + MACOS_KEYCHAIN_PASSWORD: + required: false + MACOS_NOTARIZATION_PASSWORD: + required: false + MACOS_NOTARIZATION_USERNAME: + required: false + MACOS_SIGNING_CERT: + required: false + MACOS_SIGNING_CERT_PASSWORD: + required: false + MACOS_SIGNING_IDENTITY: + required: false + MACOS_SIGNING_PROVISIONING_PROFILE: + required: false + RESTREAM_CLIENTID: + required: false + RESTREAM_HASH: + required: false + SOLDR_S3_CACHE_ACCESS_KEY: + required: false + SOLDR_S3_CACHE_SECRET_KEY: + required: false + TWITCH_CLIENT_ID: + required: false + TWITCH_HASH: + required: false + YOUTUBE_CLIENTID: + required: false + YOUTUBE_CLIENTID_HASH: + required: false + YOUTUBE_SECRET: + required: false + YOUTUBE_SECRET_HASH: + required: false outputs: skip_checks: description: Immutable-base client admission skip_checks @@ -1719,7 +1756,7 @@ jobs: # letting it hold the queue for an hour (#2593 A). timeout-minutes: 20 name: Fast Gate ⚡ - runs-on: ubuntu-26.04 + runs-on: ${{ github.event_name == 'pull_request' && inputs.mq-heavy-labels != '' && fromJSON(inputs.mq-heavy-labels) || 'ubuntu-26.04' }} needs: [check-event, ci-mode] # A docs-only PR or queue run changes no Rust, C or CMake: skipped, and # passing CI Gate; the Format job still runs the ci/ self-tests (#2079). @@ -1788,8 +1825,8 @@ jobs: s3-region: nyc3 s3-cache-limit: 50G env: - SOLDR_S3_CACHE_ACCESS_KEY: ${{ secrets.SOLDR_S3_CACHE_ACCESS_KEY }} - SOLDR_S3_CACHE_SECRET_KEY: ${{ secrets.SOLDR_S3_CACHE_SECRET_KEY }} + SOLDR_S3_CACHE_ACCESS_KEY: ${{ (github.event_name != 'pull_request' || inputs.mq-heavy-labels == '') && secrets.SOLDR_S3_CACHE_ACCESS_KEY || '' }} + SOLDR_S3_CACHE_SECRET_KEY: ${{ (github.event_name != 'pull_request' || inputs.mq-heavy-labels == '') && secrets.SOLDR_S3_CACHE_SECRET_KEY || '' }} # GATE-001 mirror (local-gate.toml): exactly the local gate's lint and # rust-test lanes (./lint, ./unittest), i.e. rustfmt, clippy -D warnings, @@ -1987,8 +2024,8 @@ jobs: if: ${{ !cancelled() && github.ref == 'refs/heads/master' }} shell: bash env: - AWS_ACCESS_KEY_ID: ${{ secrets.SOLDR_S3_CACHE_ACCESS_KEY }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.SOLDR_S3_CACHE_SECRET_KEY }} + AWS_ACCESS_KEY_ID: ${{ (github.event_name != 'pull_request' || inputs.mq-heavy-labels == '') && secrets.SOLDR_S3_CACHE_ACCESS_KEY || '' }} + AWS_SECRET_ACCESS_KEY: ${{ (github.event_name != 'pull_request' || inputs.mq-heavy-labels == '') && secrets.SOLDR_S3_CACHE_SECRET_KEY || '' }} AWS_DEFAULT_REGION: nyc3 run: | if ! command -v aws >/dev/null; then @@ -2025,7 +2062,7 @@ jobs: cross-compile: timeout-minutes: 45 name: Cross Compile 🧰 (${{ needs.check-event.outputs.cross_targets }}) - runs-on: ubuntu-26.04 + runs-on: ${{ github.event_name == 'pull_request' && inputs.mq-heavy-labels != '' && fromJSON(inputs.mq-heavy-labels) || 'ubuntu-26.04' }} needs: [check-event, ci-mode] if: ${{ (github.event_name != 'pull_request' || needs.ci-mode.outputs.fast_path != 'true') && !inputs.flatpak-measurement-only && needs.check-event.outputs.cross_targets != '' }} defaults: @@ -2058,8 +2095,8 @@ jobs: s3-region: nyc3 s3-cache-limit: 50G env: - SOLDR_S3_CACHE_ACCESS_KEY: ${{ secrets.SOLDR_S3_CACHE_ACCESS_KEY }} - SOLDR_S3_CACHE_SECRET_KEY: ${{ secrets.SOLDR_S3_CACHE_SECRET_KEY }} + SOLDR_S3_CACHE_ACCESS_KEY: ${{ (github.event_name != 'pull_request' || inputs.mq-heavy-labels == '') && secrets.SOLDR_S3_CACHE_ACCESS_KEY || '' }} + SOLDR_S3_CACHE_SECRET_KEY: ${{ (github.event_name != 'pull_request' || inputs.mq-heavy-labels == '') && secrets.SOLDR_S3_CACHE_SECRET_KEY || '' }} # The mirror step runs before every package install, and it is Python # now: soldr does not install uv, so without this the step fails with @@ -2142,8 +2179,8 @@ jobs: s3-region: nyc3 s3-cache-limit: 50G env: - SOLDR_S3_CACHE_ACCESS_KEY: ${{ secrets.SOLDR_S3_CACHE_ACCESS_KEY }} - SOLDR_S3_CACHE_SECRET_KEY: ${{ secrets.SOLDR_S3_CACHE_SECRET_KEY }} + SOLDR_S3_CACHE_ACCESS_KEY: ${{ (github.event_name != 'pull_request' || inputs.mq-heavy-labels == '') && secrets.SOLDR_S3_CACHE_ACCESS_KEY || '' }} + SOLDR_S3_CACHE_SECRET_KEY: ${{ (github.event_name != 'pull_request' || inputs.mq-heavy-labels == '') && secrets.SOLDR_S3_CACHE_SECRET_KEY || '' }} - name: Set Up uv 🐍 uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 @@ -2215,8 +2252,8 @@ jobs: s3-region: nyc3 s3-cache-limit: 50G env: - SOLDR_S3_CACHE_ACCESS_KEY: ${{ secrets.SOLDR_S3_CACHE_ACCESS_KEY }} - SOLDR_S3_CACHE_SECRET_KEY: ${{ secrets.SOLDR_S3_CACHE_SECRET_KEY }} + SOLDR_S3_CACHE_ACCESS_KEY: ${{ (github.event_name != 'pull_request' || inputs.mq-heavy-labels == '') && secrets.SOLDR_S3_CACHE_ACCESS_KEY || '' }} + SOLDR_S3_CACHE_SECRET_KEY: ${{ (github.event_name != 'pull_request' || inputs.mq-heavy-labels == '') && secrets.SOLDR_S3_CACHE_SECRET_KEY || '' }} - name: Set Up uv 🐍 uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 @@ -2270,8 +2307,8 @@ jobs: s3-region: nyc3 s3-cache-limit: 50G env: - SOLDR_S3_CACHE_ACCESS_KEY: ${{ secrets.SOLDR_S3_CACHE_ACCESS_KEY }} - SOLDR_S3_CACHE_SECRET_KEY: ${{ secrets.SOLDR_S3_CACHE_SECRET_KEY }} + SOLDR_S3_CACHE_ACCESS_KEY: ${{ (github.event_name != 'pull_request' || inputs.mq-heavy-labels == '') && secrets.SOLDR_S3_CACHE_ACCESS_KEY || '' }} + SOLDR_S3_CACHE_SECRET_KEY: ${{ (github.event_name != 'pull_request' || inputs.mq-heavy-labels == '') && secrets.SOLDR_S3_CACHE_SECRET_KEY || '' }} - name: Set Up uv 🐍 uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 @@ -2328,8 +2365,8 @@ jobs: s3-region: nyc3 s3-cache-limit: 50G env: - SOLDR_S3_CACHE_ACCESS_KEY: ${{ secrets.SOLDR_S3_CACHE_ACCESS_KEY }} - SOLDR_S3_CACHE_SECRET_KEY: ${{ secrets.SOLDR_S3_CACHE_SECRET_KEY }} + SOLDR_S3_CACHE_ACCESS_KEY: ${{ (github.event_name != 'pull_request' || inputs.mq-heavy-labels == '') && secrets.SOLDR_S3_CACHE_ACCESS_KEY || '' }} + SOLDR_S3_CACHE_SECRET_KEY: ${{ (github.event_name != 'pull_request' || inputs.mq-heavy-labels == '') && secrets.SOLDR_S3_CACHE_SECRET_KEY || '' }} - name: Set Up uv 🐍 uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 @@ -2392,8 +2429,8 @@ jobs: s3-region: nyc3 s3-cache-limit: 50G env: - SOLDR_S3_CACHE_ACCESS_KEY: ${{ secrets.SOLDR_S3_CACHE_ACCESS_KEY }} - SOLDR_S3_CACHE_SECRET_KEY: ${{ secrets.SOLDR_S3_CACHE_SECRET_KEY }} + SOLDR_S3_CACHE_ACCESS_KEY: ${{ (github.event_name != 'pull_request' || inputs.mq-heavy-labels == '') && secrets.SOLDR_S3_CACHE_ACCESS_KEY || '' }} + SOLDR_S3_CACHE_SECRET_KEY: ${{ (github.event_name != 'pull_request' || inputs.mq-heavy-labels == '') && secrets.SOLDR_S3_CACHE_SECRET_KEY || '' }} - name: Prepare Darwin standard library run: uv run --no-project ci/macos_test_artifact.py prepare-target @@ -2937,8 +2974,8 @@ jobs: s3-region: nyc3 s3-cache-limit: 50G env: - SOLDR_S3_CACHE_ACCESS_KEY: ${{ secrets.SOLDR_S3_CACHE_ACCESS_KEY }} - SOLDR_S3_CACHE_SECRET_KEY: ${{ secrets.SOLDR_S3_CACHE_SECRET_KEY }} + SOLDR_S3_CACHE_ACCESS_KEY: ${{ (github.event_name != 'pull_request' || inputs.mq-heavy-labels == '') && secrets.SOLDR_S3_CACHE_ACCESS_KEY || '' }} + SOLDR_S3_CACHE_SECRET_KEY: ${{ (github.event_name != 'pull_request' || inputs.mq-heavy-labels == '') && secrets.SOLDR_S3_CACHE_SECRET_KEY || '' }} # The libobs lib-module cache (#3264): PRs restore master's modules, # only master pushes save (a PR-ref cache is unreachable by other PRs). @@ -3251,7 +3288,7 @@ jobs: # otherwise. linux-qt-frontend: name: Qt Frontend Linux 🐧 - runs-on: ubuntu-26.04 + runs-on: ${{ github.event_name == 'pull_request' && inputs.mq-heavy-labels != '' && fromJSON(inputs.mq-heavy-labels) || 'ubuntu-26.04' }} # Independent runner: overlap frontend compilation/tests with Fast Gate. # The reusable workflow still fails if either selected job fails; this # can spend extra Qt runner time when Fast Gate fails (#1581). diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9d1e6d09cb..d517400596 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -76,13 +76,13 @@ jobs: actions: write contents: read - # PRs use hosted runners directly; a queue probe must not allocate on admission. + # Probe on hosted infrastructure, using only immutable BASE code for PR routing. select-mq-runner: name: Select Merge Queue Runner permissions: contents: read actions: read - if: github.event_name != 'pull_request' + if: github.event_name != 'pull_request' || (vars.DELL_PR_ROUTING == 'on' && github.event.pull_request.head.repo.full_name == github.repository) runs-on: ubuntu-latest timeout-minutes: 2 outputs: @@ -91,14 +91,15 @@ jobs: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 id: checkout continue-on-error: true - if: github.event_name == 'merge_group' && vars.MQ_RUNNER_LABELS != '' + if: (github.event_name == 'merge_group' || github.event_name == 'pull_request') && vars.MQ_RUNNER_LABELS != '' with: + ref: ${{ github.event.pull_request.base.sha || github.sha }} sparse-checkout: ci persist-credentials: false - uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 id: uv continue-on-error: true - if: github.event_name == 'merge_group' && vars.MQ_RUNNER_LABELS != '' + if: (github.event_name == 'merge_group' || github.event_name == 'pull_request') && vars.MQ_RUNNER_LABELS != '' with: save-cache: ${{ github.ref == 'refs/heads/master' }} enable-cache: false @@ -106,7 +107,7 @@ jobs: id: mq-probe-app uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3.3.0 continue-on-error: true - if: github.event_name == 'merge_group' && vars.MQ_RUNNER_LABELS != '' && vars.MQ_PROBE_APP_ID != '' + if: (github.event_name == 'merge_group' || github.event_name == 'pull_request') && vars.MQ_RUNNER_LABELS != '' && vars.MQ_PROBE_APP_ID != '' with: app-id: ${{ vars.MQ_PROBE_APP_ID }} private-key: ${{ secrets.MQ_PROBE_APP_PRIVATE_KEY }} @@ -114,8 +115,11 @@ jobs: - name: Check Runner Availability id: select continue-on-error: true - if: github.event_name == 'merge_group' && vars.MQ_RUNNER_LABELS != '' && steps.checkout.outcome == 'success' && steps.uv.outcome == 'success' + if: (github.event_name == 'merge_group' || github.event_name == 'pull_request') && vars.MQ_RUNNER_LABELS != '' && steps.checkout.outcome == 'success' && steps.uv.outcome == 'success' env: + BASE: ${{ github.event.pull_request.base.sha || github.sha }} + DELL_PR_ROUTING: ${{ vars.DELL_PR_ROUTING || 'off' }} + MQ_PR_RESERVE: ${{ vars.MQ_PR_RESERVE }} MQ_RUNNER_LABELS: ${{ vars.MQ_RUNNER_LABELS }} MQ_RUNNER_MODE: ${{ vars.MQ_RUNNER_MODE }} MQ_RUNNER_MIN_IDLE: ${{ vars.MQ_RUNNER_MIN_IDLE }} @@ -123,7 +127,13 @@ jobs: MQ_RUNNER_DEMAND: ${{ vars.MQ_RUNNER_DEMAND || '6' }} MQ_RUNNER_STATUS_TOKEN: ${{ steps.mq-probe-app.outputs.token || secrets.MQ_RUNNER_STATUS_TOKEN }} GITHUB_TOKEN: ${{ github.token }} - run: uv run --no-project ci/select_mq_runner.py + run: | + [[ "$BASE" =~ ^[0-9a-f]{40}$ ]] + trusted=$(mktemp -d "$RUNNER_TEMP/dell-selector.XXXXXX") + for script in select_mq_runner.py pr_attestation.py; do + git show "$BASE:ci/$script" > "$trusted/$script" || exit 1 + done + uv run --no-project "$trusted/select_mq_runner.py" - name: Report Probe Setup Failure if: ${{ !cancelled() && (steps.checkout.outcome == 'failure' || steps.uv.outcome == 'failure' || steps.select.outcome == 'failure') }} run: echo '::warning::Merge-queue runner probe could not complete; using hosted runners.' @@ -196,7 +206,26 @@ jobs: native-cache-reuse-source: ${{ github.event_name == 'workflow_dispatch' && inputs.native-cache-reuse-source || '' }} native-cache-transport-probe: ${{ github.event_name == 'workflow_dispatch' && inputs.native-cache-transport-probe }} native-cache-probe: ${{ github.event_name == 'workflow_dispatch' && inputs.native-cache-probe }} - secrets: inherit + # Routed PR jobs receive only the read-only event GITHUB_TOKEN. + secrets: + GPU_PRIORITY_VAL: ${{ (github.event_name != 'pull_request' || needs.select-mq-runner.outputs.heavy-labels == '') && secrets.GPU_PRIORITY_VAL || '' }} + MACOS_KEYCHAIN_PASSWORD: ${{ (github.event_name != 'pull_request' || needs.select-mq-runner.outputs.heavy-labels == '') && secrets.MACOS_KEYCHAIN_PASSWORD || '' }} + MACOS_NOTARIZATION_PASSWORD: ${{ (github.event_name != 'pull_request' || needs.select-mq-runner.outputs.heavy-labels == '') && secrets.MACOS_NOTARIZATION_PASSWORD || '' }} + MACOS_NOTARIZATION_USERNAME: ${{ (github.event_name != 'pull_request' || needs.select-mq-runner.outputs.heavy-labels == '') && secrets.MACOS_NOTARIZATION_USERNAME || '' }} + MACOS_SIGNING_CERT: ${{ (github.event_name != 'pull_request' || needs.select-mq-runner.outputs.heavy-labels == '') && secrets.MACOS_SIGNING_CERT || '' }} + MACOS_SIGNING_CERT_PASSWORD: ${{ (github.event_name != 'pull_request' || needs.select-mq-runner.outputs.heavy-labels == '') && secrets.MACOS_SIGNING_CERT_PASSWORD || '' }} + MACOS_SIGNING_IDENTITY: ${{ (github.event_name != 'pull_request' || needs.select-mq-runner.outputs.heavy-labels == '') && secrets.MACOS_SIGNING_IDENTITY || '' }} + MACOS_SIGNING_PROVISIONING_PROFILE: ${{ (github.event_name != 'pull_request' || needs.select-mq-runner.outputs.heavy-labels == '') && secrets.MACOS_SIGNING_PROVISIONING_PROFILE || '' }} + RESTREAM_CLIENTID: ${{ (github.event_name != 'pull_request' || needs.select-mq-runner.outputs.heavy-labels == '') && secrets.RESTREAM_CLIENTID || '' }} + RESTREAM_HASH: ${{ (github.event_name != 'pull_request' || needs.select-mq-runner.outputs.heavy-labels == '') && secrets.RESTREAM_HASH || '' }} + SOLDR_S3_CACHE_ACCESS_KEY: ${{ (github.event_name != 'pull_request' || needs.select-mq-runner.outputs.heavy-labels == '') && secrets.SOLDR_S3_CACHE_ACCESS_KEY || '' }} + SOLDR_S3_CACHE_SECRET_KEY: ${{ (github.event_name != 'pull_request' || needs.select-mq-runner.outputs.heavy-labels == '') && secrets.SOLDR_S3_CACHE_SECRET_KEY || '' }} + TWITCH_CLIENT_ID: ${{ (github.event_name != 'pull_request' || needs.select-mq-runner.outputs.heavy-labels == '') && secrets.TWITCH_CLIENT_ID || '' }} + TWITCH_HASH: ${{ (github.event_name != 'pull_request' || needs.select-mq-runner.outputs.heavy-labels == '') && secrets.TWITCH_HASH || '' }} + YOUTUBE_CLIENTID: ${{ (github.event_name != 'pull_request' || needs.select-mq-runner.outputs.heavy-labels == '') && secrets.YOUTUBE_CLIENTID || '' }} + YOUTUBE_CLIENTID_HASH: ${{ (github.event_name != 'pull_request' || needs.select-mq-runner.outputs.heavy-labels == '') && secrets.YOUTUBE_CLIENTID_HASH || '' }} + YOUTUBE_SECRET: ${{ (github.event_name != 'pull_request' || needs.select-mq-runner.outputs.heavy-labels == '') && secrets.YOUTUBE_SECRET || '' }} + YOUTUBE_SECRET_HASH: ${{ (github.event_name != 'pull_request' || needs.select-mq-runner.outputs.heavy-labels == '') && secrets.YOUTUBE_SECRET_HASH || '' }} permissions: actions: read contents: write diff --git a/ci/select_mq_runner.py b/ci/select_mq_runner.py index a5db566b73..f83b93d193 100644 --- a/ci/select_mq_runner.py +++ b/ci/select_mq_runner.py @@ -1,5 +1,5 @@ #!/usr/bin/env -S uv run --script -"""Probe opt-in queue runners; every unavailable/unknown state uses hosted CI.""" +"""Probe opt-in queue/member PR runners; unknown states always use hosted CI.""" from __future__ import annotations @@ -102,7 +102,7 @@ def retry_delay(self, error: Exception, attempt: int) -> float: pass return min(MAX_RETRY_AFTER, max(0, delay)) - def get(self, path: str, *, token: str | None = None) -> dict: + def get(self, path: str, *, token: str | None = None, retry: bool = True) -> dict: request = Request( f"{self.api_url}/{path}", headers={ @@ -111,14 +111,19 @@ def get(self, path: str, *, token: str | None = None) -> dict: "X-GitHub-Api-Version": "2022-11-28", }, ) - for attempt in range(MAX_RETRIES + 1): + public_login = path.split("/public_members/", 1)[1] if "/public_members/" in path else None + retries = MAX_RETRIES if retry else 0 + for attempt in range(retries + 1): timeout = min(API_TIMEOUT, self.time_left()) if self.remaining <= 0: raise RequestBudgetExhausted("request budget exhausted") self.remaining -= 1 # Retries consume the same shared budget. try: with urlopen(request, timeout=timeout) as response: - result = json.load(response) + if public_login and getattr(response, "status", None) == 204: + result = {"public_member": True, "user": {"login": public_login}} + else: + result = json.load(response) self.time_left() return result except SelectionDeadline: @@ -126,12 +131,14 @@ def get(self, path: str, *, token: str | None = None) -> dict: except (HTTPError, URLError, TimeoutError, ConnectionError) as error: if isinstance(error, HTTPError): error.close() + if public_login and error.code == 404: + return {"public_member": False} if error.code != 429 and not 500 <= error.code < 600: raise self.time_left() if self.remaining <= 0: raise RequestBudgetExhausted("request budget exhausted") from error - if attempt == MAX_RETRIES: + if attempt == retries: raise self.sleep(min(self.retry_delay(error, attempt), self.time_left())) @@ -307,14 +314,14 @@ def job_commitments(jobs: list[dict], wanted: set[str]) -> tuple[int, bool, bool return queued, dell, decided -def active_runs(repo: str, probe: Probe, token: str) -> dict: +def active_runs(repo: str, probe: Probe, token: str, event: str = "merge_group") -> dict: """Keep only active run snapshots; terminal snapshots release reservations.""" runs = {} finished = set() for status in ("queued", "in_progress"): try: entries = probe.entries( - f"repos/{repo}/actions/workflows/ci.yml/runs?event=merge_group&status={status}", + f"repos/{repo}/actions/workflows/ci.yml/runs?event={event}&status={status}", "workflow_runs", token, ) @@ -338,7 +345,7 @@ def active_runs(repo: str, probe: Probe, token: str) -> dict: if ( type(run_id) is not int or run_id < 1 - or run["event"] != "merge_group" + or run["event"] != event or run["status"] not in ("queued", "in_progress") ): raise ValueError(f"runs probe ({status}): invalid active run") @@ -364,6 +371,12 @@ def inflight_commitments( if current_id < 1 or not token: raise ValueError("missing Actions probe identity") runs = active_runs(repo, probe, token) + if os.environ.get("DELL_PR_ROUTING") == "on": + # Queue runs sort before every PR, even a PR created earlier. Inspect + # live PR jobs for both events so the queue sees their queued demand. + runs = {key: ((0, *value[0]), value[1]) for key, value in runs.items()} + prs = active_runs(repo, probe, token, "pull_request") + runs.update({key: ((1, *value[0]), value[1]) for key, value in prs.items()}) current = runs[current_id] # An incomplete/eventually consistent list is unsafe. queued = committed = unprobed = 0 current = current[0] @@ -403,17 +416,18 @@ def inflight_commitments( def admission_available( - repo: str, wanted: set[str], probe: Probe, online: int, idle: int, required: int + repo: str, wanted: set[str], probe: Probe, online: int, idle: int, required: int, + reserve: int = 0, ) -> tuple[bool, str]: """The idle floor and in-flight run cap must both admit this run.""" - if idle < required: + if idle - reserve < required: return False, "" queued, committed = inflight_commitments(repo, wanted, probe, required) - idle_slots = (idle - queued) // required + idle_slots = (idle - queued - reserve) // required # The total cap includes runs whose running jobs already reduce idle. Using # idle for this cap would charge those busy runners twice. Idle slots below # independently limit how much new work this snapshot can admit. - limit = online // required + limit = (online - reserve) // required counts = ( f"; queued={queued}, committed={committed}, run_limit={limit}" f", idle_run_slots={max(0, idle_slots)}" @@ -443,6 +457,34 @@ def requested_labels(labels: str) -> list[str]: return requested +def eligible_pr(repo: str, probe: Probe) -> bool: + """Read identity only from the Actions payload and authenticated membership. + + The workflow extracts this module and pr_attestation from immutable BASE. + Public membership's authenticated 204 and private membership's active state + follow the admission verifier's contract; failures never grant access. + """ + from pr_attestation import LOGIN, member + + payload = json.loads(Path(os.environ["GITHUB_EVENT_PATH"]).read_text()) + pr = payload["pull_request"] + base = pr["base"]["repo"] + head = pr["head"]["repo"] + repository = payload["repository"] + if repo.split("/", 1)[0] != "obs-rust" or any( + value["full_name"] != repo for value in (base, head, repository) + ): + return False + if type(base["id"]) is not int or base["id"] < 1 or not ( + base["id"] == head["id"] == repository["id"] + ): + return False + login = pr["user"]["login"] + if not isinstance(login, str) or not LOGIN.fullmatch(login): + return False + return member(login, lambda path: probe.get(path, retry=False)) is None + + def select_labels( event: str, labels: str, @@ -460,7 +502,8 @@ def select_labels( each HTTP request stay bounded; runners visible in both scopes count only once. In-flight commitments and older undecided runs reserve capacity. """ - if event != "merge_group" or not labels: + is_pr = event == "pull_request" + if (event != "merge_group" and not (is_pr and os.environ.get("DELL_PR_ROUTING") == "on")) or not labels: return decision("opt-in merge-group runner selection is disabled") step = "configuration" try: @@ -471,6 +514,15 @@ def select_labels( return decision("runner probe token is missing") wanted = {label.casefold() for label in requested} probe = Probe(token, api_url) + reserve = 0 + if is_pr: + step = "PR eligibility" + if not eligible_pr(repo, probe): + return decision("PR author is not a verified same-repository active member") + step = "PR reserve configuration" + reserve = int(os.environ.get("MQ_PR_RESERVE") or demand or "1") + if reserve < 0: + raise ValueError("PR reserve must be nonnegative") matched = {} unavailable = [] owner = repo.split("/", 1)[0] @@ -497,11 +549,14 @@ def select_labels( busy = sum(matched.values()) idle = len(matched) - busy counts = f"online={len(matched)}, busy={busy}, idle={idle}, required={required}" + if is_pr: + counts += f", pr_reserve={reserve}" if unavailable: counts += f"; unavailable scopes: {', '.join(unavailable)}" step = "in-flight commitments probe" + options = {"reserve": reserve} if is_pr else {} admitted, commitments = admission_available( - repo, wanted, probe, len(matched), idle, required + repo, wanted, probe, len(matched), idle, required, **options ) probe.time_left() counts += commitments diff --git a/ci/test_ci_scope.py b/ci/test_ci_scope.py index 3100774289..0300c96de4 100644 --- a/ci/test_ci_scope.py +++ b/ci/test_ci_scope.py @@ -317,6 +317,7 @@ def test_queue_runner_selection_preserves_hosted_fallbacks(self): "rust-cross-lint", "rust-cross-lint-release", "rust-tests-macos-build", "rust-linux-tier2-build", "rust-linux-tier2", "rust-windows-link", "cmocka-link", } + pr_only = {"rust-tests-linux", "cross-compile", "linux-qt-frontend"} light = { "ci.yml": { "cancel-superseded": "ubuntu-latest", "ci-gate": "ubuntu-latest", @@ -324,8 +325,7 @@ def test_queue_runner_selection_preserves_hosted_fallbacks(self): "tier2-lane-proof": "ubuntu-26.04", }, "build-project.yaml": dict.fromkeys(( - "check-event", "ci-mode", "rust-tests-linux", "cross-compile", - "rust-cross-proof", "linux-qt-frontend", + "check-event", "ci-mode", "rust-cross-proof", ), "ubuntu-26.04"), "check-format.yaml": {"checks": "ubuntu-latest"}, } @@ -343,16 +343,17 @@ def test_queue_runner_selection_preserves_hosted_fallbacks(self): if not runner: continue selected = "inputs.mq-heavy-labels" in runner[1] - self.assertEqual(selected, name == "build-project.yaml" and job in heavy, job) + self.assertEqual(selected, name == "build-project.yaml" and job in heavy | pr_only, job) if selected: selected_jobs.add(job) + prefix = "github.event_name == 'pull_request' && " if job in pr_only else "" self.assertIn( - "runs-on: ${{ inputs.mq-heavy-labels != '' && " + "runs-on: ${{ " + prefix + "inputs.mq-heavy-labels != '' && " "fromJSON(inputs.mq-heavy-labels) || 'ubuntu-26.04' }}", block, ) else: self.assertNotIn("mq-", runner[1]) - self.assertEqual(selected_jobs, heavy if name == "build-project.yaml" else set()) + self.assertEqual(selected_jobs, (heavy | pr_only) if name == "build-project.yaml" else set()) self.assertNotIn("mq-runner-labels", text) if name == "build-project.yaml": self.assertIn("runs-on: ${{ matrix.os }}", blocks["ubuntu-build"]) @@ -386,7 +387,7 @@ def test_queue_runner_probe_and_reusable_inputs(self): self.assertIn("GITHUB_TOKEN: ${{ github.token }}", probe) self.assertIn("if: github.event_name != 'pull_request'", probe) self.assertIn("!cancelled()", blocks["build-project"].split(" uses:", 1)[0]) - self.assertIn("github.event_name == 'merge_group' && vars.MQ_RUNNER_LABELS != ''", probe) + self.assertIn("(github.event_name == 'merge_group' || github.event_name == 'pull_request') && vars.MQ_RUNNER_LABELS != ''", probe) steps = dict(re.findall( r"^ - name: ([^\n]+)\n(.*?)(?=^ - |\Z)", probe, re.MULTILINE | re.DOTALL )) @@ -395,7 +396,7 @@ def test_queue_runner_probe_and_reusable_inputs(self): self.assertIn("id: mq-probe-app", app) self.assertIn("continue-on-error: true", app) self.assertIn( - "if: github.event_name == 'merge_group' && vars.MQ_RUNNER_LABELS != ''" + "if: (github.event_name == 'merge_group' || github.event_name == 'pull_request') && vars.MQ_RUNNER_LABELS != ''" " && vars.MQ_PROBE_APP_ID != ''\n", app, ) for setting in ( @@ -411,7 +412,7 @@ def test_queue_runner_probe_and_reusable_inputs(self): ) for variable in ("MQ_RUNNER_MODE", "MQ_RUNNER_MIN_IDLE"): self.assertIn(f"{variable}: ${{{{ vars.{variable} }}}}", steps["Check Runner Availability"]) - self.assertIn("run: uv run --no-project ci/select_mq_runner.py", probe) + self.assertIn('uv run --no-project "$trusted/select_mq_runner.py"', probe) self.assertIn("heavy-labels: ${{ steps.select.outputs.heavy-labels }}", probe) self.assertNotRegex(probe, r"(?m)^ continue-on-error:") self.assertEqual(probe.count("continue-on-error: true"), 4) diff --git a/ci/test_member_pr_routing.py b/ci/test_member_pr_routing.py index d921203157..f975b9fe6a 100644 --- a/ci/test_member_pr_routing.py +++ b/ci/test_member_pr_routing.py @@ -150,7 +150,16 @@ def test_dell_pr_jobs_do_not_receive_repository_secrets(self): for line in block.splitlines(): if "secrets." in line and "secrets.GITHUB_TOKEN" not in line: with self.subTest(job=name, line=line): - self.assertIn("github.event_name != 'pull_request' && secrets.", line) + self.assertIn("(github.event_name != 'pull_request' || inputs.mq-heavy-labels == '') && secrets.", line) + + def test_caller_strips_secrets_before_dell_pr_reusable_workflow(self): + ci = (ROOT / ".github/workflows/ci.yml").read_text() + block = ci.split(" build-project:\n", 1)[1].split(" flatpak-runtime-reuse:\n", 1)[0] + self.assertNotIn("secrets: inherit", block) + build = (ROOT / ".github/workflows/build-project.yaml").read_text() + names = set(re.findall(r"secrets\.([A-Z0-9_]+)", build)) + for name in names: + self.assertIn(name + ": ${{ (github.event_name != 'pull_request' || needs.select-mq-runner.outputs.heavy-labels == '') && secrets." + name + " || '' }}", block) def test_fast_gate_pr_routing_keeps_queue_hosted(self): text = (ROOT / ".github/workflows/build-project.yaml").read_text() diff --git a/docs/ci-tooling.md b/docs/ci-tooling.md index 26c625434e..13117ab1c8 100644 --- a/docs/ci-tooling.md +++ b/docs/ci-tooling.md @@ -1335,8 +1335,8 @@ required runner capacity (or its larger observed queued-job count) and one run s Warnings name the failed probe step and include validation reasons, including incomplete collections and limits. An unset or empty variable skips the -probe steps. Outside `merge_group` the selection is empty and no API call runs. -Pull requests, master pushes, manual/release runs, +probe steps. Outside `merge_group` the selection is empty unless member PR +routing is explicitly enabled below. Master pushes, manual/release runs, and macOS/Windows jobs keep their current hosted runners. Linux matrix entries retain their matrix and hosted fallback. Required job/check names are unchanged. Rust Tier 2 Linux build, its OFF/ON test jobs, Rust Cross Lint @@ -1347,6 +1347,49 @@ Event Data, CI Mode, Format, Fast Gate, Cross Compile, Qt, app builds, proof publication, queue cancellation and CI Gate use hosted runners. Empty outputs and probe/setup errors keep every heavy job hosted. +#### Idle Dell capacity for member PRs (default off) + +The maintainer approved same-repository member PR routing on 2026-11-19 +(Refs #4839, #4896). Set the repository Actions variable `DELL_PR_ROUTING=on` +to opt in; unset, `off`, or any other value keeps PR jobs hosted. This change +does not set any repository variables. Forks, non-members, pending memberships, +API errors and `pull_request_target` events always stay hosted. Eligibility uses +the immutable `pull_request` event's head/base repository identities and author, +then the admission verifier's authenticated public/private active-membership +check against obs-rust. The hosted selector checks out the PR's BASE SHA and +extracts both `ci/select_mq_runner.py` and `ci/pr_attestation.py` with `git show` +into an isolated directory. It never imports the PR's routing code. The runner +probe App/token must be able to read membership; denied private membership uses +hosted without expanding permissions. Probe credentials remain on hosted runners. + +`MQ_PR_RESERVE` keeps matching idle runners free for the queue, defaulting to +`MQ_RUNNER_DEMAND` (currently `6`). A PR must fit its entire required capacity +after queued commitments and this reserve are subtracted. Unset/empty uses the +default; a nonnegative integer overrides it, and invalid values select hosted. +On a six-runner Dell with demand six and the default reserve six, PRs therefore +remain hosted even when idle. A maintainer must explicitly tune the reserve or +increase the pool to enable useful PR capacity. `MQ_RUNNER_LABELS`, +`MQ_RUNNER_MIN_IDLE`, `MQ_RUNNER_DEMAND` and `MQ_RUNNER_MODE` retain their meanings. +No variable is changed by merging this default-off feature. + +When enabled, both selectors account for live PR and queue job commitments. +Undecided queue runs have priority over every PR, regardless of creation time; +undecided PRs use oldest-first ordering. Queue admission does not subtract the +PR reserve. This remains a bounded snapshot, with the same API visibility and +simultaneous-selection limitations as queue admission, not a preemption system. +Already-running PR jobs finish normally. Turning the kill switch off stops new +PR routing; allow existing PR jobs to drain before relying on queue-only census. + +Existing heavy Linux jobs accept the selected labels on PR events. Fast Gate, +Cross Compile and Qt Frontend Linux also accept them for PRs only; their queue +and other-event placement is unchanged. Event Data, CI Mode and lightweight +admission/gate jobs stay hosted, and macOS/Windows jobs keep their native hosts. +The reusable workflow caller passes empty repository secrets to Dell-routed PR +runs, and selected jobs also suppress credential inputs on that path. They retain +only the event's `GITHUB_TOKEN`, with `contents: read` and `actions: read`; no +permission increase or container-isolation change is introduced. Hosted/non-PR +secret handling stays unchanged. No new workflow files or required checks. + Both the default threshold mode and `MQ_RUNNER_MODE=prefer` now require idle capacity. Prefer remains accepted for existing configurations, but no longer queues work on busy runners or bypasses the capacity cap. Unknown modes diff --git a/docs/features.md b/docs/features.md index b20d9f22ec..5a3acb4496 100644 --- a/docs/features.md +++ b/docs/features.md @@ -3040,6 +3040,12 @@ area. ## Development builds +- **Idle Dell capacity for member PRs.** Maintainers can opt in with + `DELL_PR_ROUTING=on`; only same-repository PRs by verified active obs-rust + members qualify. `MQ_PR_RESERVE` keeps queue capacity free, and uncertain + eligibility or capacity stays hosted. Default off. *Linux CI*, 2026-11-19. + Issue [#4896](https://github.com/obs-rust/obs-studio/issues/4896). + - **Dated conversion tracker.** The README shows the libobs Rust percentage, source date and refresh workflow status beside the conversion graph, making a stalled tracker visible. *All*, 2026-11-08. From cb644e69d15af61891fb7fb6addbc064f2eecc2a Mon Sep 17 00:00:00 2001 From: Zach Vorhies Date: Thu, 19 Nov 2026 11:24:57 -0800 Subject: [PATCH 3/6] ci: let members' same-repo PRs use idle Dell capacity (default off) Charge PR commitments by MQ_PR_DEMAND (default 13) while preserving the default-off MQ path. Refs #4839 #4896 --- .github/workflows/ci.yml | 1 + ci/select_mq_runner.py | 42 ++++++++++++++++++++++++++++-------- ci/test_member_pr_routing.py | 41 +++++++++++++++++++++++++++++++++++ docs/ci-tooling.md | 20 ++++++++++++----- 4 files changed, 90 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d517400596..6c3777eb10 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -120,6 +120,7 @@ jobs: BASE: ${{ github.event.pull_request.base.sha || github.sha }} DELL_PR_ROUTING: ${{ vars.DELL_PR_ROUTING || 'off' }} MQ_PR_RESERVE: ${{ vars.MQ_PR_RESERVE }} + MQ_PR_DEMAND: ${{ vars.MQ_PR_DEMAND }} MQ_RUNNER_LABELS: ${{ vars.MQ_RUNNER_LABELS }} MQ_RUNNER_MODE: ${{ vars.MQ_RUNNER_MODE }} MQ_RUNNER_MIN_IDLE: ${{ vars.MQ_RUNNER_MIN_IDLE }} diff --git a/ci/select_mq_runner.py b/ci/select_mq_runner.py index f83b93d193..17490601b3 100644 --- a/ci/select_mq_runner.py +++ b/ci/select_mq_runner.py @@ -25,6 +25,9 @@ # Two runner scopes, two run statuses, and a jobs collection for each active run. # Include pagination and retries, while retaining the shared wall-clock deadline. MAX_REQUESTS = (4 + MAX_ACTIVE_RUNS) * MAX_PAGES * (MAX_RETRIES + 1) +# All expanded PR-routable jobs, including both sequential Tier 2 phases. +# ci/test_member_pr_routing.py checks this bound against the workflow inventory. +PR_RUNNER_DEMAND = 13 class RequestBudgetExhausted(ValueError): @@ -357,7 +360,8 @@ def active_runs(repo: str, probe: Probe, token: str, event: str = "merge_group") def inflight_commitments( - repo: str, wanted: set[str], probe: Probe, demand: int = 1 + repo: str, wanted: set[str], probe: Probe, demand: int = 1, + *, mq_required: int = 0, pr_required: int = 0, ) -> tuple[int, int]: """Count queued Dell jobs and active Dell/older undecided runs. @@ -371,7 +375,8 @@ def inflight_commitments( if current_id < 1 or not token: raise ValueError("missing Actions probe identity") runs = active_runs(repo, probe, token) - if os.environ.get("DELL_PR_ROUTING") == "on": + mixed = os.environ.get("DELL_PR_ROUTING") == "on" + if mixed: # Queue runs sort before every PR, even a PR created earlier. Inspect # live PR jobs for both events so the queue sees their queued demand. runs = {key: ((0, *value[0]), value[1]) for key, value in runs.items()} @@ -383,6 +388,8 @@ def inflight_commitments( for run_id, (priority, _) in sorted(runs.items(), key=lambda item: item[1][0]): if run_id == current_id: continue + weight = (pr_required if priority[0] == 1 else mq_required) if mixed else 1 + peer_demand = weight if mixed else demand try: jobs = probe.entries( f"repos/{repo}/actions/runs/{run_id}/jobs?filter=latest", "jobs", token @@ -391,8 +398,8 @@ def inflight_commitments( except RequestBudgetExhausted as error: unprobed += 1 observed, _, _ = job_commitments(error.entries, wanted) - queued += max(observed, demand) - committed += 1 + queued += max(observed, peer_demand) + committed += weight continue except ValueError as error: raise ValueError(f"jobs probe (run {run_id}): {error}") from error @@ -405,11 +412,15 @@ def inflight_commitments( queued += pending # Count known Dell runs regardless of age. Unknown older runs also # reserve slots; hosted or finished Dell work releases their reservation. - committed += dell or (not decided and priority < current) + committed += weight * (dell or (not decided and priority < current)) if unprobed: + reservation = ( + "reserving each unprobed active run's required capacity" + if mixed else f"reserving at least {demand} runners per unprobed active run" + ) warning( f"jobs probe: request budget exhausted; unprobed={unprobed}; " - f"reserving at least {demand} runners per unprobed active run", + f"{reservation}", hosted=False, ) return queued, committed @@ -417,21 +428,26 @@ def inflight_commitments( def admission_available( repo: str, wanted: set[str], probe: Probe, online: int, idle: int, required: int, - reserve: int = 0, + reserve: int = 0, mq_required: int = 0, pr_required: int = 0, ) -> tuple[bool, str]: """The idle floor and in-flight run cap must both admit this run.""" if idle - reserve < required: return False, "" - queued, committed = inflight_commitments(repo, wanted, probe, required) + mixed = os.environ.get("DELL_PR_ROUTING") == "on" + options = {"mq_required": mq_required, "pr_required": pr_required} if mixed else {} + queued, committed = inflight_commitments(repo, wanted, probe, required, **options) idle_slots = (idle - queued - reserve) // required # The total cap includes runs whose running jobs already reduce idle. Using # idle for this cap would charge those busy runners twice. Idle slots below # independently limit how much new work this snapshot can admit. - limit = (online - reserve) // required + limit = online - reserve if mixed else (online - reserve) // required counts = ( f"; queued={queued}, committed={committed}, run_limit={limit}" f", idle_run_slots={max(0, idle_slots)}" ) + if mixed: + counts += "; commitments measured in runner capacity" + return idle_slots >= 1 and committed + required <= limit, counts return idle_slots >= 1 and committed < limit, counts @@ -509,6 +525,12 @@ def select_labels( try: requested = requested_labels(labels) required = required_capacity(min_idle, demand, mode) + mq_required = required + pr_required = required + if os.environ.get("DELL_PR_ROUTING") == "on": + pr_required = max(required, required_capacity("1", os.environ.get("MQ_PR_DEMAND") or str(PR_RUNNER_DEMAND))) + if is_pr: + required = pr_required if not token: warning("MQ_RUNNER_STATUS_TOKEN is missing") return decision("runner probe token is missing") @@ -555,6 +577,8 @@ def select_labels( counts += f"; unavailable scopes: {', '.join(unavailable)}" step = "in-flight commitments probe" options = {"reserve": reserve} if is_pr else {} + if os.environ.get("DELL_PR_ROUTING") == "on": + options.update(mq_required=mq_required, pr_required=pr_required) admitted, commitments = admission_available( repo, wanted, probe, len(matched), idle, required, **options ) diff --git a/ci/test_member_pr_routing.py b/ci/test_member_pr_routing.py index f975b9fe6a..f36af01e8a 100644 --- a/ci/test_member_pr_routing.py +++ b/ci/test_member_pr_routing.py @@ -67,6 +67,7 @@ def respond(request, **kwargs): "GITHUB_EVENT_NAME": event, "GITHUB_EVENT_PATH": str(event_path), "GITHUB_RUN_ID": "20", "GITHUB_TOKEN": "actions-token", "DELL_PR_ROUTING": enabled, "MQ_PR_RESERVE": reserve, + "MQ_PR_DEMAND": getattr(self, "pr_demand", "6"), }, clear=True), patch.object(selector, "urlopen", side_effect=respond), patch.object(selector.time, "sleep"): return selector.select_labels(event, LABELS, "probe-token", "obs-rust/obs-studio", demand="6") @@ -74,6 +75,16 @@ def test_member_with_idle_capacity_uses_dell(self): self.assertEqual(self.select(), LABELS) self.assertIn("/orgs/obs-rust/memberships/alice", self.paths) + def test_default_pr_demand_preserves_reserve_for_full_inventory(self): + self.pr_demand = "" + self.assertEqual(self.select(capacity=18), "") + self.assertEqual(self.select(capacity=19), LABELS) + + def test_invalid_pr_demand_uses_hosted(self): + for demand in ("0", "-1", "bad"): + self.pr_demand = demand + self.assertEqual(self.select(), "") + def test_fork_uses_hosted_without_api(self): self.assertEqual(self.select(head="fork/obs-studio"), "") self.assertEqual(self.paths, []) @@ -128,8 +139,38 @@ def test_existing_pr_dell_jobs_reduce_capacity_for_prs_and_queue(self): for event in ("pull_request", "merge_group"): self.assertEqual(self.select(event=event, peers=[peer], jobs=jobs), "") + def test_mixed_demands_admit_only_one_queue_run_beside_large_pr(self): + self.pr_demand = "13" + pr = {"id": 10, "event": "pull_request", "status": "in_progress", + "created_at": "2026-11-19T11:59:00Z"} + mq = {"id": 15, "event": "merge_group", "status": "in_progress", + "created_at": "2026-11-19T11:59:30Z"} + # Idle census can lag a routing choice: charge the PR's full 13-slot + # commitment, not one six-slot queue run, even before its jobs start. + jobs = {10: [{"name": "Fast Gate ⚡", "status": "in_progress", + "conclusion": None, "labels": json.loads(LABELS)}]} + self.assertEqual(self.select(event="merge_group", capacity=19, peers=[pr], jobs=jobs), LABELS) + self.assertEqual(self.select(event="merge_group", capacity=19, peers=[pr, mq], jobs=jobs), "") + class MemberPrWorkflowTest(unittest.TestCase): + def test_default_pr_demand_covers_expanded_routed_jobs(self): + from tier2_matrix import matrix + + text = (ROOT / ".github/workflows/build-project.yaml").read_text() + blocks = dict(re.findall(r"^ ([\w-]+):\n(.*?)(?=^ [\w-]+:|\Z)", text, re.M | re.S)) + count = 0 + for name, block in blocks.items(): + if not re.search(r"runs-on: .*inputs.mq-heavy-labels", block): + continue + if name == "rust-linux-tier2": + scheduling = matrix("pull_request") + count += len(scheduling["mode"]) * len(scheduling["part"]) + else: + variants = re.search(r"^ (?:target|alias): \[([^\]]+)\]", block, re.M) + count += len(variants[1].split(",")) if variants else 1 + self.assertEqual(selector.PR_RUNNER_DEMAND, count) + def test_base_owned_selector_and_membership_helper(self): ci = (ROOT / ".github/workflows/ci.yml").read_text() probe = ci.split(" select-mq-runner:\n", 1)[1].split(" cancel-superseded:\n", 1)[0] diff --git a/docs/ci-tooling.md b/docs/ci-tooling.md index 13117ab1c8..f62b1ab580 100644 --- a/docs/ci-tooling.md +++ b/docs/ci-tooling.md @@ -1366,16 +1366,26 @@ hosted without expanding permissions. Probe credentials remain on hosted runners `MQ_RUNNER_DEMAND` (currently `6`). A PR must fit its entire required capacity after queued commitments and this reserve are subtracted. Unset/empty uses the default; a nonnegative integer overrides it, and invalid values select hosted. -On a six-runner Dell with demand six and the default reserve six, PRs therefore -remain hosted even when idle. A maintainer must explicitly tune the reserve or -increase the pool to enable useful PR capacity. `MQ_RUNNER_LABELS`, +PRs have a separate `MQ_PR_DEMAND`: unset/empty defaults to `13`, the complete +expanded PR job inventory, including both sequential Tier 2 phases. The PR's +requirement is the maximum of this value and the existing MQ requirement. +Invalid/nonpositive values use hosted. This conservative default covers Fast +Gate, Cross Compile, Qt and link jobs in addition to the existing heavy jobs; +the inventory regression must change when that topology changes. A maintainer +may tune it to measured scoped PR concurrency, but must not set it below the +actual concurrent job demand. With default PR demand thirteen and reserve six, +nineteen matching idle runners are needed. A six-runner Dell stays hosted until +the maintainer explicitly tunes both PR settings for its workload or expands +the pool. `MQ_RUNNER_LABELS`, `MQ_RUNNER_MIN_IDLE`, `MQ_RUNNER_DEMAND` and `MQ_RUNNER_MODE` retain their meanings. No variable is changed by merging this default-off feature. When enabled, both selectors account for live PR and queue job commitments. Undecided queue runs have priority over every PR, regardless of creation time; -undecided PRs use oldest-first ordering. Queue admission does not subtract the -PR reserve. This remains a bounded snapshot, with the same API visibility and +undecided PRs use oldest-first ordering. Commitments are weighted by each +event's required runner capacity, so a large PR cannot count as one small MQ +run. Queue admission does not subtract the PR reserve. This remains a bounded +snapshot, with the same API visibility and simultaneous-selection limitations as queue admission, not a preemption system. Already-running PR jobs finish normally. Turning the kill switch off stops new PR routing; allow existing PR jobs to drain before relying on queue-only census. From 7be7186aedae8ff71da2743e9094e9f502444d21 Mon Sep 17 00:00:00 2001 From: Zach Vorhies Date: Thu, 19 Nov 2026 11:26:54 -0800 Subject: [PATCH 4/6] ci: keep selector helpers within complexity policy (Refs #4896) --- ci/select_mq_runner.py | 90 +++++++++++++++++++++++++++--------------- 1 file changed, 58 insertions(+), 32 deletions(-) diff --git a/ci/select_mq_runner.py b/ci/select_mq_runner.py index 17490601b3..706536fcaa 100644 --- a/ci/select_mq_runner.py +++ b/ci/select_mq_runner.py @@ -114,7 +114,6 @@ def get(self, path: str, *, token: str | None = None, retry: bool = True) -> dic "X-GitHub-Api-Version": "2022-11-28", }, ) - public_login = path.split("/public_members/", 1)[1] if "/public_members/" in path else None retries = MAX_RETRIES if retry else 0 for attempt in range(retries + 1): timeout = min(API_TIMEOUT, self.time_left()) @@ -123,10 +122,7 @@ def get(self, path: str, *, token: str | None = None, retry: bool = True) -> dic self.remaining -= 1 # Retries consume the same shared budget. try: with urlopen(request, timeout=timeout) as response: - if public_login and getattr(response, "status", None) == 204: - result = {"public_member": True, "user": {"login": public_login}} - else: - result = json.load(response) + result = response_data(path, response) self.time_left() return result except SelectionDeadline: @@ -134,8 +130,6 @@ def get(self, path: str, *, token: str | None = None, retry: bool = True) -> dic except (HTTPError, URLError, TimeoutError, ConnectionError) as error: if isinstance(error, HTTPError): error.close() - if public_login and error.code == 404: - return {"public_member": False} if error.code != 429 and not 500 <= error.code < 600: raise self.time_left() @@ -498,7 +492,55 @@ def eligible_pr(repo: str, probe: Probe) -> bool: login = pr["user"]["login"] if not isinstance(login, str) or not LOGIN.fullmatch(login): return False - return member(login, lambda path: probe.get(path, retry=False)) is None + return member(login, lambda path: membership_read(probe, path)) is None + + +def response_data(path: str, response) -> dict: + """The authenticated public-membership endpoint returns no JSON on 204.""" + if "/public_members/" in path and getattr(response, "status", None) == 204: + return {"public_member": True, "user": {"login": path.rsplit("/", 1)[1]}} + return json.load(response) + + +def membership_read(probe: Probe, path: str) -> dict: + """A public 404 needs the private active-state check; other errors deny.""" + try: + return probe.get(path, retry=False) + except HTTPError as error: + if error.code == 404 and "/public_members/" in path: + return {"public_member": False} + raise + + +def selection_enabled(event: str, labels: str) -> bool: + return bool(labels) and ( + event == "merge_group" + or (event == "pull_request" and os.environ.get("DELL_PR_ROUTING") == "on") + ) + + +def routing_capacity(event: str, min_idle: str, demand: str, mode: str) -> dict[str, int]: + """Keep MQ-only arithmetic unchanged; enabled PRs have their own demand.""" + required = required_capacity(min_idle, demand, mode) + capacity = {"required": required} + if os.environ.get("DELL_PR_ROUTING") != "on": + return capacity + pr_required = max(required, required_capacity("1", os.environ.get("MQ_PR_DEMAND") or str(PR_RUNNER_DEMAND))) + capacity.update(mq_required=required, pr_required=pr_required) + if event == "pull_request": + reserve = int(os.environ.get("MQ_PR_RESERVE") or demand or "1") + if reserve < 0: + raise ValueError("PR reserve must be nonnegative") + capacity.update(required=pr_required, reserve=reserve) + return capacity + + +def capacity_summary(event: str, capacity: dict[str, int]) -> str: + return f", pr_reserve={capacity['reserve']}" if event == "pull_request" else "" + + +def routing_eligible(event: str, repo: str, probe: Probe) -> bool: + return event != "pull_request" or eligible_pr(repo, probe) def select_labels( @@ -518,33 +560,21 @@ def select_labels( each HTTP request stay bounded; runners visible in both scopes count only once. In-flight commitments and older undecided runs reserve capacity. """ - is_pr = event == "pull_request" - if (event != "merge_group" and not (is_pr and os.environ.get("DELL_PR_ROUTING") == "on")) or not labels: + if not selection_enabled(event, labels): return decision("opt-in merge-group runner selection is disabled") step = "configuration" try: requested = requested_labels(labels) - required = required_capacity(min_idle, demand, mode) - mq_required = required - pr_required = required - if os.environ.get("DELL_PR_ROUTING") == "on": - pr_required = max(required, required_capacity("1", os.environ.get("MQ_PR_DEMAND") or str(PR_RUNNER_DEMAND))) - if is_pr: - required = pr_required + capacity = routing_capacity(event, min_idle, demand, mode) + required = capacity["required"] if not token: warning("MQ_RUNNER_STATUS_TOKEN is missing") return decision("runner probe token is missing") wanted = {label.casefold() for label in requested} probe = Probe(token, api_url) - reserve = 0 - if is_pr: - step = "PR eligibility" - if not eligible_pr(repo, probe): - return decision("PR author is not a verified same-repository active member") - step = "PR reserve configuration" - reserve = int(os.environ.get("MQ_PR_RESERVE") or demand or "1") - if reserve < 0: - raise ValueError("PR reserve must be nonnegative") + step = "PR eligibility" + if not routing_eligible(event, repo, probe): + return decision("PR author is not a verified same-repository active member") matched = {} unavailable = [] owner = repo.split("/", 1)[0] @@ -571,16 +601,12 @@ def select_labels( busy = sum(matched.values()) idle = len(matched) - busy counts = f"online={len(matched)}, busy={busy}, idle={idle}, required={required}" - if is_pr: - counts += f", pr_reserve={reserve}" + counts += capacity_summary(event, capacity) if unavailable: counts += f"; unavailable scopes: {', '.join(unavailable)}" step = "in-flight commitments probe" - options = {"reserve": reserve} if is_pr else {} - if os.environ.get("DELL_PR_ROUTING") == "on": - options.update(mq_required=mq_required, pr_required=pr_required) admitted, commitments = admission_available( - repo, wanted, probe, len(matched), idle, required, **options + repo, wanted, probe, len(matched), idle, **capacity ) probe.time_left() counts += commitments From 431c7a09acac6cdec46dc0da9fa083ac37a28044 Mon Sep 17 00:00:00 2001 From: Zach Vorhies Date: Thu, 19 Nov 2026 11:27:28 -0800 Subject: [PATCH 5/6] ci: let members' same-repo PRs use idle Dell capacity (default off) Keep base-owned routing within the repository complexity policy. Refs #4839 #4896. Ci-Attestation: {"at":1795117094,"gate":"general/all/client-lint","host":"linux-x86_64","key":"2f3e7f266796642b8976ac1d0f5cd5929127e275d2faec99e2a5004a1a678d95","lane":"client-lint","parents":["db010579c5141bde73b679814339899ba8b5ea42"],"secs":639,"stamp":"bf899947912f740133ac880c7ef1a4d5","tree":"6bc1ce0022ee495ff4d57513958f40ab0eff5fe9","v":1,"via":"run"} Ci-Attestation: {"at":1795117097,"gate":"rust/aarch64-apple-darwin/client-clippy","host":"linux-x86_64","key":"2f3e7f266796642b8976ac1d0f5cd5929127e275d2faec99e2a5004a1a678d95","lane":"client-lint","parents":["db010579c5141bde73b679814339899ba8b5ea42"],"secs":639,"stamp":"43a1120dd77be6e324c94bb6be34d04a","tree":"6bc1ce0022ee495ff4d57513958f40ab0eff5fe9","v":1,"via":"run"} Ci-Attestation: {"at":1795117097,"gate":"rust/aarch64-apple-darwin/client-dylint","host":"linux-x86_64","key":"2f3e7f266796642b8976ac1d0f5cd5929127e275d2faec99e2a5004a1a678d95","lane":"client-lint","parents":["db010579c5141bde73b679814339899ba8b5ea42"],"secs":639,"stamp":"4ecca3afad133fb09e879ae415b07028","tree":"6bc1ce0022ee495ff4d57513958f40ab0eff5fe9","v":1,"via":"run"} Ci-Attestation: {"at":1795117096,"gate":"rust/aarch64-pc-windows-msvc/client-clippy","host":"linux-x86_64","key":"2f3e7f266796642b8976ac1d0f5cd5929127e275d2faec99e2a5004a1a678d95","lane":"client-lint","parents":["db010579c5141bde73b679814339899ba8b5ea42"],"secs":639,"stamp":"908ed2a4237312bee40c6fc5e4404fc0","tree":"6bc1ce0022ee495ff4d57513958f40ab0eff5fe9","v":1,"via":"run"} Ci-Attestation: {"at":1795117096,"gate":"rust/aarch64-pc-windows-msvc/client-dylint","host":"linux-x86_64","key":"2f3e7f266796642b8976ac1d0f5cd5929127e275d2faec99e2a5004a1a678d95","lane":"client-lint","parents":["db010579c5141bde73b679814339899ba8b5ea42"],"secs":639,"stamp":"55d0ef1115bf1bce02a42c9e612b2929","tree":"6bc1ce0022ee495ff4d57513958f40ab0eff5fe9","v":1,"via":"run"} Ci-Attestation: {"at":1795117095,"gate":"rust/aarch64-unknown-linux-gnu/client-clippy","host":"linux-x86_64","key":"2f3e7f266796642b8976ac1d0f5cd5929127e275d2faec99e2a5004a1a678d95","lane":"client-lint","parents":["db010579c5141bde73b679814339899ba8b5ea42"],"secs":639,"stamp":"d218aafbbdcfb8ee699ea98687095368","tree":"6bc1ce0022ee495ff4d57513958f40ab0eff5fe9","v":1,"via":"run"} Ci-Attestation: {"at":1795117095,"gate":"rust/aarch64-unknown-linux-gnu/client-dylint","host":"linux-x86_64","key":"2f3e7f266796642b8976ac1d0f5cd5929127e275d2faec99e2a5004a1a678d95","lane":"client-lint","parents":["db010579c5141bde73b679814339899ba8b5ea42"],"secs":639,"stamp":"270703fce4f1152a587ca10050793bfc","tree":"6bc1ce0022ee495ff4d57513958f40ab0eff5fe9","v":1,"via":"run"} Ci-Attestation: {"at":1795117096,"gate":"rust/x86_64-apple-darwin/client-clippy","host":"linux-x86_64","key":"2f3e7f266796642b8976ac1d0f5cd5929127e275d2faec99e2a5004a1a678d95","lane":"client-lint","parents":["db010579c5141bde73b679814339899ba8b5ea42"],"secs":639,"stamp":"a8b0d6a16f37c1d0a5b0882c8ac0ed15","tree":"6bc1ce0022ee495ff4d57513958f40ab0eff5fe9","v":1,"via":"run"} Ci-Attestation: {"at":1795117096,"gate":"rust/x86_64-apple-darwin/client-dylint","host":"linux-x86_64","key":"2f3e7f266796642b8976ac1d0f5cd5929127e275d2faec99e2a5004a1a678d95","lane":"client-lint","parents":["db010579c5141bde73b679814339899ba8b5ea42"],"secs":639,"stamp":"f395f1bfb608674b2f0509a85a6df1f0","tree":"6bc1ce0022ee495ff4d57513958f40ab0eff5fe9","v":1,"via":"run"} Ci-Attestation: {"at":1795117095,"gate":"rust/x86_64-pc-windows-msvc/client-clippy","host":"linux-x86_64","key":"2f3e7f266796642b8976ac1d0f5cd5929127e275d2faec99e2a5004a1a678d95","lane":"client-lint","parents":["db010579c5141bde73b679814339899ba8b5ea42"],"secs":639,"stamp":"06b496357377d4d1409bf82cc43d8af6","tree":"6bc1ce0022ee495ff4d57513958f40ab0eff5fe9","v":1,"via":"run"} Ci-Attestation: {"at":1795117096,"gate":"rust/x86_64-pc-windows-msvc/client-dylint","host":"linux-x86_64","key":"2f3e7f266796642b8976ac1d0f5cd5929127e275d2faec99e2a5004a1a678d95","lane":"client-lint","parents":["db010579c5141bde73b679814339899ba8b5ea42"],"secs":639,"stamp":"0bd9c19e2bf2611da5d50d7ec5e206bb","tree":"6bc1ce0022ee495ff4d57513958f40ab0eff5fe9","v":1,"via":"run"} Ci-Attestation: {"at":1795117094,"gate":"rust/x86_64-unknown-linux-gnu/client-clippy","host":"linux-x86_64","key":"2f3e7f266796642b8976ac1d0f5cd5929127e275d2faec99e2a5004a1a678d95","lane":"client-lint","parents":["db010579c5141bde73b679814339899ba8b5ea42"],"secs":639,"stamp":"b9aa020c7f91d9baffc7fafe9934eb72","tree":"6bc1ce0022ee495ff4d57513958f40ab0eff5fe9","v":1,"via":"run"} Ci-Attestation: {"at":1795117094,"gate":"rust/x86_64-unknown-linux-gnu/client-dylint","host":"linux-x86_64","key":"2f3e7f266796642b8976ac1d0f5cd5929127e275d2faec99e2a5004a1a678d95","lane":"client-lint","parents":["db010579c5141bde73b679814339899ba8b5ea42"],"secs":639,"stamp":"8be6f6dde91433c91605ea4746994b48","tree":"6bc1ce0022ee495ff4d57513958f40ab0eff5fe9","v":1,"via":"run"} --- ci/select_mq_runner.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/ci/select_mq_runner.py b/ci/select_mq_runner.py index 706536fcaa..995ba161ae 100644 --- a/ci/select_mq_runner.py +++ b/ci/select_mq_runner.py @@ -543,6 +543,13 @@ def routing_eligible(event: str, repo: str, probe: Probe) -> bool: return event != "pull_request" or eligible_pr(repo, probe) +def availability_summary(online: int, busy: int, idle: int, required: int, unavailable: list[str]) -> str: + counts = f"online={online}, busy={busy}, idle={idle}, required={required}" + if unavailable: + counts += f"; unavailable scopes: {', '.join(unavailable)}" + return counts + + def select_labels( event: str, labels: str, @@ -600,10 +607,8 @@ def select_labels( ) busy = sum(matched.values()) idle = len(matched) - busy - counts = f"online={len(matched)}, busy={busy}, idle={idle}, required={required}" + counts = availability_summary(len(matched), busy, idle, required, unavailable) counts += capacity_summary(event, capacity) - if unavailable: - counts += f"; unavailable scopes: {', '.join(unavailable)}" step = "in-flight commitments probe" admitted, commitments = admission_available( repo, wanted, probe, len(matched), idle, **capacity From 17aebbc8b491f8072105e6a037e6460c6cb59edd Mon Sep 17 00:00:00 2001 From: Zach Vorhies Date: Thu, 19 Nov 2026 11:47:33 -0800 Subject: [PATCH 6/6] ci: let members' same-repo PRs use idle Dell capacity (default off) Use a BASE-owned Python launcher for extraction under GEN-005; test poisoned working-tree modules and preserve the S3 reporter hosted fallback. Refs #4839 #4896 Ci-Attestation: {"at":1795117866,"gate":"general/all/client-lint","host":"linux-x86_64","key":"3ea01f3762055374f1bf92ebc437eef1fcb4a729d55a5fa7a57e869d7c470256","lane":"client-lint","parents":["431c7a09acac6cdec46dc0da9fa083ac37a28044"],"secs":186,"stamp":"6f7f98475f2f88f2e758b7d3490d7482","tree":"3106d00e8ef2e18ad281227bcccecba3dda91344","v":1,"via":"run"} Ci-Attestation: {"at":1795117870,"gate":"rust/aarch64-apple-darwin/client-clippy","host":"linux-x86_64","key":"3ea01f3762055374f1bf92ebc437eef1fcb4a729d55a5fa7a57e869d7c470256","lane":"client-lint","parents":["431c7a09acac6cdec46dc0da9fa083ac37a28044"],"secs":186,"stamp":"16bb6cd873d0124d89c0743ae3c39caa","tree":"3106d00e8ef2e18ad281227bcccecba3dda91344","v":1,"via":"run"} Ci-Attestation: {"at":1795117870,"gate":"rust/aarch64-apple-darwin/client-dylint","host":"linux-x86_64","key":"3ea01f3762055374f1bf92ebc437eef1fcb4a729d55a5fa7a57e869d7c470256","lane":"client-lint","parents":["431c7a09acac6cdec46dc0da9fa083ac37a28044"],"secs":186,"stamp":"aad50d3db80269697bab4be08ba2e496","tree":"3106d00e8ef2e18ad281227bcccecba3dda91344","v":1,"via":"run"} Ci-Attestation: {"at":1795117869,"gate":"rust/aarch64-pc-windows-msvc/client-clippy","host":"linux-x86_64","key":"3ea01f3762055374f1bf92ebc437eef1fcb4a729d55a5fa7a57e869d7c470256","lane":"client-lint","parents":["431c7a09acac6cdec46dc0da9fa083ac37a28044"],"secs":186,"stamp":"e09aa07a469bbbad37f13c3a802f997f","tree":"3106d00e8ef2e18ad281227bcccecba3dda91344","v":1,"via":"run"} Ci-Attestation: {"at":1795117869,"gate":"rust/aarch64-pc-windows-msvc/client-dylint","host":"linux-x86_64","key":"3ea01f3762055374f1bf92ebc437eef1fcb4a729d55a5fa7a57e869d7c470256","lane":"client-lint","parents":["431c7a09acac6cdec46dc0da9fa083ac37a28044"],"secs":186,"stamp":"0875e7d5ee04507cce773e3eb211eb72","tree":"3106d00e8ef2e18ad281227bcccecba3dda91344","v":1,"via":"run"} Ci-Attestation: {"at":1795117867,"gate":"rust/aarch64-unknown-linux-gnu/client-clippy","host":"linux-x86_64","key":"3ea01f3762055374f1bf92ebc437eef1fcb4a729d55a5fa7a57e869d7c470256","lane":"client-lint","parents":["431c7a09acac6cdec46dc0da9fa083ac37a28044"],"secs":186,"stamp":"b723574419d55c150fc81fb66143e17d","tree":"3106d00e8ef2e18ad281227bcccecba3dda91344","v":1,"via":"run"} Ci-Attestation: {"at":1795117867,"gate":"rust/aarch64-unknown-linux-gnu/client-dylint","host":"linux-x86_64","key":"3ea01f3762055374f1bf92ebc437eef1fcb4a729d55a5fa7a57e869d7c470256","lane":"client-lint","parents":["431c7a09acac6cdec46dc0da9fa083ac37a28044"],"secs":186,"stamp":"162e078e0997318b55089310f332ac5c","tree":"3106d00e8ef2e18ad281227bcccecba3dda91344","v":1,"via":"run"} Ci-Attestation: {"at":1795117869,"gate":"rust/x86_64-apple-darwin/client-clippy","host":"linux-x86_64","key":"3ea01f3762055374f1bf92ebc437eef1fcb4a729d55a5fa7a57e869d7c470256","lane":"client-lint","parents":["431c7a09acac6cdec46dc0da9fa083ac37a28044"],"secs":186,"stamp":"4e4d042f6de18cc380ff1b3e32df23e0","tree":"3106d00e8ef2e18ad281227bcccecba3dda91344","v":1,"via":"run"} Ci-Attestation: {"at":1795117870,"gate":"rust/x86_64-apple-darwin/client-dylint","host":"linux-x86_64","key":"3ea01f3762055374f1bf92ebc437eef1fcb4a729d55a5fa7a57e869d7c470256","lane":"client-lint","parents":["431c7a09acac6cdec46dc0da9fa083ac37a28044"],"secs":186,"stamp":"b24bbccbef773a68f16e6ed270c9a18a","tree":"3106d00e8ef2e18ad281227bcccecba3dda91344","v":1,"via":"run"} Ci-Attestation: {"at":1795117868,"gate":"rust/x86_64-pc-windows-msvc/client-clippy","host":"linux-x86_64","key":"3ea01f3762055374f1bf92ebc437eef1fcb4a729d55a5fa7a57e869d7c470256","lane":"client-lint","parents":["431c7a09acac6cdec46dc0da9fa083ac37a28044"],"secs":186,"stamp":"e9d185e8d05f345a3efb74f23c4612e2","tree":"3106d00e8ef2e18ad281227bcccecba3dda91344","v":1,"via":"run"} Ci-Attestation: {"at":1795117868,"gate":"rust/x86_64-pc-windows-msvc/client-dylint","host":"linux-x86_64","key":"3ea01f3762055374f1bf92ebc437eef1fcb4a729d55a5fa7a57e869d7c470256","lane":"client-lint","parents":["431c7a09acac6cdec46dc0da9fa083ac37a28044"],"secs":186,"stamp":"4b950367d59dff8ee0644e1a32e2e0cb","tree":"3106d00e8ef2e18ad281227bcccecba3dda91344","v":1,"via":"run"} Ci-Attestation: {"at":1795117866,"gate":"rust/x86_64-unknown-linux-gnu/client-clippy","host":"linux-x86_64","key":"3ea01f3762055374f1bf92ebc437eef1fcb4a729d55a5fa7a57e869d7c470256","lane":"client-lint","parents":["431c7a09acac6cdec46dc0da9fa083ac37a28044"],"secs":186,"stamp":"03d97d87f131d7eb85c1628dc18ba2e0","tree":"3106d00e8ef2e18ad281227bcccecba3dda91344","v":1,"via":"run"} Ci-Attestation: {"at":1795117866,"gate":"rust/x86_64-unknown-linux-gnu/client-dylint","host":"linux-x86_64","key":"3ea01f3762055374f1bf92ebc437eef1fcb4a729d55a5fa7a57e869d7c470256","lane":"client-lint","parents":["431c7a09acac6cdec46dc0da9fa083ac37a28044"],"secs":186,"stamp":"592159aca9c1438b94f5841194f40580","tree":"3106d00e8ef2e18ad281227bcccecba3dda91344","v":1,"via":"run"} --- .github/workflows/ci.yml | 8 +----- ci/run_base_runner_selector.py | 34 ++++++++++++++++++++++ ci/test_ci_scope.py | 2 +- ci/test_member_pr_routing.py | 52 ++++++++++++++++++++++++++++++++-- ci/test_soldr_cache_report.py | 5 +++- 5 files changed, 89 insertions(+), 12 deletions(-) create mode 100644 ci/run_base_runner_selector.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6c3777eb10..f359828f55 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -128,13 +128,7 @@ jobs: MQ_RUNNER_DEMAND: ${{ vars.MQ_RUNNER_DEMAND || '6' }} MQ_RUNNER_STATUS_TOKEN: ${{ steps.mq-probe-app.outputs.token || secrets.MQ_RUNNER_STATUS_TOKEN }} GITHUB_TOKEN: ${{ github.token }} - run: | - [[ "$BASE" =~ ^[0-9a-f]{40}$ ]] - trusted=$(mktemp -d "$RUNNER_TEMP/dell-selector.XXXXXX") - for script in select_mq_runner.py pr_attestation.py; do - git show "$BASE:ci/$script" > "$trusted/$script" || exit 1 - done - uv run --no-project "$trusted/select_mq_runner.py" + run: uv run --no-project ci/run_base_runner_selector.py - name: Report Probe Setup Failure if: ${{ !cancelled() && (steps.checkout.outcome == 'failure' || steps.uv.outcome == 'failure' || steps.select.outcome == 'failure') }} run: echo '::warning::Merge-queue runner probe could not complete; using hosted runners.' diff --git a/ci/run_base_runner_selector.py b/ci/run_base_runner_selector.py new file mode 100644 index 0000000000..4b2f8b3fe2 --- /dev/null +++ b/ci/run_base_runner_selector.py @@ -0,0 +1,34 @@ +#!/usr/bin/env -S uv run --script +"""Launch runner selection from immutable BASE objects, without head imports. + +The hosted workflow checks out BASE before invoking this launcher. Keep the +selector and membership verifier together in an isolated module directory. +""" + +import os +from pathlib import Path +import re +import subprocess +import tempfile + + +def main() -> int: + base = os.environ["BASE"] + if re.fullmatch(r"[0-9a-f]{40}", base) is None: + raise ValueError("invalid immutable base SHA") + with tempfile.TemporaryDirectory(prefix="dell-selector-", dir=os.environ.get("RUNNER_TEMP")) as directory: + trusted = Path(directory) + for script in ("select_mq_runner.py", "pr_attestation.py"): + source = subprocess.run( + ["git", "show", f"{base}:ci/{script}"], + capture_output=True, check=True, + ).stdout + (trusted / script).write_bytes(source) + return subprocess.run( + ["uv", "run", "--no-project", str(trusted / "select_mq_runner.py")], + check=False, + ).returncode + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/ci/test_ci_scope.py b/ci/test_ci_scope.py index 0300c96de4..1494524cc2 100644 --- a/ci/test_ci_scope.py +++ b/ci/test_ci_scope.py @@ -412,7 +412,7 @@ def test_queue_runner_probe_and_reusable_inputs(self): ) for variable in ("MQ_RUNNER_MODE", "MQ_RUNNER_MIN_IDLE"): self.assertIn(f"{variable}: ${{{{ vars.{variable} }}}}", steps["Check Runner Availability"]) - self.assertIn('uv run --no-project "$trusted/select_mq_runner.py"', probe) + self.assertIn('run: uv run --no-project ci/run_base_runner_selector.py', probe) self.assertIn("heavy-labels: ${{ steps.select.outputs.heavy-labels }}", probe) self.assertNotRegex(probe, r"(?m)^ continue-on-error:") self.assertEqual(probe.count("continue-on-error: true"), 4) diff --git a/ci/test_member_pr_routing.py b/ci/test_member_pr_routing.py index f36af01e8a..b7c8d807e9 100644 --- a/ci/test_member_pr_routing.py +++ b/ci/test_member_pr_routing.py @@ -6,6 +6,7 @@ import os from pathlib import Path import re +import subprocess import tempfile import unittest from unittest.mock import patch @@ -175,9 +176,10 @@ def test_base_owned_selector_and_membership_helper(self): ci = (ROOT / ".github/workflows/ci.yml").read_text() probe = ci.split(" select-mq-runner:\n", 1)[1].split(" cancel-superseded:\n", 1)[0] self.assertIn("ref: ${{ github.event.pull_request.base.sha || github.sha }}", probe) - self.assertIn('git show "$BASE:ci/$script"', probe) - self.assertIn("select_mq_runner.py pr_attestation.py", probe) - self.assertIn('uv run --no-project "$trusted/select_mq_runner.py"', probe) + self.assertIn('run: uv run --no-project ci/run_base_runner_selector.py', probe) + launcher = (ROOT / "ci/run_base_runner_selector.py").read_text() + self.assertIn('["git", "show", f"{base}:ci/{script}"]', launcher) + self.assertIn('("select_mq_runner.py", "pr_attestation.py")', launcher) self.assertIn("DELL_PR_ROUTING: ${{ vars.DELL_PR_ROUTING || 'off' }}", probe) self.assertIn("MQ_PR_RESERVE: ${{ vars.MQ_PR_RESERVE }}", probe) self.assertNotIn("if: github.event_name != 'pull_request'\n", probe) @@ -209,5 +211,49 @@ def test_fast_gate_pr_routing_keeps_queue_hosted(self): self.assertIn("github.event_name == 'pull_request' && inputs.mq-heavy-labels", runner) +class BaseRunnerLauncherTest(unittest.TestCase): + def test_selector_and_membership_are_loaded_from_base_objects(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + ci = root / "ci" + ci.mkdir() + + def git(*args): + return subprocess.run(["git", *args], cwd=root, check=True, + capture_output=True, text=True).stdout.strip() + + git("init", "--quiet") + (ci / "pr_attestation.py").write_text("identity = 'base-owned'\n") + (ci / "select_mq_runner.py").write_text( + "import os\nfrom pathlib import Path\nimport pr_attestation\n" + "Path(os.environ['GITHUB_OUTPUT']).write_text(pr_attestation.identity)\n" + ) + git("add", "ci") + git("-c", "user.name=Test", "-c", "user.email=test@example.invalid", + "commit", "--quiet", "--no-gpg-sign", "-m", "trusted base") + base = git("rev-parse", "HEAD") + # A PR changes both files; executing either working-tree module fails. + for path in ci.iterdir(): + path.write_text("raise RuntimeError('PR-controlled code executed')\n") + output = root / "output" + result = subprocess.run( + ["uv", "run", "--no-project", str(ROOT / "ci/run_base_runner_selector.py")], + cwd=root, env={**os.environ, "BASE": base, "RUNNER_TEMP": directory, + "GITHUB_OUTPUT": str(output)}, + capture_output=True, text=True, timeout=30, + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(output.read_text(), "base-owned") + + def test_invalid_base_never_launches_selector(self): + result = subprocess.run( + ["uv", "run", "--no-project", str(ROOT / "ci/run_base_runner_selector.py")], + env={**os.environ, "BASE": "HEAD:evil"}, + capture_output=True, text=True, timeout=30, + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn("invalid immutable base SHA", result.stderr) + + if __name__ == "__main__": unittest.main() diff --git a/ci/test_soldr_cache_report.py b/ci/test_soldr_cache_report.py index 938ce92dee..cc32ba5f0a 100644 --- a/ci/test_soldr_cache_report.py +++ b/ci/test_soldr_cache_report.py @@ -145,7 +145,10 @@ def test_macos_build_installs_uv_when_native_reuse_did_not(self) -> None: def test_s3_size_step_lives_where_its_condition_can_hold(self) -> None: self.assertNotIn("Report S3 cache size", job_block("macos-build")) job = job_block("rust-tests-linux") - self.assertIn("runs-on: ubuntu-26.04\n", job) + self.assertIn( + "runs-on: ${{ github.event_name == 'pull_request' && inputs.mq-heavy-labels != '' " + "&& fromJSON(inputs.mq-heavy-labels) || 'ubuntu-26.04' }}", job, + ) step = step_block(job, "Report S3 cache size") self.assertNotIn("matrix.os", step) self.assertIn("command -v aws", step)