diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f359828f55..176fad7aff 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -124,6 +124,7 @@ jobs: MQ_RUNNER_LABELS: ${{ vars.MQ_RUNNER_LABELS }} MQ_RUNNER_MODE: ${{ vars.MQ_RUNNER_MODE }} MQ_RUNNER_MIN_IDLE: ${{ vars.MQ_RUNNER_MIN_IDLE }} + MQ_RUNNER_MAX_RUNS: ${{ vars.MQ_RUNNER_MAX_RUNS }} # Tier 2 build + OFF/ON tests + two cross-lint legs + release lint. MQ_RUNNER_DEMAND: ${{ vars.MQ_RUNNER_DEMAND || '6' }} MQ_RUNNER_STATUS_TOKEN: ${{ steps.mq-probe-app.outputs.token || secrets.MQ_RUNNER_STATUS_TOKEN }} diff --git a/ci/select_mq_runner.py b/ci/select_mq_runner.py index 995ba161ae..bf195a341f 100644 --- a/ci/select_mq_runner.py +++ b/ci/select_mq_runner.py @@ -1,5 +1,13 @@ #!/usr/bin/env -S uv run --script -"""Probe opt-in queue/member PR runners; unknown states always use hosted CI.""" +"""Prefer bounded queueing on the opt-in Dell over slower hosted completion. + +MQ_RUNNER_MAX_RUNS defaults to 3 concurrent runs on six runners. Reports 22/23 +(2026-11-19) measured Dell/hosted medians of 27.8/46.3 minutes and Dell job waits +of p90 349 s, max 615 s: even the observed maximum wait leaves Dell ahead by +8.25 minutes. Keep the idle-capacity path, then allow busy-Dell admission below +that configurable run cap; online capacity must still cover one run's demand. +Older undecided runs reserve slots to bound simultaneous selector decisions. +""" from __future__ import annotations @@ -9,8 +17,9 @@ import signal import time from contextlib import contextmanager -from datetime import datetime +from datetime import datetime, timezone from email.utils import parsedate_to_datetime +from itertools import count from pathlib import Path from urllib.error import HTTPError, URLError from urllib.request import Request, urlopen @@ -25,7 +34,7 @@ # Two runner scopes, two run statuses, and a jobs collection for each active run. # Include pagination and retries, while retaining the shared wall-clock deadline. MAX_REQUESTS = (4 + MAX_ACTIVE_RUNS) * MAX_PAGES * (MAX_RETRIES + 1) -# All expanded PR-routable jobs, including both sequential Tier 2 phases. +# Hard admission floor: all expanded PR-routable jobs, including both Tier 2 phases. # ci/test_member_pr_routing.py checks this bound against the workflow inventory. PR_RUNNER_DEMAND = 13 @@ -173,7 +182,12 @@ def entries(self, path: str, key: str, token: str) -> list[dict]: if len(entries) < 100: raise ValueError("incomplete collection") elif len(entries) < 100: - ids = frozenset(entry["id"] for entry in result) + # A missing identity still occupies a conservative run + # slot; active_runs normalizes it without a jobs lookup. + ids = frozenset( + entry.get("id", ("missing", index)) + for index, entry in enumerate(result) + ) if ( totals == {len(ids)} and len(ids) == len(result) ) or ids == previous: @@ -204,7 +218,7 @@ def warning(reason: str, *, hosted: bool = True) -> None: def probe_error(error: Exception) -> str: """Include validation reasons, escaping Actions annotation control characters.""" detail = "" - if isinstance(error, ValueError): + if isinstance(error, (ValueError, KeyError)): detail = ": " + str(error).replace("%", "%25").replace("\r", "%0D").replace( "\n", "%0A" ) @@ -311,10 +325,32 @@ def job_commitments(jobs: list[dict], wanted: set[str]) -> tuple[int, bool, bool return queued, dell, decided +# Unknown timestamps sort before every real run, conservatively holding slots. +UNKNOWN_CREATED = datetime.min.replace(tzinfo=timezone.utc) + + +def run_snapshot(run: dict, status: str, missing_id: int, event: str) -> dict: + """Normalize missing routing fields conservatively within a filtered list.""" + if "id" in run and (type(run["id"]) is not int or run["id"] < 1): + raise ValueError("invalid active run ID") + defaults = { + "id": missing_id, + "event": event, + "status": status, + "created_at": UNKNOWN_CREATED.isoformat(), + } + for key, value in defaults.items(): + if key not in run: + warning(f"runs probe: missing key '{key}'; reserving run", hosted=False) + run = {key: value, **run} + return run + + def active_runs(repo: str, probe: Probe, token: str, event: str = "merge_group") -> dict: """Keep only active run snapshots; terminal snapshots release reservations.""" runs = {} finished = set() + missing_ids = count(-1, -1) for status in ("queued", "in_progress"): try: entries = probe.entries( @@ -329,6 +365,7 @@ def active_runs(repo: str, probe: Probe, token: str, event: str = "merge_group") except Exception as error: raise ValueError(f"runs probe ({status}): {probe_error(error)}") from error for run in entries: + run = run_snapshot(run, status, next(missing_ids), event) run_id = run["id"] if ( run.get("status") in ("completed", "cancelled") @@ -341,7 +378,6 @@ def active_runs(repo: str, probe: Probe, token: str, event: str = "merge_group") continue if ( type(run_id) is not int - or run_id < 1 or run["event"] != event or run["status"] not in ("queued", "in_progress") ): @@ -353,6 +389,48 @@ def active_runs(repo: str, probe: Probe, token: str, event: str = "merge_group") return runs +def observed_job_commitments( + jobs: list[dict], wanted: set[str], demand: int, run_id: int +) -> tuple[int, bool, bool]: + """An incomplete job snapshot holds one run rather than aborting admission.""" + try: + return job_commitments(jobs, wanted) + except KeyError as error: + warning( + f"jobs probe (run {run_id}): {probe_error(error)}; reserving run", + hosted=False, + ) + return demand, True, True + + +def current_priority(runs: dict, current_id: int): + """Unknown current ordering must never outrank an undecided peer.""" + current = runs.get(current_id) + if current is None: + # A stable retried page can omit this run. Without its timestamp every + # undecided peer must rank ahead of us; do not guess its priority. + warning( + f"runs probe: missing key {current_id} (current run); " + "reserving all undecided peers", hosted=False, + ) + current = current[0] if current and UNKNOWN_CREATED not in current[0] else None + return current + + +def commitment_runs(repo: str, probe: Probe, token: str, mixed: bool) -> dict: + """Queue priorities precede PRs, retaining distinct unknown-identity slots.""" + runs = active_runs(repo, probe, token) + if mixed: + # Queue runs sort before every PR, even a PR created earlier. Inspect + # live PR jobs for both events so the queue sees their queued demand. + runs = {key: ((0, *value[0]), value[1]) for key, value in runs.items()} + prs = active_runs(repo, probe, token, "pull_request") + offset = min(0, min(runs, default=0)) + runs.update({key if key > 0 else key + offset: ((1, *value[0]), value[1]) + for key, value in prs.items()}) + return runs + + def inflight_commitments( repo: str, wanted: set[str], probe: Probe, demand: int = 1, *, mq_required: int = 0, pr_required: int = 0, @@ -368,30 +446,27 @@ def inflight_commitments( token = os.environ["GITHUB_TOKEN"] if current_id < 1 or not token: raise ValueError("missing Actions probe identity") - runs = active_runs(repo, probe, token) mixed = os.environ.get("DELL_PR_ROUTING") == "on" - if mixed: - # Queue runs sort before every PR, even a PR created earlier. Inspect - # live PR jobs for both events so the queue sees their queued demand. - runs = {key: ((0, *value[0]), value[1]) for key, value in runs.items()} - prs = active_runs(repo, probe, token, "pull_request") - runs.update({key: ((1, *value[0]), value[1]) for key, value in prs.items()}) - current = runs[current_id] # An incomplete/eventually consistent list is unsafe. + runs = commitment_runs(repo, probe, token, mixed) + current = current_priority(runs, current_id) queued = committed = unprobed = 0 - current = current[0] for run_id, (priority, _) in sorted(runs.items(), key=lambda item: item[1][0]): if run_id == current_id: continue weight = (pr_required if priority[0] == 1 else mq_required) if mixed else 1 peer_demand = weight if mixed else demand + if run_id < 0: # Missing identity: cannot safely probe or release this peer. + queued += peer_demand + committed += weight + continue try: jobs = probe.entries( f"repos/{repo}/actions/runs/{run_id}/jobs?filter=latest", "jobs", token ) - pending, dell, decided = job_commitments(jobs, wanted) + pending, dell, decided = observed_job_commitments(jobs, wanted, peer_demand, run_id) except RequestBudgetExhausted as error: unprobed += 1 - observed, _, _ = job_commitments(error.entries, wanted) + observed, _, _ = observed_job_commitments(error.entries, wanted, peer_demand, run_id) queued += max(observed, peer_demand) committed += weight continue @@ -406,7 +481,7 @@ def inflight_commitments( queued += pending # Count known Dell runs regardless of age. Unknown older runs also # reserve slots; hosted or finished Dell work releases their reservation. - committed += weight * (dell or (not decided and priority < current)) + committed += weight * (dell or (not decided and (current is None or priority < current))) if unprobed: reservation = ( "reserving each unprobed active run's required capacity" @@ -420,20 +495,48 @@ def inflight_commitments( return queued, committed +def queue_admission( + repo: str, wanted: set[str], probe: Probe, online: int, idle: int, + required: int, options: dict, +) -> tuple[bool, str]: + """Allow busy MQ work; mixed traffic retains PR capacity weights.""" + max_runs = int(os.environ.get("MQ_RUNNER_MAX_RUNS") or "3") + if max_runs < 1: + raise ValueError("MQ_RUNNER_MAX_RUNS must be positive") + if online < required: + return False, f"; run_limit={max_runs}" + queued, committed = inflight_commitments(repo, wanted, probe, required, **options) + idle_slots = max(0, (idle - queued) // required) + # With PR routing on, commitments retain each event's full runner demand: + # the MQ budget is max_runs * required, so a large PR is never one small MQ. + weight = required if options else 1 + limit = max_runs * weight + counts = ( + f"; queued={queued}, committed={committed}, run_limit={limit}" + f", idle_run_slots={idle_slots}" + ) + if options: + counts += f"; commitments measured in runner capacity, max_runs={max_runs}" + if committed + weight > limit: + return False, counts + if idle_slots >= 1: + return True, counts + "; admission=idle" + return True, counts + "; admission=busy queue (expected faster than hosted)" + + def admission_available( repo: str, wanted: set[str], probe: Probe, online: int, idle: int, required: int, - reserve: int = 0, mq_required: int = 0, pr_required: int = 0, + reserve: int = 0, mq_required: int = 0, pr_required: int = 0, queue: bool = True, ) -> tuple[bool, str]: - """The idle floor and in-flight run cap must both admit this run.""" - if idle - reserve < required: - return False, "" + """Queue overcommit is MQ-only; PRs retain their idle floor and MQ reserve.""" mixed = os.environ.get("DELL_PR_ROUTING") == "on" options = {"mq_required": mq_required, "pr_required": pr_required} if mixed else {} + if queue: + return queue_admission(repo, wanted, probe, online, idle, required, options) + if idle - reserve < required: + return False, "" queued, committed = inflight_commitments(repo, wanted, probe, required, **options) idle_slots = (idle - queued - reserve) // required - # The total cap includes runs whose running jobs already reduce idle. Using - # idle for this cap would charge those busy runners twice. Idle slots below - # independently limit how much new work this snapshot can admit. limit = online - reserve if mixed else (online - reserve) // required counts = ( f"; queued={queued}, committed={committed}, run_limit={limit}" @@ -520,12 +623,20 @@ def selection_enabled(event: str, labels: str) -> bool: def routing_capacity(event: str, min_idle: str, demand: str, mode: str) -> dict[str, int]: - """Keep MQ-only arithmetic unchanged; enabled PRs have their own demand.""" + """Only MQs may overcommit; enabled PRs retain their own demand and reserve.""" required = required_capacity(min_idle, demand, mode) - capacity = {"required": required} + capacity = {"required": required, "queue": event == "merge_group"} if os.environ.get("DELL_PR_ROUTING") != "on": return capacity - pr_required = max(required, required_capacity("1", os.environ.get("MQ_PR_DEMAND") or str(PR_RUNNER_DEMAND))) + requested = required_capacity("1", os.environ.get("MQ_PR_DEMAND") or str(PR_RUNNER_DEMAND)) + # Selection labels reach the entire PR routing graph; a variable cannot + # shrink that graph. Keep both admission and peer weights above its count. + pr_required = max(required, PR_RUNNER_DEMAND, requested) + if event == "pull_request" and requested < PR_RUNNER_DEMAND: + warning( + f"MQ_PR_DEMAND={requested} is below the PR-routed job count " + f"{PR_RUNNER_DEMAND}; reserving the full inventory", hosted=False, + ) capacity.update(mq_required=required, pr_required=pr_required) if event == "pull_request": reserve = int(os.environ.get("MQ_PR_RESERVE") or demand or "1") @@ -563,7 +674,7 @@ def select_labels( ) -> str: """Return compact label JSON or an empty Actions output (hosted fallback). - Require spare capacity in both threshold and prefer modes. Pagination and + Both threshold and prefer modes allow bounded busy-Dell queueing. Pagination and each HTTP request stay bounded; runners visible in both scopes count only once. In-flight commitments and older undecided runs reserve capacity. """ @@ -617,12 +728,12 @@ def select_labels( counts += commitments prefix = {"prefer": "prefer mode: "}.get(mode, "") if admitted: - reason = f"{prefix}sufficient idle capacity" + reason = f"{prefix}Dell capacity available" return decision( f"{reason} ({counts})", json.dumps(requested, separators=(",", ":")), ) - reason = f"{prefix}insufficient idle capacity" + reason = f"{prefix}Dell capacity unavailable" reason = ( f"{reason} after in-flight commitments within the probe limit ({counts})" ) diff --git a/ci/test_ci_scope.py b/ci/test_ci_scope.py index 1494524cc2..4f322aee07 100644 --- a/ci/test_ci_scope.py +++ b/ci/test_ci_scope.py @@ -410,7 +410,7 @@ def test_queue_runner_probe_and_reusable_inputs(self): "MQ_RUNNER_STATUS_TOKEN: ${{ steps.mq-probe-app.outputs.token" " || secrets.MQ_RUNNER_STATUS_TOKEN }}", steps["Check Runner Availability"], ) - for variable in ("MQ_RUNNER_MODE", "MQ_RUNNER_MIN_IDLE"): + for variable in ("MQ_RUNNER_MODE", "MQ_RUNNER_MIN_IDLE", "MQ_RUNNER_MAX_RUNS"): self.assertIn(f"{variable}: ${{{{ vars.{variable} }}}}", steps["Check Runner Availability"]) self.assertIn('run: uv run --no-project ci/run_base_runner_selector.py', probe) self.assertIn("heavy-labels: ${{ steps.select.outputs.heavy-labels }}", probe) diff --git a/ci/test_member_pr_routing.py b/ci/test_member_pr_routing.py index b7c8d807e9..ff40bdb55a 100644 --- a/ci/test_member_pr_routing.py +++ b/ci/test_member_pr_routing.py @@ -21,8 +21,8 @@ class MemberPrRoutingTest(unittest.TestCase): def select(self, *, head="obs-rust/obs-studio", author="alice", state="active", - error=False, capacity=12, reserve="", enabled="on", event="pull_request", - peers=(), jobs=None, payload=None): + error=False, capacity=19, reserve="", enabled="on", event="pull_request", + peers=(), jobs=None, payload=None, busy=0, max_runs="", missing_peer_ids=False, mq_demand="6"): self.paths = [] current = {"id": 20, "event": event, "status": "in_progress", "created_at": "2026-11-19T12:00:00Z"} @@ -46,7 +46,7 @@ def respond(request, **kwargs): raise HTTPError(url.geturl(), 503, "unavailable", {}, None) result = {"state": state, "user": {"login": author}} elif url.path.endswith("/runners"): - entries = [{"id": i + 1, "status": "online", "busy": False, + entries = [{"id": i + 1, "status": "online", "busy": i < busy, "labels": [{"name": name} for name in json.loads(LABELS)]} for i in range(capacity)] result = {"total_count": len(entries), "runners": entries} @@ -54,6 +54,9 @@ def respond(request, **kwargs): query = parse_qs(url.query) entries = [r for r in runs if r["event"] == query["event"][0] and r["status"] == query["status"][0]] + if missing_peer_ids: + entries = [{key: value for key, value in run.items() if key != "id"} + if run["id"] != 20 else run for run in entries] result = {"total_count": len(entries), "workflow_runs": entries} else: run_id = int(url.path.split("/runs/")[1].split("/")[0]) @@ -68,9 +71,48 @@ def respond(request, **kwargs): "GITHUB_EVENT_NAME": event, "GITHUB_EVENT_PATH": str(event_path), "GITHUB_RUN_ID": "20", "GITHUB_TOKEN": "actions-token", "DELL_PR_ROUTING": enabled, "MQ_PR_RESERVE": reserve, - "MQ_PR_DEMAND": getattr(self, "pr_demand", "6"), + "MQ_RUNNER_MAX_RUNS": max_runs, + "MQ_PR_DEMAND": getattr(self, "pr_demand", "13"), }, clear=True), patch.object(selector, "urlopen", side_effect=respond), patch.object(selector.time, "sleep"): - return selector.select_labels(event, LABELS, "probe-token", "obs-rust/obs-studio", demand="6") + return selector.select_labels(event, LABELS, "probe-token", "obs-rust/obs-studio", demand=mq_demand) + + def test_manager_two_job_override_cannot_admit_thirteen_job_pr(self): + self.pr_demand = "2" + self.assertEqual(self.select(capacity=6, reserve="3", mq_demand="3"), "") + + def test_low_override_still_requires_full_pr_inventory_plus_reserve(self): + self.pr_demand = "2" + self.assertEqual(self.select(capacity=15, reserve="3", mq_demand="3"), "") + self.assertEqual(self.select(capacity=16, reserve="3", mq_demand="3"), LABELS) + + def test_small_override_cannot_underweight_an_existing_pr_for_queue(self): + self.pr_demand = "2" + peer = {"id": 10, "event": "pull_request", "status": "in_progress", + "created_at": "2026-11-19T11:59:00Z"} + jobs = {10: [{"name": "Build", "status": "in_progress", "labels": json.loads(LABELS)}]} + self.assertEqual(self.select(event="merge_group", capacity=6, mq_demand="3", + peers=[peer], jobs=jobs), "") + + def test_busy_queue_overcommit_never_relaxes_pr_idle_reserve(self): + self.assertEqual(self.select(event="merge_group", capacity=6, busy=6), LABELS) + self.assertEqual(self.select(capacity=19, busy=1, max_runs="100"), "") + self.assertEqual(self.select(capacity=19, max_runs="bad"), LABELS) + self.assertEqual(self.select(capacity=13, busy=1, reserve="0", max_runs="100"), "") + + def test_missing_ids_across_queue_and_pr_keep_both_weighted_reservations(self): + peers = [{"id": index, "event": event, "status": "in_progress", + "created_at": "2026-11-19T11:59:00Z"} + for index, event in ((10, "merge_group"), (11, "pull_request"))] + self.assertEqual(self.select(event="merge_group", capacity=6, + peers=peers, max_runs="2", missing_peer_ids=True), "") + self.assertFalse(any("/runs/-" in path for path in self.paths)) + + def test_mixed_queue_at_run_cap_uses_hosted(self): + peer = {"id": 10, "event": "merge_group", "status": "in_progress", + "created_at": "2026-11-19T11:59:00Z"} + jobs = {10: [{"name": "Build", "status": "in_progress", "labels": json.loads(LABELS)}]} + self.assertEqual(self.select(event="merge_group", capacity=6, busy=6, + peers=[peer], jobs=jobs, max_runs="1"), "") def test_member_with_idle_capacity_uses_dell(self): self.assertEqual(self.select(), LABELS) @@ -101,11 +143,11 @@ def test_membership_api_error_uses_hosted_without_retry(self): self.assertFalse(any(path.endswith("/runners") for path in self.paths)) def test_capacity_below_default_reserve_uses_hosted(self): - self.assertEqual(self.select(capacity=11), "") + self.assertEqual(self.select(capacity=18), "") def test_reserve_can_be_configured(self): - self.assertEqual(self.select(capacity=8, reserve="2"), LABELS) - self.assertEqual(self.select(capacity=7, reserve="2"), "") + self.assertEqual(self.select(capacity=15, reserve="2"), LABELS) + self.assertEqual(self.select(capacity=14, reserve="2"), "") def test_invalid_reserve_uses_hosted(self): for reserve in ("-1", "oops"): @@ -137,8 +179,8 @@ def test_existing_pr_dell_jobs_reduce_capacity_for_prs_and_queue(self): "created_at": "2026-11-19T11:59:00Z"} jobs = {10: [{"name": "Fast Gate ⚡", "status": "queued", "conclusion": None, "labels": json.loads(LABELS)} for _ in range(7)]} - for event in ("pull_request", "merge_group"): - self.assertEqual(self.select(event=event, peers=[peer], jobs=jobs), "") + self.assertEqual(self.select(event="pull_request", peers=[peer], jobs=jobs), "") + self.assertEqual(self.select(event="merge_group", peers=[peer], jobs=jobs, max_runs="4"), LABELS) def test_mixed_demands_admit_only_one_queue_run_beside_large_pr(self): self.pr_demand = "13" @@ -150,27 +192,42 @@ def test_mixed_demands_admit_only_one_queue_run_beside_large_pr(self): # commitment, not one six-slot queue run, even before its jobs start. jobs = {10: [{"name": "Fast Gate ⚡", "status": "in_progress", "conclusion": None, "labels": json.loads(LABELS)}]} - self.assertEqual(self.select(event="merge_group", capacity=19, peers=[pr], jobs=jobs), LABELS) - self.assertEqual(self.select(event="merge_group", capacity=19, peers=[pr, mq], jobs=jobs), "") + self.assertEqual(self.select(event="merge_group", capacity=19, peers=[pr], jobs=jobs, max_runs="4"), LABELS) + self.assertEqual(self.select(event="merge_group", capacity=19, peers=[pr, mq], jobs=jobs, max_runs="4"), "") class MemberPrWorkflowTest(unittest.TestCase): - def test_default_pr_demand_covers_expanded_routed_jobs(self): + def routed_pr_jobs(self): from tier2_matrix import matrix text = (ROOT / ".github/workflows/build-project.yaml").read_text() blocks = dict(re.findall(r"^ ([\w-]+):\n(.*?)(?=^ [\w-]+:|\Z)", text, re.M | re.S)) - count = 0 + jobs = {} for name, block in blocks.items(): if not re.search(r"runs-on: .*inputs.mq-heavy-labels", block): continue if name == "rust-linux-tier2": scheduling = matrix("pull_request") - count += len(scheduling["mode"]) * len(scheduling["part"]) + jobs[name] = len(scheduling["mode"]) * len(scheduling["part"]) else: variants = re.search(r"^ (?:target|alias): \[([^\]]+)\]", block, re.M) - count += len(variants[1].split(",")) if variants else 1 - self.assertEqual(selector.PR_RUNNER_DEMAND, count) + jobs[name] = len(variants[1].split(",")) if variants else 1 + return jobs + + def test_default_pr_demand_covers_expanded_routed_jobs(self): + self.assertEqual(selector.PR_RUNNER_DEMAND, sum(self.routed_pr_jobs().values())) + + def test_configured_pr_admission_demand_covers_actual_job_graph(self): + count = sum(self.routed_pr_jobs().values()) + for configured in ("", "2", "3", "13", "20"): + with self.subTest(configured=configured), patch.dict(os.environ, { + "DELL_PR_ROUTING": "on", "MQ_PR_DEMAND": configured, + "MQ_PR_RESERVE": "3", + }, clear=True): + capacity = selector.routing_capacity("pull_request", "1", "3", "") + self.assertGreaterEqual(capacity["required"], count) + self.assertEqual(capacity["reserve"], 3) + self.assertGreater(capacity["required"] + capacity["reserve"], 6) def test_base_owned_selector_and_membership_helper(self): ci = (ROOT / ".github/workflows/ci.yml").read_text() diff --git a/ci/test_select_mq_runner.py b/ci/test_select_mq_runner.py index 6ccf3d2cc3..537de37540 100644 --- a/ci/test_select_mq_runner.py +++ b/ci/test_select_mq_runner.py @@ -93,20 +93,20 @@ def test_online_all_labels(self): self.assertEqual(request.get_header("Authorization"), "Bearer fake-token") self.assertEqual(http.call_args.kwargs["timeout"], 10) - def test_all_six_online_runners_busy_use_hosted(self): + def test_all_six_online_runners_busy_with_no_commitments_use_dell(self): result, _, output = self.select([runner(busy=True) for _ in range(6)]) - self.assertEqual(result, "") + self.assertEqual(result, LABELS) self.assertIn("idle=0", output) self.assertIn("busy=6", output) - def test_prefer_busy_runners_do_not_override_capacity(self): + def test_prefer_busy_runners_queue_below_run_cap(self): entries = [runner(busy=True) for _ in range(6)] for scope in ("repository", "organization"): with self.subTest(scope=scope): options = { "runners" if scope == "repository" else "org_runners": entries } - self.assertEqual(self.select(mode="prefer", **options)[0], "") + self.assertEqual(self.select(mode="prefer", **options)[0], LABELS) def test_prefer_all_offline_uses_hosted(self): result, _, output = self.select([runner("offline", busy=True)], mode="prefer") @@ -131,8 +131,8 @@ def test_prefer_missing_token_uses_hosted_without_api(self): self.assertEqual(result, "") http.assert_not_called() - def test_empty_mode_keeps_idle_threshold_behavior(self): - self.assertEqual(self.select([runner(busy=True)], mode="")[0], "") + def test_empty_mode_allows_busy_but_requires_online_capacity(self): + self.assertEqual(self.select([runner(busy=True)], mode="")[0], LABELS) self.assertEqual(self.select([runner()], mode="")[0], LABELS) self.assertEqual(self.select([runner()], mode="", min_idle="2")[0], "") @@ -144,7 +144,7 @@ def test_prefer_respects_minimum_and_per_run_demand(self): entries += [runner(busy=True) for _ in range(6 - idle)] self.assertEqual( self.select(entries, mode=mode, demand="6")[0], - LABELS if idle == 6 else "", + LABELS, ) self.assertEqual(self.select([runner()], mode=mode, min_idle="2")[0], "") for option in ("min_idle", "demand"): @@ -157,27 +157,27 @@ def test_unknown_mode_uses_hosted_without_api(self): self.assertEqual(result, "") http.assert_not_called() - def test_default_requires_one_idle_runner(self): + def test_default_requires_one_online_runner(self): self.assertEqual(self.select([runner()])[0], LABELS) self.assertEqual(self.select([runner(), runner()])[0], LABELS) def test_five_busy_runners_do_not_strand_the_last_idle_runner(self): entries = [runner(busy=True) for _ in range(5)] self.assertEqual(self.select([*entries, runner()])[0], LABELS) - self.assertEqual(self.select([*entries, runner("offline")])[0], "") - self.assertEqual(self.select([*entries, runner()], demand="2")[0], "") + self.assertEqual(self.select([*entries, runner("offline")])[0], LABELS) + self.assertEqual(self.select([*entries, runner()], demand="2")[0], LABELS) - def test_empty_actions_thresholds_use_one_idle_runner(self): + def test_empty_actions_thresholds_use_one_online_runner(self): self.assertEqual(self.select([runner()], min_idle="", demand="")[0], LABELS) - def test_busy_offline_and_nonmatching_do_not_supply_idle_capacity(self): + def test_only_matching_online_runners_supply_capacity(self): entries = [ runner(), runner(busy=True), runner("offline"), runner(labels=["x64"]), ] - self.assertEqual(self.select(entries, min_idle="2")[0], "") + self.assertEqual(self.select(entries, min_idle="2")[0], LABELS) self.assertEqual(self.select([*entries, runner()], min_idle="2")[0], LABELS) def test_configurable_min_idle_and_demand(self): @@ -212,12 +212,12 @@ def test_same_runner_in_both_scopes_counts_once(self): def test_disagreeing_busy_state_is_conservative(self): same = runner() - self.assertEqual( - self.select( - [same, runner()], org_runners=[{**same, "busy": True}], min_idle="2" - )[0], - "", + selected, _, output = self.select( + [same, runner()], org_runners=[{**same, "busy": True}], min_idle="2" ) + self.assertEqual(selected, LABELS) + self.assertIn("online=2, busy=1, idle=1", output) + self.assertIn("admission=busy queue", output) def test_unknown_busy_state_is_not_idle(self): for busy in (None, "false", 0): @@ -288,7 +288,7 @@ def test_org_only_online_runner(self): ) self.assertEqual(request.get_header("Authorization"), "Bearer fake-token") self.assertEqual(http.call_args.kwargs["timeout"], 10) - self.assertIn("sufficient idle capacity", output) + self.assertIn("admission=idle", output) def test_org_forbidden_or_missing_preserves_repo_result(self): for status in (403, 404): @@ -507,15 +507,15 @@ def test_main_summary_and_environment_thresholds(self): ( [runner(busy=True) for _ in range(6)], {"MQ_RUNNER_MODE": "prefer"}, - "", + LABELS, "online=6, busy=6, idle=0", None, ), - ([runner(busy=True)], {"MQ_RUNNER_MODE": ""}, "", "idle=0", None), + ([runner(busy=True)], {"MQ_RUNNER_MODE": ""}, LABELS, "idle=0", None), ( [runner(busy=True) for _ in range(6)], {}, - "", + LABELS, "busy=6, idle=0, required=1", None, ), @@ -967,6 +967,9 @@ def respond(request, **kwargs): { "GITHUB_RUN_ID": str(run_id), "GITHUB_TOKEN": "actions-token", + # Preserve the recorded snapshot's original run bound for + # legacy reservation cases; default-cap tests unset it. + "MQ_RUNNER_MAX_RUNS": str(max(1, capacity // int(demand))), **(environment or {}), }, clear=True, @@ -987,6 +990,94 @@ def respond(request, **kwargs): self.assertLessEqual(http.call_count, selector.MAX_REQUESTS) return selected + def test_busy_dell_under_default_max_runs_queues_on_dell(self): + self.runs = self.runs[:2] + self.jobs[self.runs[0]["id"]] = self.fixture["queued_jobs"] + self.assertTrue(self.select(self.runs[1]["id"], busy=6, environment={"MQ_RUNNER_MAX_RUNS": ""})) + self.assertIn("committed=1, run_limit=3", self.output) + + def test_at_default_max_runs_uses_hosted_even_when_idle(self): + for run in self.runs[:3]: + self.jobs[run["id"]] = self.fixture["queued_jobs"][:1] + self.assertEqual(self.select(self.runs[3]["id"], environment={"MQ_RUNNER_MAX_RUNS": ""}), "") + self.assertIn("committed=3, run_limit=3", self.output) + + def test_configured_max_runs_caps_busy_admission(self): + self.runs = self.runs[:3] + for run in self.runs[:2]: + self.jobs[run["id"]] = self.fixture["queued_jobs"][:1] + self.assertEqual(self.select( + self.runs[2]["id"], busy=6, environment={"MQ_RUNNER_MAX_RUNS": "2"} + ), "") + self.assertIn("committed=2, run_limit=2", self.output) + + def test_invalid_max_runs_uses_hosted(self): + for value in ("0", "-1", "oops"): + with self.subTest(value=value): + self.assertEqual(self.select( + self.runs[0]["id"], environment={"MQ_RUNNER_MAX_RUNS": value} + ), "") + self.assertIn("ValueError", self.output) + + def test_busy_default_cap_admits_same_oldest_three_on_identical_snapshots(self): + actual = [bool(self.select( + run["id"], busy=6, environment={"MQ_RUNNER_MAX_RUNS": ""} + )) for run in reversed(self.runs)] + self.assertEqual(actual, [False, True, True, True]) + + def test_recorded_retry_without_current_run_reserves_all_peers(self): + # 44586086466 / 135779273252: entries=2, total_count=3, then an + # unwrapped KeyError. Reconstruct a stable missing-current snapshot; + # the historical log did not retain its API response or missing key. + current = self.runs[2]["id"] + self.runs = self.runs[:2] + + def stale_count(path, payload): + if path.endswith("/runs") and payload["workflow_runs"]: + return dict(payload, total_count=3) + return payload + + self.assertTrue(self.select(current, transform=stale_count, environment={"MQ_RUNNER_MAX_RUNS": ""})) + self.assertIn("collection changed (entries=2, total_count=3)", self.output) + self.assertIn(str(current), self.output) + self.assertIn("committed=2, run_limit=3", self.output) + self.assertNotIn("probe failed", self.output) + + def test_missing_run_fields_reserve_without_probe_failure(self): + self.runs = self.runs[:2] + peer, current = self.runs + for field in ("id", "created_at", "event", "status"): + with self.subTest(field=field): + def missing_field(path, payload): + if path.endswith("/runs"): + return dict(payload, workflow_runs=[ + {key: value for key, value in run.items() if key != field} + if run["id"] == peer["id"] else run + for run in payload["workflow_runs"] + ]) + return payload + + self.assertTrue(self.select(current["id"], transform=missing_field)) + self.assertIn(field, self.output) + self.assertIn("committed=1", self.output) + self.assertNotIn("probe failed", self.output) + + def test_missing_current_timestamp_reserves_every_undecided_peer(self): + current = self.runs[0]["id"] + for run in self.runs: + run.pop("created_at") + self.assertEqual(self.select(current, environment={"MQ_RUNNER_MAX_RUNS": ""}), "") + self.assertIn("committed=3", self.output) + self.assertNotIn("probe failed", self.output) + + def test_missing_job_labels_reserve_peer_without_probe_failure(self): + self.runs = self.runs[:2] + self.jobs[self.runs[0]["id"]] = [{"name": "Build", "status": "queued"}] + self.assertTrue(self.select(self.runs[1]["id"])) + self.assertIn("labels", self.output) + self.assertIn("committed=1", self.output) + self.assertNotIn("probe failed", self.output) + def test_four_simultaneous_recorded_runs_admit_oldest_two(self): # Each probe sees the same idle census and no heavy jobs yet. Reverse # probe order too: priority follows creation, not which probe ran first. @@ -998,9 +1089,11 @@ def test_four_simultaneous_recorded_runs_admit_oldest_two(self): {run["id"]: index < 2 for index, run in enumerate(self.runs)}, ) - def test_queued_dell_jobs_reduce_idle_capacity(self): + def test_queued_dell_jobs_reduce_idle_capacity_but_allow_queueing(self): self.jobs[self.runs[0]["id"]] = self.fixture["queued_jobs"][:4] - self.assertEqual(self.select(self.runs[1]["id"]), "") + self.assertTrue(self.select(self.runs[1]["id"])) + self.assertIn("queued=4, committed=1", self.output) + self.assertIn("idle_run_slots=0; admission=busy queue", self.output) def test_actions_api_error_uses_hosted(self): for error in ( @@ -1226,7 +1319,8 @@ def test_queued_label_matching_is_case_insensitive(self): dict(job, labels=[label.upper() for label in job["labels"]]) for job in self.fixture["queued_jobs"][:4] ] - self.assertEqual(self.select(self.runs[1]["id"]), "") + self.assertTrue(self.select(self.runs[1]["id"])) + self.assertIn("queued=4, committed=1", self.output) def test_incomplete_jobs_use_hosted(self): for endpoint in ("/jobs",): @@ -1290,9 +1384,10 @@ def more_jobs(path, payload): return dict(payload, total_count=101) if path.endswith("/jobs") else payload with patch.object(selector, "MAX_REQUESTS", 5): - self.assertEqual(self.select(self.runs[0]["id"], transform=more_jobs), "") + self.assertTrue(self.select(self.runs[0]["id"], transform=more_jobs)) self.assertIn("unprobed=1", self.output) self.assertIn("queued=100", self.output) + self.assertIn("committed=1, run_limit=2", self.output) self.assertIn("reserving at least 3 runners", self.output) self.assertNotIn("probe failed", self.output) diff --git a/docs/ci-tooling.md b/docs/ci-tooling.md index b0a807da80..a82734efbb 100644 --- a/docs/ci-tooling.md +++ b/docs/ci-tooling.md @@ -1367,24 +1367,36 @@ hosted without expanding permissions. Probe credentials remain on hosted runners after queued commitments and this reserve are subtracted. Unset/empty uses the default; a nonnegative integer overrides it, and invalid values select hosted. PRs have a separate `MQ_PR_DEMAND`: unset/empty defaults to `13`, the complete -expanded PR job inventory, including both sequential Tier 2 phases. The PR's -requirement is the maximum of this value and the existing MQ requirement. -Invalid/nonpositive values use hosted. This conservative default covers Fast -Gate, Cross Compile, Qt and link jobs in addition to the existing heavy jobs; -the inventory regression must change when that topology changes. A maintainer -may tune it to measured scoped PR concurrency, but must not set it below the -actual concurrent job demand. With default PR demand thirteen and reserve six, -nineteen matching idle runners are needed. A six-runner Dell stays hosted until -the maintainer explicitly tunes both PR settings for its workload or expands -the pool. `MQ_RUNNER_LABELS`, -`MQ_RUNNER_MIN_IDLE`, `MQ_RUNNER_DEMAND` and `MQ_RUNNER_MODE` retain their meanings. +expanded PR job inventory, including both sequential Tier 2 phases. This count +is a **hard admission floor**, checked against every expanded job that can +receive Dell labels for `pull_request` in `build-project.yaml`. Selection does +not throttle or restrict jobs after admission: Fast Gate, Cross Compile, Qt, +link jobs and the existing heavy jobs can all receive those labels. A smaller +positive override is clamped to the full inventory (and logged); it cannot turn +that thirteen-job graph into a two-job allocation. Larger overrides reserve +more capacity. Invalid/nonpositive values use hosted. The effective PR demand +is the maximum of this inventory, `MQ_PR_DEMAND` and the MQ requirement, and is +also used for in-flight PR commitments. Update the inventory floor and its +regression whenever the workflow graph changes. + +**Six-runner Dell:** keep `DELL_PR_ROUTING=off`, `MQ_PR_RESERVE=3` and +`MQ_PR_DEMAND=13`. There is no PR-demand setting that safely admits the current +thirteen-job graph with a three-runner queue reserve on six runners; at least +sixteen online/idle runners are needed. In particular, the reported +`MQ_PR_DEMAND=2`, `MQ_PR_RESERVE=3` configuration must select hosted. Useful PR +routing on six runners needs a separately bounded job subset or execution +limit before reducing the inventory floor. With reserve six, nineteen matching +idle runners are needed. `MQ_RUNNER_LABELS`, `MQ_RUNNER_MIN_IDLE`, +`MQ_RUNNER_DEMAND` and `MQ_RUNNER_MODE` retain their meanings. No variable is changed by merging this default-off feature. When enabled, both selectors account for live PR and queue job commitments. Undecided queue runs have priority over every PR, regardless of creation time; undecided PRs use oldest-first ordering. Commitments are weighted by each event's required runner capacity, so a large PR cannot count as one small MQ -run. Queue admission does not subtract the PR reserve. This remains a bounded +run. For MQ admission the budget is `MQ_RUNNER_MAX_RUNS * required`; PR admission +keeps the online-capacity limit and idle floor after subtracting the MQ reserve. +Queue admission does not subtract the PR reserve. This remains a bounded snapshot, with the same API visibility and simultaneous-selection limitations as queue admission, not a preemption system. Already-running PR jobs finish normally. Turning the kill switch off stops new @@ -1400,10 +1412,9 @@ only the event's `GITHUB_TOKEN`, with `contents: read` and `actions: read`; no permission increase or container-isolation change is introduced. Hosted/non-PR secret handling stays unchanged. No new workflow files or required checks. -Both the default threshold mode and `MQ_RUNNER_MODE=prefer` now require -idle capacity. Prefer remains accepted for existing configurations, but no -longer queues work on busy runners or bypasses the capacity cap. Unknown modes -fall back to hosted. +Both the default threshold mode and `MQ_RUNNER_MODE=prefer` use the same +bounded Dell queue admission. Prefer remains accepted for existing configurations; +unknown modes fall back to hosted. Set the optional repository Actions variable `MQ_RUNNER_DEMAND` to tune the per-run idle-capacity requirement without a workflow change. Unset or empty @@ -1430,58 +1441,62 @@ configured demand and actual queued-job labels; it does not hard-code a five-job peak or automatically infer future jobs before they appear in the API. The same configured requirement applies when path scope skips some heavy jobs. `MQ_RUNNER_MIN_IDLE` is an optional repository Actions variable (default `1`); -required idle capacity is `max(MQ_RUNNER_MIN_IDLE, MQ_RUNNER_DEMAND)`. With the -default demand, a six-runner Dell is selected only when all six matching -runners are idle. Otherwise the entire heavy pool falls back to hosted, -preserving the build/test handoff agreement. - -Busy runners do not supply idle capacity, and runner IDs visible in both -scopes count once (a busy observation wins). Empty threshold inputs use their -defaults; invalid or nonpositive values fall back to hosted. The job summary -records the choice, reason, online/busy/idle counts and required capacity, -plus unavailable scopes. With sufficient idle runners, admission also lists -queued/in-progress `ci.yml` merge-group runs and their latest-attempt jobs: - -- Subtract jobs queued for **all** the Dell labels from idle capacity. Running - jobs are already represented by busy runner state and are not subtracted again. -- Cap unfinished Dell-routed runs at `floor(online / required)`, without another - variable, and require at least one `idle_run_slots = floor((idle - queued) / - required)` for a new admission. Both counts are computed and logged for hosted - and Dell decisions. With six idle runners and required capacity three, two - runs fit. The total run cap uses online capacity because running Dell jobs - already reduce idle; using idle for that cap would charge busy runners twice. - For example, three busy and three idle runners can admit a second run alongside - one running Dell group. Only live Dell jobs (queued or in progress, without a - conclusion) commit a run slot. A routed heavy job's labels establish Dell or - hosted selection, including the macOS Rust artifact producer and completed jobs. - Inspect latest-attempt jobs even for workflows reporting `queued`: that status - does not prove their selector has yet to run. Before heavy jobs appear, older - unresolved runs reserve a slot, ordered by creation time then run ID; a visible - hosted choice releases that reservation regardless of workflow status. Thus - the same four-run idle snapshot admits the oldest two, regardless of probe order. - Known Dell commitments count even when newer than the probing run. - -The run cap covers the gap between selection and heavy-job creation; queued-job -subtraction also protects against a run queuing more work than its configured -demand. Unknown older choices conservatively hold slots while their selector has -not completed. The existing jobs read is the cheapest routing evidence: REST -does not expose job outputs, so no extra log/artifact read is needed. A completed -selector with no visible live Dell jobs releases its reservation during the API -visibility gap; success alone cannot distinguish an empty hosted output from -Dell labels. Heavy-job creation may lag selector completion by minutes while -hosted prerequisites wait. Live Dell-labelled jobs still commit capacity regardless -of selector status. A skipped selector also releases its reservation. Completed/cancelled -runs release reservations. Terminal jobs never commit Dell capacity, but their -labels retain the route so a run waiting only on hosted work cannot regain an -undecided reservation. Queued Dell jobs still hold a slot while waiting for build -dependencies. A hosted backlog therefore cannot retain undecided reservations. -Actions API errors, missing identity/token and unresolved collections select hosted; -jobs-budget exhaustion uses the conservative reservations above. Capacity-driven -hosted decisions are identified explicitly in the log and job summary for later -measurement. This is a bounded snapshot, not an -atomic reservation: API visibility lag or runners becoming busy/offline after -selection can still affect capacity. Fixtures reproduce the recorded four-run -burst and the hosted backlog in `ci/fixtures/mq-hosted-backlog.json`. +the idle fast case requires `max(MQ_RUNNER_MIN_IDLE, MQ_RUNNER_DEMAND)` runners +after subtracting queued Dell jobs. Online capacity must cover that requirement +even for busy admission, so offline or undersized pools still select hosted. + +`MQ_RUNNER_MAX_RUNS` caps concurrent committed Dell merge-group runs, including +the new admission. Unset or empty defaults to **3**; nonpositive or noninteger +values select hosted. Below this cap, use the idle fast case when available, +otherwise queue on the busy Dell. At the cap, select hosted even if the runner +census is idle. This cap replaces `floor(online / required)`; busy runners and +queued jobs no longer force hosted fallback while a run slot remains. + +The default comes from the six-runner Dell measurements on 2026-11-19 (reports +22/23, Refs #4896): 6 of 13 runs went hosted because 4–6 runners were busy with +required capacity 3. Dell completion had a 27.8-minute median against hosted's +46.3 minutes, while Dell job queue wait was p90 349 seconds and max 615 seconds. +Adding even that observed maximum wait to Dell's median gives 38.05 minutes, +8.25 minutes ahead of hosted's median. Three concurrent runs is a bounded first +step above the old two-run cap; tune the variable from subsequent completion +measurements rather than assuming this estimate is a measured overcommit result. +This queue policy does not authorize PRs to consume their reserved MQ capacity; +member-PR routing (#5279) retains its separate idle floor and queue reserve. +With PR routing enabled, retain capacity-weighted commitments and use a queue +budget of `MQ_RUNNER_MAX_RUNS * required` so a larger PR cannot count as one +smaller queue run. The cap does not relax PR admission, even with a zero reserve. + +Runner IDs visible in both scopes count once (a busy observation wins). The job +summary records online/busy/idle counts, required capacity, queued Dell jobs, +committed runs, run limit, idle run slots and the idle/busy admission reason. +Queued/in-progress `ci.yml` merge-group runs and latest-attempt jobs establish +commitments: + +- Live Dell-labelled jobs commit one run slot, including newer runs. Completed + heavy jobs establish the route but release capacity when no Dell jobs remain, + including runs waiting only for hosted macOS work. +- Older undecided runs reserve a slot, ordered by creation time then run ID. + Identical snapshots therefore admit the same oldest runs. Hosted choices, + completed/skipped selectors with no live Dell jobs, and terminal runs release + reservations. Inspect jobs even for workflows reporting `queued`. +- A stable retried run listing may omit the current run (run `44586086466`). Without + its timestamp, count every undecided peer as older, and log the missing run ID + rather than raising `KeyError`. Missing run timestamps sort before known peers; + a missing current timestamp reserves every undecided peer. Missing run IDs + hold a slot without a jobs lookup. Missing event/status fields use the filtered + endpoint's values. Missing job fields similarly reserve that + peer's run and required demand, with the missing key named in the warning. + Run `44586086466` logged a bare `KeyError` after a snapshot retry; its API + response was not retained, so tests reconstruct both missing-current and + missing-field shapes rather than claiming an exact historical payload. +- Jobs-budget exhaustion reserves each unprobed run and at least its required + demand (or its larger observed queued-job count). API errors, missing Actions + identity/token and unresolved collections still select hosted. + +Capacity-driven hosted routing is identified in the log and summary. This is a +bounded snapshot, not an atomic reservation: API visibility lag or runners going +offline after selection can still change capacity. Existing fixtures reproduce +the four-run burst, hosted backlog and collection races in `ci/fixtures/`. Report 15 measured 168 idle Dell minutes (six idle runners, required three), while 78 Dell-routable hosted jobs waited over five minutes (803 job-minutes), and only 3 of 19 queue runs used the Dell. Read-only selector logs for