From 5bd55d718b329e379a809e7bb54fb9d09ea55d51 Mon Sep 17 00:00:00 2001 From: Zach Vorhies Date: Thu, 19 Nov 2026 12:22:01 -0800 Subject: [PATCH 1/2] test(ci): reproduce local-gate timeout aggregation failures (Refs #4896) --- ci/test_local_gate.py | 75 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/ci/test_local_gate.py b/ci/test_local_gate.py index 85807067a8..f3e88be341 100644 --- a/ci/test_local_gate.py +++ b/ci/test_local_gate.py @@ -113,6 +113,81 @@ def test_workers_follow_usable_cpus_capped(self): with mock.patch.object(local_gate.os, "process_cpu_count", create=True, return_value=usable): self.assertEqual(local_gate.self_test_workers(), want, usable) + def test_timeout_output_is_readable_and_other_failures_are_aggregated(self): + for stdout, stderr, expected in ( + (b"stdout\xff\n", None, "stdout\ufffd"), + (None, b"stderr\xff\n", "stderr\ufffd"), + (b"stdout\xff\n", b"stderr\xff\n", "stdout\ufffd\nstderr\ufffd"), + (None, None, "timed out"), + ("text stdout\n", None, "text stdout"), + (None, "text stderr\n", "text stderr"), + ("text stdout\n", "text stderr\n", "text stdout\ntext stderr"), + (b"bytes stdout\n", "text stderr\n", "bytes stdout\ntext stderr"), + ("text stdout\n", b"bytes stderr\n", "text stdout\nbytes stderr"), + ): + with self.subTest(stdout=stdout, stderr=stderr): + output = io.StringIO() + calls = [] + + def invoke(cmd, **kwargs): + name = cmd[cmd.index("-p") + 1] + calls.append(name) + if name == "test_timeout.py": + raise subprocess.TimeoutExpired(cmd, 600, output=stdout, stderr=stderr) + return subprocess.CompletedProcess(cmd, 1, "other failure\n", "diagnostic\n") + + files = [Path("test_timeout.py"), Path("test_other.py")] + with ( + mock.patch.object(Path, "glob", return_value=files), + mock.patch.object(local_gate.subprocess, "run", side_effect=invoke), + contextlib.redirect_stdout(output), + self.assertRaises(subprocess.CalledProcessError) as failure, + ): + local_gate.self_test() + self.assertCountEqual(calls, [p.name for p in files]) + self.assertCountEqual(failure.exception.cmd[1:], calls) + self.assertIn(expected, output.getvalue()) + self.assertIn("timed out after 600 s", output.getvalue()) + self.assertIn("other failure\ndiagnostic", output.getvalue()) + + def test_launch_error_does_not_hide_other_failures(self): + output = io.StringIO() + + def invoke(cmd, **kwargs): + if cmd[-1] == "test_launch.py": + raise OSError("process launch failed") + return subprocess.CompletedProcess(cmd, 1, b"other failure\xff\n", None) + + with ( + mock.patch.object(Path, "glob", return_value=[Path("test_launch.py"), Path("test_other.py")]), + mock.patch.object(local_gate.subprocess, "run", side_effect=invoke), + contextlib.redirect_stdout(output), + self.assertRaises(subprocess.CalledProcessError) as failure, + ): + local_gate.self_test() + self.assertCountEqual(failure.exception.cmd[1:], ["test_launch.py", "test_other.py"]) + self.assertIn("process launch failed", output.getvalue()) + self.assertIn("other failure\ufffd", output.getvalue()) + + def test_only_git_heavy_verifier_has_a_longer_deadline(self): + calls = {} + output = io.StringIO() + + def invoke(cmd, **kwargs): + calls[cmd[-1]] = kwargs["timeout"] + raise subprocess.TimeoutExpired(cmd, kwargs["timeout"]) + + with ( + mock.patch.object(Path, "glob", return_value=[Path("test_pr_attestation.py"), Path("test_other.py")]), + mock.patch.object(local_gate.subprocess, "run", side_effect=invoke), + contextlib.redirect_stdout(output), + self.assertRaises(subprocess.CalledProcessError), + ): + local_gate.self_test() + self.assertEqual(calls, {"test_pr_attestation.py": 1800, "test_other.py": 600}) + self.assertIn("timed out after 1800 s", output.getvalue()) + self.assertIn("timed out after 600 s", output.getvalue()) + class EnvironmentHintTest(unittest.TestCase): """Environment failures get a one-line hint naming the documented fix From 926bc28c81b14bef5fa1d125ee08d473b5f32712 Mon Sep 17 00:00:00 2001 From: Zach Vorhies Date: Thu, 19 Nov 2026 12:26:38 -0800 Subject: [PATCH 2/2] ci: keep local-gate failure reporting from crashing on subprocess timeouts Refs #4896 --- ci/local_gate.py | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/ci/local_gate.py b/ci/local_gate.py index 8d9b6630cd..11944fc6db 100644 --- a/ci/local_gate.py +++ b/ci/local_gate.py @@ -256,6 +256,10 @@ def rustc(*a: str) -> str: SELF_TEST_MAX_WORKERS = 8 SELF_TEST_TIMEOUT = 600 +# The offline verifier performs many serial Git object lookups. Shared worker +# hosts can take >600 s even though an isolated run took 160 s (Refs #4896). +# Leave the pool and ordinary self-test deadlines unchanged. +SELF_TEST_TIMEOUTS = {"test_pr_attestation.py": 1800} def self_test_workers() -> int: @@ -264,6 +268,11 @@ def self_test_workers() -> int: return max(1, min(SELF_TEST_MAX_WORKERS, usable)) +def subprocess_output(stdout: str | bytes | None, stderr: str | bytes | None) -> str: + """TimeoutExpired can carry bytes even when the child runs with text=True.""" + return "".join(part.decode(errors="replace") if isinstance(part, bytes) else part or "" for part in (stdout, stderr)) + + def self_test() -> None: # noqa: C901 """The gate's and the repo layout's unit tests: every ci/test_*.py. @@ -284,6 +293,7 @@ def self_test() -> None: # noqa: C901 def one(path: Path) -> tuple[str, int, str, float]: start = time.monotonic() + timeout = SELF_TEST_TIMEOUTS.get(path.name, SELF_TEST_TIMEOUT) try: result = subprocess.run( [sys.executable, "-W", "default::ResourceWarning", "-m", "unittest", "discover", "-s", "ci", "-p", path.name], @@ -291,16 +301,14 @@ def one(path: Path) -> tuple[str, int, str, float]: capture_output=True, text=True, errors="replace", - timeout=SELF_TEST_TIMEOUT, + timeout=timeout, ) except subprocess.TimeoutExpired as e: - output = (e.stdout or "") + (e.stderr or "") - if isinstance(output, bytes): - output = output.decode(errors="replace") - return path.name, 1, f"{output}\ntimed out after {SELF_TEST_TIMEOUT} s", time.monotonic() - start + output = subprocess_output(e.stdout, e.stderr) + return path.name, 1, f"{output}\ntimed out after {timeout} s", time.monotonic() - start # 5: the file has no tests (as the old single discover allowed). code = 0 if result.returncode == 5 else result.returncode - output = result.stdout + result.stderr + output = subprocess_output(result.stdout, result.stderr) # A leaked file or response warns from a destructor, which no -W error # can turn into a failure; 781 such lines buried the Checks log (#2235). if code == 0 and "ResourceWarning" in output: @@ -312,8 +320,13 @@ def one(path: Path) -> tuple[str, int, str, float]: print(f"+ unittest: {len(files)} ci/test_*.py files, {workers} at a time", flush=True) failed = [] with ThreadPoolExecutor(max_workers=workers) as pool: - for future in as_completed([pool.submit(one, path) for path in files]): - name, code, output, secs = future.result() + futures = {pool.submit(one, path): path for path in files} + for future in as_completed(futures): + try: + name, code, output, secs = future.result() + except Exception as e: + # A launch/worker error must not discard the other files' results. + name, code, output, secs = futures[future].name, 1, f"{type(e).__name__}: {e}", 0.0 print(f" {'ok ' if code == 0 else 'FAIL'} {secs:6.1f}s {name}", flush=True) if code: failed.append((name, output))