Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 21 additions & 8 deletions ci/local_gate.py
Original file line number Diff line number Diff line change
Expand Up @@ -256,6 +256,10 @@ def rustc(*a: str) -> str:

SELF_TEST_MAX_WORKERS = 8
SELF_TEST_TIMEOUT = 600
# The offline verifier performs many serial Git object lookups. Shared worker
# hosts can take >600 s even though an isolated run took 160 s (Refs #4896).
# Leave the pool and ordinary self-test deadlines unchanged.
SELF_TEST_TIMEOUTS = {"test_pr_attestation.py": 1800}


def self_test_workers() -> int:
Expand All @@ -264,6 +268,11 @@ def self_test_workers() -> int:
return max(1, min(SELF_TEST_MAX_WORKERS, usable))


def subprocess_output(stdout: str | bytes | None, stderr: str | bytes | None) -> str:
"""TimeoutExpired can carry bytes even when the child runs with text=True."""
return "".join(part.decode(errors="replace") if isinstance(part, bytes) else part or "" for part in (stdout, stderr))


def self_test() -> None: # noqa: C901
"""The gate's and the repo layout's unit tests: every ci/test_*.py.

Expand All @@ -284,23 +293,22 @@ def self_test() -> None: # noqa: C901

def one(path: Path) -> tuple[str, int, str, float]:
start = time.monotonic()
timeout = SELF_TEST_TIMEOUTS.get(path.name, SELF_TEST_TIMEOUT)
try:
result = subprocess.run(
[sys.executable, "-W", "default::ResourceWarning", "-m", "unittest", "discover", "-s", "ci", "-p", path.name],
cwd=ROOT,
capture_output=True,
text=True,
errors="replace",
timeout=SELF_TEST_TIMEOUT,
timeout=timeout,
)
except subprocess.TimeoutExpired as e:
output = (e.stdout or "") + (e.stderr or "")
if isinstance(output, bytes):
output = output.decode(errors="replace")
return path.name, 1, f"{output}\ntimed out after {SELF_TEST_TIMEOUT} s", time.monotonic() - start
output = subprocess_output(e.stdout, e.stderr)
return path.name, 1, f"{output}\ntimed out after {timeout} s", time.monotonic() - start
# 5: the file has no tests (as the old single discover allowed).
code = 0 if result.returncode == 5 else result.returncode
output = result.stdout + result.stderr
output = subprocess_output(result.stdout, result.stderr)
# A leaked file or response warns from a destructor, which no -W error
# can turn into a failure; 781 such lines buried the Checks log (#2235).
if code == 0 and "ResourceWarning" in output:
Expand All @@ -312,8 +320,13 @@ def one(path: Path) -> tuple[str, int, str, float]:
print(f"+ unittest: {len(files)} ci/test_*.py files, {workers} at a time", flush=True)
failed = []
with ThreadPoolExecutor(max_workers=workers) as pool:
for future in as_completed([pool.submit(one, path) for path in files]):
name, code, output, secs = future.result()
futures = {pool.submit(one, path): path for path in files}
for future in as_completed(futures):
try:
name, code, output, secs = future.result()
except Exception as e:
# A launch/worker error must not discard the other files' results.
name, code, output, secs = futures[future].name, 1, f"{type(e).__name__}: {e}", 0.0
print(f" {'ok ' if code == 0 else 'FAIL'} {secs:6.1f}s {name}", flush=True)
if code:
failed.append((name, output))
Expand Down
75 changes: 75 additions & 0 deletions ci/test_local_gate.py
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,81 @@ def test_workers_follow_usable_cpus_capped(self):
with mock.patch.object(local_gate.os, "process_cpu_count", create=True, return_value=usable):
self.assertEqual(local_gate.self_test_workers(), want, usable)

def test_timeout_output_is_readable_and_other_failures_are_aggregated(self):
for stdout, stderr, expected in (
(b"stdout\xff\n", None, "stdout\ufffd"),
(None, b"stderr\xff\n", "stderr\ufffd"),
(b"stdout\xff\n", b"stderr\xff\n", "stdout\ufffd\nstderr\ufffd"),
(None, None, "timed out"),
("text stdout\n", None, "text stdout"),
(None, "text stderr\n", "text stderr"),
("text stdout\n", "text stderr\n", "text stdout\ntext stderr"),
(b"bytes stdout\n", "text stderr\n", "bytes stdout\ntext stderr"),
("text stdout\n", b"bytes stderr\n", "text stdout\nbytes stderr"),
):
with self.subTest(stdout=stdout, stderr=stderr):
output = io.StringIO()
calls = []

def invoke(cmd, **kwargs):
name = cmd[cmd.index("-p") + 1]
calls.append(name)
if name == "test_timeout.py":
raise subprocess.TimeoutExpired(cmd, 600, output=stdout, stderr=stderr)
return subprocess.CompletedProcess(cmd, 1, "other failure\n", "diagnostic\n")

files = [Path("test_timeout.py"), Path("test_other.py")]
with (
mock.patch.object(Path, "glob", return_value=files),
mock.patch.object(local_gate.subprocess, "run", side_effect=invoke),
contextlib.redirect_stdout(output),
self.assertRaises(subprocess.CalledProcessError) as failure,
):
local_gate.self_test()
self.assertCountEqual(calls, [p.name for p in files])
self.assertCountEqual(failure.exception.cmd[1:], calls)
self.assertIn(expected, output.getvalue())
self.assertIn("timed out after 600 s", output.getvalue())
self.assertIn("other failure\ndiagnostic", output.getvalue())

def test_launch_error_does_not_hide_other_failures(self):
output = io.StringIO()

def invoke(cmd, **kwargs):
if cmd[-1] == "test_launch.py":
raise OSError("process launch failed")
return subprocess.CompletedProcess(cmd, 1, b"other failure\xff\n", None)

with (
mock.patch.object(Path, "glob", return_value=[Path("test_launch.py"), Path("test_other.py")]),
mock.patch.object(local_gate.subprocess, "run", side_effect=invoke),
contextlib.redirect_stdout(output),
self.assertRaises(subprocess.CalledProcessError) as failure,
):
local_gate.self_test()
self.assertCountEqual(failure.exception.cmd[1:], ["test_launch.py", "test_other.py"])
self.assertIn("process launch failed", output.getvalue())
self.assertIn("other failure\ufffd", output.getvalue())

def test_only_git_heavy_verifier_has_a_longer_deadline(self):
calls = {}
output = io.StringIO()

def invoke(cmd, **kwargs):
calls[cmd[-1]] = kwargs["timeout"]
raise subprocess.TimeoutExpired(cmd, kwargs["timeout"])

with (
mock.patch.object(Path, "glob", return_value=[Path("test_pr_attestation.py"), Path("test_other.py")]),
mock.patch.object(local_gate.subprocess, "run", side_effect=invoke),
contextlib.redirect_stdout(output),
self.assertRaises(subprocess.CalledProcessError),
):
local_gate.self_test()
self.assertEqual(calls, {"test_pr_attestation.py": 1800, "test_other.py": 600})
self.assertIn("timed out after 1800 s", output.getvalue())
self.assertIn("timed out after 600 s", output.getvalue())


class EnvironmentHintTest(unittest.TestCase):
"""Environment failures get a one-line hint naming the documented fix
Expand Down
Loading