diff --git a/.github/workflows/package-tests.yaml b/.github/workflows/package-tests.yaml new file mode 100644 index 000000000..c0f33aab1 --- /dev/null +++ b/.github/workflows/package-tests.yaml @@ -0,0 +1,115 @@ +# Nightly: the tests bundled in the published packages, run again against the newest +# conda-forge dependencies (tools/package_tests.py), to catch breakages that come from +# conda-forge rather than from RoboStack, e.g. ABI changes (RoboStack/ros-humble#274). +# Every package with a test file (shared/tests/, distros//tests/) is tested in its +# newest build, on each platform. The results are kept in one issue per distribution +# (": package tests", label package-tests), closed when every test passes. +name: package-tests + +on: + schedule: + - cron: "31 2 * * *" + workflow_dispatch: + inputs: + distros: + description: Space-separated distributions to test (default all) + required: false + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false + +permissions: + contents: read + +env: + PYTHONIOENCODING: utf-8 + +jobs: + plan: + runs-on: ubuntu-latest + outputs: + distros: ${{ steps.plan.outputs.distros }} + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - name: Distributions + id: plan + env: + SELECTED: ${{ inputs.distros }} + run: | + distros=$(python3 -c 'import json, os, pathlib + selected = os.environ.get("SELECTED", "").split() + print(json.dumps(selected or sorted(p.parent.name for p in pathlib.Path("distros").glob("*/vinca.yaml"))))') + echo "distros=$distros" | tee -a "$GITHUB_OUTPUT" + + test: + needs: plan + strategy: + fail-fast: false + matrix: + distro: ${{ fromJSON(needs.plan.outputs.distros) }} + target: + - { os: ubuntu-latest, platform: linux-64 } + - { os: ubuntu-24.04-arm, platform: linux-aarch64 } + - { os: macos-15-intel, platform: osx-64 } + - { os: macos-15, platform: osx-arm64 } + - { os: windows-2022, platform: win-64 } + name: ${{ matrix.distro }} ${{ matrix.target.platform }} + runs-on: ${{ matrix.target.os }} + timeout-minutes: 180 + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + # git auto-maintenance can leave a lock file in the git clones pixi caches + # (e.g. vinca's), which breaks every later run restoring that cache + - name: Disable git auto-maintenance + run: git config --global maintenance.auto false && git config --global gc.auto 0 && git config --global core.longpaths true + - uses: prefix-dev/setup-pixi@v0.11.0 + with: + locked: true # the lock file must match pixi.toml + pixi-version: v0.81.0 + cache-write: false # saved from main only (pixi-cache.yaml) + - name: Test the published packages + # a failed test is reported in the issue (report job), not as a failed job + continue-on-error: true + shell: bash + env: + DISTRO: ${{ matrix.distro }} + PLATFORM: ${{ matrix.target.platform }} + run: pixi run python tools/package_tests.py run --distro "$DISTRO" --platform "$PLATFORM" --results "results/$DISTRO-$PLATFORM.json" + - uses: actions/upload-artifact@v6 + if: always() + with: + name: package-tests-${{ matrix.distro }}-${{ matrix.target.platform }} + path: results/ + if-no-files-found: ignore + retention-days: 14 + + report: + needs: [plan, test] + if: always() + runs-on: ubuntu-latest + permissions: + contents: read + issues: write # the ": package tests" issues + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - uses: prefix-dev/setup-pixi@v0.11.0 + with: + locked: true # the lock file must match pixi.toml + pixi-version: v0.81.0 + cache-write: false # saved from main only (pixi-cache.yaml) + - uses: actions/download-artifact@v7 + with: + pattern: package-tests-* + path: results/ + - name: Update the issues + env: + GH_TOKEN: ${{ github.token }} + DISTROS: ${{ join(fromJSON(needs.plan.outputs.distros), ' ') }} + run: pixi run python tools/package_tests.py report results/ --distros "$DISTROS" diff --git a/README.md b/README.md index 8908e6034..68e64f679 100644 --- a/README.md +++ b/README.md @@ -151,6 +151,20 @@ There are three ways to trigger a command: "Maintainers" means the repository's owners, members and collaborators. +### Nightly package tests + +`.github/workflows/package-tests.yaml` runs `tools/package_tests.py` every night (or via *Actions › package-tests › Run workflow*, for some distributions): +- **What:** every package with a test file (`shared/tests/.yaml`, `distros//tests/.yaml`), in its newest build on the distribution's channel, with every variant of that build (e.g. each Python), on all five platforms. +- **How:** `rattler-build test` runs the tests bundled in the published package, with the newest dependencies from the distribution's channel and conda-forge. This catches breakages that come from conda-forge rather than from RoboStack, e.g. an ABI change in a dependency a package was built against. +- **Report:** one issue per distribution, *``: package tests* (label `package-tests`), with the failed tests and their output; it is closed when every test passes. Builds from before their test file was added carry no tests and are listed as such. + +Locally: + +```bash +pixi run python tools/package_tests.py run --distro jazzy --platform osx-arm64 --results results/jazzy.json [--package ros2-rclpy] +pixi run python tools/package_tests.py report results/ +``` + ### New distribution ```bash diff --git a/tools/maintenance.py b/tools/maintenance.py index 38d2d9527..60e93d429 100644 --- a/tools/maintenance.py +++ b/tools/maintenance.py @@ -1077,16 +1077,16 @@ def report_issue_body(distro: str, data: dict, migrations: str = "", run_url: st return body -def publish_report(repo: str, distro: str, body: str, conflicts: bool) -> str | None: - """Create or update the distribution's dependency-report issue: open while there - are conflicts, closed otherwise. Returns its URL.""" - title = f"{distro}: dependency report" +def publish_issue(repo: str, title: str, label: str, body: str, open_: bool) -> str | None: + """Create or update the issue with this title and label: open while open_ (e.g. + there are conflicts), closed otherwise. Returns its URL (None: nothing to report + and no issue to close).""" query = f'.[] | select(.title == {json.dumps(title)}) | {{number, state, html_url}} | @json' - out = _gh(["api", "--paginate", f"repos/{repo}/issues?labels={REPORT_LABEL}&state=all&per_page=100", "--jq", query]) + out = _gh(["api", "--paginate", f"repos/{repo}/issues?labels={label}&state=all&per_page=100", "--jq", query]) # the open one, else the newest issues = sorted((json.loads(l) for l in out.splitlines() if l.strip()), key=lambda i: (i["state"] == "open", i["number"])) - if not issues and not conflicts: + if not issues and not open_: return None with tempfile.NamedTemporaryFile("w", suffix=".md", delete=False) as fh: fh.write(body) @@ -1094,15 +1094,21 @@ def publish_report(repo: str, distro: str, body: str, conflicts: bool) -> str | if issues: issue = issues[-1] _gh(["api", "-X", "PATCH", f"repos/{repo}/issues/{issue['number']}", "-F", f"body=@{fh.name}", - "-f", f"state={'open' if conflicts else 'closed'}"]) + "-f", f"state={'open' if open_ else 'closed'}"]) return issue["html_url"] created = _gh(["api", f"repos/{repo}/issues", "-f", f"title={title}", "-F", f"body=@{fh.name}", - "-f", f"labels[]={REPORT_LABEL}"]) + "-f", f"labels[]={label}"]) return json.loads(created)["html_url"] finally: os.unlink(fh.name) +def publish_report(repo: str, distro: str, body: str, conflicts: bool) -> str | None: + """Create or update the distribution's dependency-report issue: open while there + are conflicts, closed otherwise. Returns its URL.""" + return publish_issue(repo, f"{distro}: dependency report", REPORT_LABEL, body, conflicts) + + def dependency_report(distro: str) -> Result: """Check the distribution's dependencies against its pins (linux-64) and keep its dependency-report issue up to date (in GitHub Actions; locally only the report).""" diff --git a/tools/package_tests.py b/tools/package_tests.py new file mode 100644 index 000000000..a79573fc9 --- /dev/null +++ b/tools/package_tests.py @@ -0,0 +1,273 @@ +"""Run the tests bundled in the published packages again, against today's dependencies. + +Packages carry their recipe tests (rattler-build bundles them in info/tests), so they +can be tested again after they were uploaded. Installing each one with the newest +dependencies from conda-forge catches breakages that come from conda-forge rather +than from RoboStack, e.g. an ABI change in a dependency that a RoboStack package was +built against (RoboStack/ros-humble#274). The nightly package-tests workflow runs this +for every distribution and platform and keeps one issue per distribution +(": package tests", label package-tests), closed when every test passes. + + python tools/package_tests.py select --distro jazzy --platform linux-64 + python tools/package_tests.py run --distro jazzy --platform linux-64 --results results/jazzy-linux-64.json + python tools/package_tests.py report results/ # in GitHub Actions also updates the issues + +The tested packages are the ones with a test file (shared/tests/.yaml or +distros//tests/.yaml): vinca adds no tests of its own. Each is tested in +its newest build on the distribution's channel, every variant of it (e.g. each Python). +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import subprocess +import sys +import tempfile +import time +import urllib.request +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import robostack as rs + +REPORT_LABEL = "package-tests" +REPORT_MARKER = "" +MAX_ISSUE_BODY = 60000 # GitHub's limit is 65536 characters +LOG_TAIL = 60 # lines of a failed test's output kept in the results +# a test that rattler-build ran: a script test prints "Testing commands:", the others +# "✔ test passed!" (not its closing "all tests passed!", which it also prints +# for a package without tests) +RAN_TEST = re.compile(r"^Testing commands:|\b(? list[str]: + """Package names with a test file, shared or the distribution's own.""" + names = set() + for tests in (rs.ROOT / "shared" / "tests", rs.DISTROS / distro / "tests"): + names.update(p.stem for p in tests.glob("*.yaml")) + return sorted(names) + + +def fetch_json(url: str, attempts: int = 3) -> dict: + for attempt in range(1, attempts + 1): + try: + with urllib.request.urlopen(url, timeout=120) as response: + return json.load(response) + except urllib.error.HTTPError as error: + if error.code == 404: + return {} + if attempt == attempts: + raise + except OSError: + if attempt == attempts: + raise + time.sleep(10 * attempt) + return {} + + +def newest_builds(repodata: dict, names: list[str]) -> list[dict]: + """For each name, every variant of its newest build: the records with the version + and build number of the newest upload (one per build string, e.g. per Python).""" + records: dict[str, list[dict]] = {} + for section in ("packages", "packages.conda"): + for filename, record in (repodata.get(section) or {}).items(): + if record.get("name") in names: + records.setdefault(record["name"], []).append({**record, "fn": filename}) + selected = [] + for name in sorted(records): + newest = max(records[name], key=lambda r: (r.get("build_number", 0), r.get("timestamp", 0))) + variants: dict[str, dict] = {} + for record in records[name]: + if (record["version"], record.get("build_number")) != (newest["version"], newest.get("build_number")): + continue + # the same build in both formats: prefer .conda + if record["build"] not in variants or record["fn"].endswith(".conda"): + variants[record["build"]] = record + selected += sorted(variants.values(), key=lambda r: r["build"]) + return selected + + +def select(distro: str, platform: str) -> list[dict]: + """The package files to test on this platform: {name, version, build, subdir, url}.""" + names = tested_packages(distro) + base = rs.repodata_url(distro) + selected = [] + for subdir in (platform, "noarch"): + repodata = fetch_json(f"{base}/{subdir}/repodata.json") + for record in newest_builds(repodata, names): + selected.append({"name": record["name"], "version": record["version"], "build": record["build"], + "subdir": subdir, "url": f"{base}/{subdir}/{record['fn']}"}) + return selected + + +def test_command(distro: str, package_file: Path) -> list[str]: + """rattler-build test with the build channels: the distribution's channel for the + other RoboStack packages, conda-forge for the rest (newest versions, not a lock).""" + return ["rattler-build", "test", "--package-file", str(package_file), "-c", rs.channel_url(distro), + "-c", rs.CONDA_FORGE, "--channel-priority", "disabled", "--log-style", "plain"] + + +def test_package(distro: str, package_file: Path) -> tuple[bool, str, int]: + """Run the package's bundled tests: (passed, output, number of tests that ran).""" + proc = subprocess.run(test_command(distro, package_file), capture_output=True, text=True, + encoding="utf-8", errors="replace", check=False) + log = proc.stdout + proc.stderr + return proc.returncode == 0, log, sum(bool(RAN_TEST.search(line.strip())) for line in log.splitlines()) + + +def tail(text: str, lines: int) -> str: + return "\n".join(text.splitlines()[-lines:]) + + +def run(distro: str, platform: str, results: Path, only: list[str] | None = None) -> int: + """Test the selected packages (only these names, if given) one by one; write the + results (JSON) and return 1 when one failed.""" + packages = [p for p in select(distro, platform) if not only or p["name"] in only] + print(f"{distro} {platform}: {len(packages)} package files to test", flush=True) + outcome = [] + with tempfile.TemporaryDirectory() as tmp: + for package in packages: + label = f"{package['name']} {package['version']} {package['build']}" + print(f"::group::{label}" if os.environ.get("GITHUB_ACTIONS") else f"== {label}", flush=True) + package_file = Path(tmp) / package["url"].rsplit("/", 1)[1] + started = time.monotonic() + try: + urllib.request.urlretrieve(package["url"], package_file) + ok, log, count = test_package(distro, package_file) + # launch tests can time out on DDS discovery: a failure is tried once more, + # and a pass then is reported as flaky + flaky = False + if not ok: + print(log, "\n❌ failed, trying again", flush=True) + ok, log, count = test_package(distro, package_file) + flaky = ok + except OSError as error: + ok, log, count, flaky = False, f"download failed: {error}", 0, False + finally: + package_file.unlink(missing_ok=True) + print(log, flush=True) + if os.environ.get("GITHUB_ACTIONS"): + print("::endgroup::", flush=True) + # a build from before its test file was added carries no tests: not a pass + status = "failed" if not ok else "flaky" if flaky else "passed" if count else "no-tests" + print(f"{ {'passed': '✅', 'flaky': '⚠️', 'failed': '❌', 'no-tests': '➖'}[status]} {label}: {status} " + f"({count} tests, {time.monotonic() - started:.0f} s)", flush=True) + outcome.append({**package, "ok": ok, "status": status, "tests": count, + **({} if ok else {"log": tail(log, LOG_TAIL)})}) + results.parent.mkdir(parents=True, exist_ok=True) + results.write_text(json.dumps({"distro": distro, "platform": platform, "packages": outcome}, indent=1)) + failed = [p for p in outcome if not p["ok"]] + untested = [p for p in outcome if p["status"] == "no-tests"] + flaky = [p for p in outcome if p["status"] == "flaky"] + print(f"{distro} {platform}: {len(outcome) - len(failed) - len(untested)} passed ({len(flaky)} on the second " + f"try), {len(failed)} failed, {len(untested)} without bundled tests", flush=True) + return 1 if failed else 0 + + +def load_results(directory: Path) -> dict[str, list[dict]]: + """distro -> results of each platform (from run's JSON files, searched recursively: + GitHub Actions downloads every artifact into its own directory).""" + by_distro: dict[str, list[dict]] = {} + for path in sorted(directory.rglob("*.json")): + data = json.loads(path.read_text()) + by_distro.setdefault(data["distro"], []).append(data) + return by_distro + + +def issue_body(distro: str, results: list[dict], missing: list[str], run_url: str = "") -> str: + """The package-tests issue of a distribution.""" + failed = [(r["platform"], p) for r in sorted(results, key=lambda r: r["platform"]) + for p in r["packages"] if not p["ok"]] + tested = sum(p.get("status") != "no-tests" for r in results for p in r["packages"]) + untested = sorted({(p["name"], p["build"]) for r in results for p in r["packages"] if p.get("status") == "no-tests"}) + flaky = [(r["platform"], p) for r in sorted(results, key=lambda r: r["platform"]) + for p in r["packages"] if p.get("status") == "flaky"] + lines = [ + REPORT_MARKER, + f"Nightly test of the published **{distro}** packages: the tests bundled in each package with a test file " + f"(`shared/tests/`, `distros/{distro}/tests/`), run with `rattler-build test` against the newest " + "dependencies from conda-forge, to catch breakages that come from conda-forge (e.g. ABI changes). " + "robostack-bot updates this issue every night and closes it when every test passes.", + "", + f"**{len(failed)} of {tested} failed**" if failed else f"✅ **All {tested} passed.**", + "", + ] + if missing: + lines += [f"⚠️ No results for {', '.join(missing)} (the job failed before testing, see the workflow run).", ""] + if failed: + lines += ["| platform | package | build |", "|---|---|---|"] + lines += [f"| {platform} | `{p['name']}` {p['version']} | `{p['build']}` |" for platform, p in failed] + lines.append("") + for platform, p in failed: + lines += [f"
{platform}: {p['name']} {p['build']}", "", "```text", + p.get("log", ""), "```", "", "
", ""] + if flaky: + lines += [("Passed only on the second try: " + + ", ".join(f"`{p['name']}` ({platform})" for platform, p in flaky)), ""] + if untested: + lines += [("Without bundled tests (built before their test file was added; tested again once rebuilt): " + + ", ".join(f"`{name}` (`{build}`)" for name, build in untested)), ""] + if run_url: + lines.append(f"[Workflow run]({run_url})") + body = "\n".join(lines) + if len(body) > MAX_ISSUE_BODY: + body = body[:MAX_ISSUE_BODY] + "\n\n… (truncated, see the workflow run)" + return body + + +def report(directory: Path, platforms: list[str], distros: list[str] | None = None) -> int: + """Summarise the results and, in GitHub Actions, update each distribution's issue + (of the given distributions, also those without any results).""" + import maintenance # only for the issue helpers + + run_url = "" + if run_id := os.environ.get("GITHUB_RUN_ID"): + run_url = f"{os.environ.get('GITHUB_SERVER_URL', 'https://github.com')}/{os.environ.get('GITHUB_REPOSITORY')}/actions/runs/{run_id}" + failing = False + by_distro = load_results(directory) + for distro in sorted(set(distros or []) | set(by_distro)): + results = by_distro.get(distro, []) + missing = sorted(set(platforms) - {r["platform"] for r in results}) + failed = sum(not p["ok"] for r in results for p in r["packages"]) + bad = bool(failed or missing) + failing |= bad + body = issue_body(distro, results, missing, run_url) + print(f"{distro}: {failed} failed" + (f", no results for {', '.join(missing)}" if missing else ""), flush=True) + if os.environ.get("GITHUB_ACTIONS"): + url = maintenance.publish_issue(maintenance._repository(), f"{distro}: package tests", REPORT_LABEL, body, bad) + print(f" {url or 'all passed, no issue to close'}", flush=True) + else: + print(body) + return 1 if failing else 0 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + sub = parser.add_subparsers(dest="command", required=True) + p = sub.add_parser("select", help="list the package files that would be tested (JSON)") + p.add_argument("--distro", required=True) + p.add_argument("--platform", required=True) + p = sub.add_parser("run", help="test them, writing the results") + p.add_argument("--distro", required=True) + p.add_argument("--platform", required=True) + p.add_argument("--results", type=Path, required=True) + p.add_argument("--package", action="append", help="only this package (repeatable; for local runs)") + p = sub.add_parser("report", help="summarise results (and update the issues in GitHub Actions)") + p.add_argument("directory", type=Path) + p.add_argument("--platforms", default="linux-64 linux-aarch64 osx-64 osx-arm64 win-64", + help="platforms that should have results (default: all five)") + p.add_argument("--distros", default="", help="distributions that should have results (default: those found)") + ns = parser.parse_args() + if ns.command == "select": + print(json.dumps(select(ns.distro, ns.platform), indent=1)) + return 0 + if ns.command == "run": + return run(ns.distro, ns.platform, ns.results, ns.package) + return report(ns.directory, ns.platforms.split(), ns.distros.split()) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/tests/test_package_tests.py b/tools/tests/test_package_tests.py new file mode 100644 index 000000000..153653d2d --- /dev/null +++ b/tools/tests/test_package_tests.py @@ -0,0 +1,78 @@ +import json + +import package_tests + + +def record(name, version, build, number, timestamp=0): + return {"name": name, "version": version, "build": build, "build_number": number, "timestamp": timestamp} + + +def test_tested_packages_are_the_test_files(tmp_path, monkeypatch): + (tmp_path / "shared" / "tests").mkdir(parents=True) + (tmp_path / "distros" / "jazzy" / "tests").mkdir(parents=True) + (tmp_path / "shared" / "tests" / "ros2-rclpy.yaml").write_text("tests: []\n") + (tmp_path / "shared" / "tests" / "ros2-std-msgs").mkdir() # the files a test uses + (tmp_path / "distros" / "jazzy" / "tests" / "ros2-rcutils.yaml").write_text("tests: []\n") + monkeypatch.setattr(package_tests.rs, "ROOT", tmp_path) + monkeypatch.setattr(package_tests.rs, "DISTROS", tmp_path / "distros") + assert package_tests.tested_packages("jazzy") == ["ros2-rclpy", "ros2-rcutils"] + assert package_tests.tested_packages("humble") == ["ros2-rclpy"] + + +def test_newest_builds_keeps_every_variant_of_the_newest_build(): + repodata = { + "packages.conda": { + "a-1-py312_25.conda": record("ros2-a", "1", "py312_25", 25, 1), + "a-1-py314_27.conda": record("ros2-a", "1", "py314_27", 27, 2), + "a-1-py313_27.conda": record("ros2-a", "1", "py313_27", 27, 3), + "b-2-h1_3.conda": record("ros2-b", "2", "h1_3", 3), + "c-1-h_1.conda": record("ros2-untested", "1", "h_1", 1), + }, + # the same build in the old format too: the .conda one is tested + "packages": {"a-1-py314_27.tar.bz2": record("ros2-a", "1", "py314_27", 27, 2)}, + } + selected = package_tests.newest_builds(repodata, ["ros2-a", "ros2-b"]) + assert [(r["name"], r["build"], r["fn"]) for r in selected] == [ + ("ros2-a", "py313_27", "a-1-py313_27.conda"), + ("ros2-a", "py314_27", "a-1-py314_27.conda"), + ("ros2-b", "h1_3", "b-2-h1_3.conda"), + ] + assert package_tests.newest_builds({}, ["ros2-a"]) == [] + + +def test_ran_test_counts_the_tests_rattler_build_ran(): + log = ["Testing commands:", "✔ python imports test passed!", "✔ package contents test passed!", + "✔ all tests passed!", "✔ Successfully updated the test environment"] + assert sum(bool(package_tests.RAN_TEST.search(line)) for line in log) == 3 + + +def test_issue_body_lists_failures_and_untested_builds(): + results = [ + {"distro": "jazzy", "platform": "win-64", "packages": [ + {"name": "ros2-a", "version": "1", "build": "h_27", "ok": False, "status": "failed", "log": "boom"}, + {"name": "ros2-b", "version": "2", "build": "h_25", "ok": True, "status": "no-tests"}, + {"name": "ros2-c", "version": "3", "build": "h_27", "ok": True, "status": "flaky"}, + ]}, + {"distro": "jazzy", "platform": "linux-64", "packages": [ + {"name": "ros2-a", "version": "1", "build": "h_27", "ok": True, "status": "passed"}, + ]}, + ] + body = package_tests.issue_body("jazzy", results, ["osx-64"], "https://run") + assert "**1 of 3 failed**" in body and "second try: `ros2-c` (win-64)" in body + assert "| win-64 | `ros2-a` 1 | `h_27` |" in body and "boom" in body + assert "`ros2-b` (`h_25`)" in body and "No results for osx-64" in body + assert body.endswith("[Workflow run](https://run)") + clean = package_tests.issue_body("jazzy", results[1:], []) + assert "All 1 passed" in clean and "|---|" not in clean + + +def test_report_includes_distributions_without_results(tmp_path, capsys, monkeypatch): + monkeypatch.delenv("GITHUB_ACTIONS", raising=False) + (tmp_path / "package-tests-jazzy-linux-64").mkdir() + (tmp_path / "package-tests-jazzy-linux-64" / "jazzy-linux-64.json").write_text(json.dumps( + {"distro": "jazzy", "platform": "linux-64", + "packages": [{"name": "ros2-a", "version": "1", "build": "h_1", "ok": True, "status": "passed"}]})) + assert package_tests.report(tmp_path, ["linux-64"], ["jazzy"]) == 0 + # humble's jobs failed before writing results: reported, and the run fails + assert package_tests.report(tmp_path, ["linux-64"], ["humble", "jazzy"]) == 1 + assert "humble: 0 failed, no results for linux-64" in capsys.readouterr().out