From c4da6f7c46ee1fd7bbb0aa7505b76465a2a7b184 Mon Sep 17 00:00:00 2001 From: Tobias Fischer Date: Fri, 23 Oct 2026 15:44:46 +1000 Subject: [PATCH] Record each recipe's build time in the build jobs; pack the build jobs by time Each build-branch job records how long each recipe took to build (`rs build-ci`, in distros//work/build_timings.jsonl) and keeps that file as an artifact (vinca-gha --keep-file). robostack-bot update-build-timings (weekly, with the dependency reports) merges the last week's into shared/build_timings/.json, {recipe: {distro: seconds}}. With times for the platform, `rs gha` passes vinca-gha --timings (each recipe's average over the distributions, so a recipe new to one distribution still has a time): it packs each stage's recipes into jobs by time and builds compatibility packages right after their package. Until then the jobs are cut by count as before. A workflow over GitHub's size limit is retried with fewer runners first. The vinca pin moves to RoboStack/vinca@c52ce0c (#179: --timings, --keep-file). Co-Authored-By: Claude Opus 5.5 --- .github/ISSUE_TEMPLATE/3_robostack_bot.yaml | 3 +- .github/workflows/bot.yaml | 14 +++- README.md | 2 + pixi.lock | 12 +-- pixi.toml | 2 +- tools/build_timings.py | 78 ++++++++++++++++++ tools/maintenance.py | 17 ++++ tools/robostack.py | 31 +++++++- tools/tests/test_build_timings.py | 87 +++++++++++++++++++++ 9 files changed, 230 insertions(+), 16 deletions(-) create mode 100644 tools/build_timings.py create mode 100644 tools/tests/test_build_timings.py diff --git a/.github/ISSUE_TEMPLATE/3_robostack_bot.yaml b/.github/ISSUE_TEMPLATE/3_robostack_bot.yaml index 5f274e991..5fbd56593 100644 --- a/.github/ISSUE_TEMPLATE/3_robostack_bot.yaml +++ b/.github/ISSUE_TEMPLATE/3_robostack_bot.yaml @@ -18,6 +18,7 @@ body: | `rebuild` | maintainers | New builds of the given ROS packages (per-package build numbers above every released build); add `--with-dependents` to the packages to also rebuild everything that depends on them. The PR lists the packages. | | `update-vinca` | maintainers | Move the vinca pin in `pixi.toml` to the newest commit of its branch (or the ref given under Packages), lock and check. The PR lists the vinca commits. | | `dependency-report` | maintainers | Check the dependencies against the pins and update the distribution's dependency-report issue (closed when nothing conflicts). Runs weekly. | + | `update-build-timings` | maintainers | Record the build times of the last week's build-branch runs in `shared/build_timings/`, which the build workflows use to pack each stage's recipes into jobs by time. Runs weekly. | On a pull request, `@robostack-bot rebuild-dependents` (maintainers) commits the rebuilds its ABI check suggests to the pull request's branch. - type: dropdown @@ -25,7 +26,7 @@ body: attributes: label: Command options: [add-package, update-rosdistro-snapshot, find-stale-packages, update-conda-forge-pinning, rebuild, update-vinca, - dependency-report] + dependency-report, update-build-timings] validations: required: true - type: input diff --git a/.github/workflows/bot.yaml b/.github/workflows/bot.yaml index dc7e43553..cbc0337e6 100644 --- a/.github/workflows/bot.yaml +++ b/.github/workflows/bot.yaml @@ -8,14 +8,15 @@ # new pins can't be installed together). # - weekly (Mondays): a dependency check of every distribution, kept in one issue per # distribution (": dependency report", label dependency-report), closed when -# nothing conflicts. +# nothing conflicts; and a PR with the build times the week's build-branch runs +# recorded (shared/build_timings/, for packing the build jobs by time). # - on demand (owners, members and collaborators): # Actions > robostack-bot > Run workflow, an issue from the "robostack-bot command" # template, or a comment: @robostack-bot [] # commands: update-rosdistro-snapshot [...|all] | find-stale-packages [...|all] | # update-conda-forge-pinning [...|all] | add-package ... [...] | # rebuild ... [...|all] [--with-dependents] | update-vinca [] | -# dependency-report [...|all] | +# dependency-report [...|all] | update-build-timings | # rebuild-dependents (a comment on a pull request: commits the rebuilds its ABI # check suggests to its branch; not possible for pull requests from forks) # - package requests (the "Package request" issue form or `@robostack-bot add-package`, @@ -35,7 +36,7 @@ on: command: type: choice options: [add-package, update-rosdistro-snapshot, find-stale-packages, update-conda-forge-pinning, rebuild, - update-vinca, dependency-report] + update-vinca, dependency-report, update-build-timings] distro: description: Distribution (for the per-distribution commands; empty = all) required: false @@ -90,7 +91,9 @@ jobs: case "$EVENT" in schedule) if [ "$SCHEDULE" = "43 4 * * 1" ]; then - jobs=$(ROBOSTACK_BOT_BODY="@robostack-bot dependency-report all" ROBOSTACK_BOT_ASSOCIATION=OWNER pixi run -q rs parse-command) + # weekly: the dependency reports, and the build times of the week's builds + jobs=$(ROBOSTACK_BOT_BODY="@robostack-bot dependency-report all" ROBOSTACK_BOT_ASSOCIATION=OWNER pixi run -q rs parse-command | + python3 -c 'import json,sys; print(json.dumps(json.load(sys.stdin) + [{"command":"update-build-timings","distro":"","args":"","preview":False}]))') echo "jobs=$jobs" | tee -a "$GITHUB_OUTPUT" exit 0 fi @@ -113,6 +116,7 @@ jobs: matrix: include: ${{ fromJSON(needs.plan.outputs.jobs) }} permissions: + actions: read # update-build-timings: the build-branch runs' artifacts contents: write pull-requests: write issues: write @@ -160,6 +164,8 @@ jobs: # pushes to the pull request's branch itself (with this job's checkout token) rebuild-dependents) pixi run rs rebuild-dependents "$ARGS" --summary "$RUNNER_TEMP/summary.md" ;; update-vinca) pixi run rs update-vinca $ARGS --summary "$RUNNER_TEMP/summary.md" ;; + # the build-branch runs' build-times artifacts (actions: read) + update-build-timings) pixi run rs update-build-timings --summary "$RUNNER_TEMP/summary.md" ;; # creates / updates the distribution's dependency-report issue dependency-report) pixi run rs "$DISTRO" dependency-report --summary "$RUNNER_TEMP/summary.md" ;; esac diff --git a/README.md b/README.md index 8908e6034..077611dfa 100644 --- a/README.md +++ b/README.md @@ -119,6 +119,7 @@ label doesn't start one by itself. Nothing has to be reset before merging. - Each selected distribution is built on all five platforms, with its own build cache. The `check` job runs `pixi run rs check`. - **After a merge** (`main.yaml`): for every changed distribution and platform, the job regenerates the recipes and the staged build workflow, and pushes them to `buildbranch__`. - Each build branch carries its workflow as `.github/workflows/build.yaml`, which runs `.scripts/build_*.sh` in `distros//work`. + - The workflow builds in stages (by dependency depth), each stage in parallel jobs. Each job records how long its recipes took to build and keeps that as an artifact; `robostack-bot update-build-timings` merges them weekly into `shared/build_timings/.json` (each recipe's time per distribution). With those times, each stage's recipes are packed into jobs by time (vinca-gha `--timings`, each recipe's average over the distributions), so that no single job holds up its stage; until a platform has times, its jobs are cut by count as before. - Those workflows build the packages and upload them to the distribution's channel on the servers `upload_to` in `distro.yaml` lists (prefix.dev and/or anaconda.org, the same channel name on each; packages already there are skipped). `read_from` says where the builds look up what's already published. - **Repository variables:** - `ROBOSTACK_UPLOAD_CHANNEL`: upload only to this prefix.dev channel instead (for example a test channel), or `none` to build without uploading. Use the canonical reference: `/` for a channel that isn't the namespace's primary one (e.g. `tobias-fischer/robostack-distro-test`). @@ -143,6 +144,7 @@ label doesn't start one by itself. Nothing has to be reset before merging. | `rebuild-dependents` | maintainers | Only as a comment on a pull request: commits the build-number entries the ABI check's comment suggests (its `yaml` blocks per distribution) to the pull request's branch, keeping existing entries' other keys. Pull requests from forks can't be pushed to: the bot replies with the entries to add instead. | | `update-vinca []` | maintainers | Moves the vinca pin in `pixi.toml` to the newest commit of the pinned repository's branch (`robostack-integration` for Tobias-Fischer/vinca, else its default branch) or of ``, runs `pixi lock`, re-renders the pinning and runs `rs check`. The PR links the comparison and lists the vinca commits. | | `dependency-report [... \| all]` | maintainers | Runs the dependency check (`check-deps`, linux-64) and keeps one issue per distribution, *``: dependency report* (label `dependency-report`), up to date with the conflicts, notes and conda-forge migration status; the issue is closed when nothing conflicts. Runs weekly (Mondays). | +| `update-build-timings` | maintainers | Merges the build times the last week's build-branch runs recorded (`rs build-ci`, kept as each job's artifact) into `shared/build_timings/.json` and opens a PR. The build workflows pack each stage's recipes into jobs by these times. Runs weekly (Mondays). | There are three ways to trigger a command: - a *Package request* or *robostack-bot command* issue; diff --git a/pixi.lock b/pixi.lock index 51ff7b8a4..410aa5803 100644 --- a/pixi.lock +++ b/pixi.lock @@ -90,7 +90,7 @@ environments: - conda: https://prefix.dev/conda-forge/noarch/tomli-2.5.0-pyh5ded981_0.conda - conda: https://prefix.dev/conda-forge/noarch/typing_extensions-4.16.0-pyhcf101f3_0.conda - conda: https://prefix.dev/conda-forge/noarch/tzdata-2026c-h151e31d_0.conda - - pypi: git+https://github.com/RoboStack/vinca.git?rev=3d552f4001634fd26376b2bf5fe37fe81ef3dc22#3d552f4001634fd26376b2bf5fe37fe81ef3dc22 + - pypi: git+https://github.com/RoboStack/vinca.git?rev=c52ce0cc30bc9f121dae0980b1f947debf1d8c1c#c52ce0cc30bc9f121dae0980b1f947debf1d8c1c - pypi: https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/12/74/ef303ec980ef962ffe548fa36cb138bb399d1d1315731ba7bb81da85f7e0/rosdistro-1.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl @@ -170,7 +170,7 @@ environments: - conda: https://prefix.dev/conda-forge/noarch/tomli-2.5.0-pyh5ded981_0.conda - conda: https://prefix.dev/conda-forge/noarch/typing_extensions-4.16.0-pyhcf101f3_0.conda - conda: https://prefix.dev/conda-forge/noarch/tzdata-2026c-h151e31d_0.conda - - pypi: git+https://github.com/RoboStack/vinca.git?rev=3d552f4001634fd26376b2bf5fe37fe81ef3dc22#3d552f4001634fd26376b2bf5fe37fe81ef3dc22 + - pypi: git+https://github.com/RoboStack/vinca.git?rev=c52ce0cc30bc9f121dae0980b1f947debf1d8c1c#c52ce0cc30bc9f121dae0980b1f947debf1d8c1c - pypi: https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/12/74/ef303ec980ef962ffe548fa36cb138bb399d1d1315731ba7bb81da85f7e0/rosdistro-1.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl @@ -243,7 +243,7 @@ environments: - conda: https://prefix.dev/conda-forge/osx-64/tk-8.6.13-ha6c374e_4.conda - conda: https://prefix.dev/conda-forge/osx-64/yaml-0.2.5-had63097_3.conda - conda: https://prefix.dev/conda-forge/osx-64/zstd-1.5.7-hbc1a06c_7.conda - - pypi: git+https://github.com/RoboStack/vinca.git?rev=3d552f4001634fd26376b2bf5fe37fe81ef3dc22#3d552f4001634fd26376b2bf5fe37fe81ef3dc22 + - pypi: git+https://github.com/RoboStack/vinca.git?rev=c52ce0cc30bc9f121dae0980b1f947debf1d8c1c#c52ce0cc30bc9f121dae0980b1f947debf1d8c1c - pypi: https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/12/74/ef303ec980ef962ffe548fa36cb138bb399d1d1315731ba7bb81da85f7e0/rosdistro-1.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl @@ -316,7 +316,7 @@ environments: - conda: https://prefix.dev/conda-forge/osx-arm64/tk-8.6.13-hbeba79b_4.conda - conda: https://prefix.dev/conda-forge/osx-arm64/yaml-0.2.5-h74c22ad_3.conda - conda: https://prefix.dev/conda-forge/osx-arm64/zstd-1.5.7-hf451053_7.conda - - pypi: git+https://github.com/RoboStack/vinca.git?rev=3d552f4001634fd26376b2bf5fe37fe81ef3dc22#3d552f4001634fd26376b2bf5fe37fe81ef3dc22 + - pypi: git+https://github.com/RoboStack/vinca.git?rev=c52ce0cc30bc9f121dae0980b1f947debf1d8c1c#c52ce0cc30bc9f121dae0980b1f947debf1d8c1c - pypi: https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/12/74/ef303ec980ef962ffe548fa36cb138bb399d1d1315731ba7bb81da85f7e0/rosdistro-1.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl @@ -389,7 +389,7 @@ environments: - conda: https://prefix.dev/conda-forge/win-64/vcomp14-14.51.36247-habf1de7_41.conda - conda: https://prefix.dev/conda-forge/win-64/yaml-0.2.5-h6a83c73_3.conda - conda: https://prefix.dev/conda-forge/win-64/zstd-1.5.7-h534d264_7.conda - - pypi: git+https://github.com/RoboStack/vinca.git?rev=3d552f4001634fd26376b2bf5fe37fe81ef3dc22#3d552f4001634fd26376b2bf5fe37fe81ef3dc22 + - pypi: git+https://github.com/RoboStack/vinca.git?rev=c52ce0cc30bc9f121dae0980b1f947debf1d8c1c#c52ce0cc30bc9f121dae0980b1f947debf1d8c1c - pypi: https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/12/74/ef303ec980ef962ffe548fa36cb138bb399d1d1315731ba7bb81da85f7e0/rosdistro-1.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl @@ -3289,7 +3289,7 @@ packages: - zstd >=1.5.7,<1.6.0a0 size: 387535 timestamp: 1786599623274 -- pypi: git+https://github.com/RoboStack/vinca.git?rev=3d552f4001634fd26376b2bf5fe37fe81ef3dc22#3d552f4001634fd26376b2bf5fe37fe81ef3dc22 +- pypi: git+https://github.com/RoboStack/vinca.git?rev=c52ce0cc30bc9f121dae0980b1f947debf1d8c1c#c52ce0cc30bc9f121dae0980b1f947debf1d8c1c name: vinca version: 0.2.0 requires_dist: diff --git a/pixi.toml b/pixi.toml index 716bb690b..923c413a0 100644 --- a/pixi.toml +++ b/pixi.toml @@ -36,7 +36,7 @@ git = "*" [pypi-dependencies] # One vinca for all distributions. -vinca = { git = "https://github.com/RoboStack/vinca.git", rev = "3d552f4001634fd26376b2bf5fe37fe81ef3dc22" } +vinca = { git = "https://github.com/RoboStack/vinca.git", rev = "c52ce0cc30bc9f121dae0980b1f947debf1d8c1c" } # For local development, temporarily use (and regenerate pixi.lock): # vinca = { path = "../vinca", editable = true } diff --git a/tools/build_timings.py b/tools/build_timings.py new file mode 100644 index 000000000..820753048 --- /dev/null +++ b/tools/build_timings.py @@ -0,0 +1,78 @@ +"""Measured build times, for packing the build jobs by time (vinca-gha --timings). + +Each build-branch job records how long each of its recipes took to build (`rs +build-ci`, in distros//work/build_timings.jsonl) and keeps that file as an artifact +(vinca-gha --keep-file). `robostack-bot update-build-timings` merges the last week's into +shared/build_timings/.json, {recipe: {distro: seconds}}, and `rs gha` +hands vinca-gha each recipe's average over the distributions: a recipe new to one +distribution still has a time (the names are the same, ros2-). + + pixi run python tools/build_timings.py # what the bot command does +""" + +from __future__ import annotations + +import json +import statistics +import subprocess +import sys +import tempfile +from datetime import datetime, timedelta, timezone +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import robostack as rs + +TIMINGS = rs.ROOT / "shared" / "build_timings" +JOURNAL = "build_timings.jsonl" + + +def note(distro: str, recipe: str, seconds: float) -> None: + """Record a recipe's build time in the work directory's journal.""" + from vinca.platforms import get_conda_subdir + + entry = {"platform": get_conda_subdir(), "recipe": recipe, "seconds": round(seconds, 1)} + with (rs.work_dir(distro) / JOURNAL).open("a", encoding="utf-8") as fh: + fh.write(json.dumps(entry) + "\n") + + +def collect(days: int = 8) -> int: + """Merge the build times the build-branch runs of the last `days` days recorded into + shared/build_timings/ (newer runs win); returns how many times were recorded.""" + since = (datetime.now(timezone.utc) - timedelta(days=days)).strftime("%Y-%m-%d") + runs = json.loads(subprocess.run( + ["gh", "run", "list", "--repo", "RoboStack/robostack", "--workflow", "build.yaml", "--status", "completed", + "--created", f">={since}", "--limit", "1000", "--json", "databaseId,headBranch,createdAt"], + capture_output=True, text=True, check=True).stdout) + files = {p.stem: json.loads(p.read_text()) for p in TIMINGS.glob("*.json")} + count = 0 + with tempfile.TemporaryDirectory() as tmp: + for run in sorted(runs, key=lambda r: r["createdAt"]): + distro = run["headBranch"].removeprefix("buildbranch_").split("_")[0] + out = Path(tmp) / str(run["databaseId"]) + subprocess.run(["gh", "run", "download", str(run["databaseId"]), "--repo", "RoboStack/robostack", + "--dir", str(out)], capture_output=True, check=False) + for journal in sorted(out.rglob(JOURNAL)): + for line in journal.read_text().splitlines(): + entry = json.loads(line) + files.setdefault(entry["platform"], {}).setdefault(entry["recipe"], {})[distro] = entry["seconds"] + count += 1 + TIMINGS.mkdir(parents=True, exist_ok=True) + for platform, recipes in files.items(): + (TIMINGS / f"{platform}.json").write_text(json.dumps(recipes, indent=0, sort_keys=True) + "\n") + return count + + +def vinca_timings(platform: str, path: Path) -> bool: + """Write vinca-gha's --timings file for the platform: each recipe's average over the + distributions. False when there are no times yet.""" + source = TIMINGS / f"{platform}.json" + if not source.is_file(): + return False + recipes = {r: round(statistics.fmean(by.values()), 1) for r, by in json.loads(source.read_text()).items()} + path.write_text(json.dumps({"recipes": recipes}) + "\n") + return True + + +if __name__ == "__main__": + print(f"{collect()} build times recorded") diff --git a/tools/maintenance.py b/tools/maintenance.py index 38d2d9527..db1719a35 100644 --- a/tools/maintenance.py +++ b/tools/maintenance.py @@ -43,6 +43,7 @@ "rebuild-dependents": "on a pull request: add the rebuilds its ABI check suggests (pushes to its branch)", "update-vinca": "move the vinca pin to the newest commit of its branch (opens a PR)", "dependency-report": "check the dependencies against the pins, in the distribution's dependency-report issue", + "update-build-timings": "record the build times of the last week's build-branch runs, for batching by time (opens a PR)", } PER_DISTRO_COMMANDS = ("update-rosdistro-snapshot", "find-stale-packages", "update-conda-forge-pinning", "dependency-report") @@ -1127,6 +1128,20 @@ def dependency_report(distro: str) -> Result: return Result(title, f"Report: {url}" if url else "No conflicts, and no report issue to close.") +def update_build_timings() -> Result: + """Merge the build times of the last week's build-branch runs into shared/build_timings/.""" + import build_timings + + before = {p: p.read_text() for p in build_timings.TIMINGS.glob("*.json")} + count = build_timings.collect() + changed = sorted(p.name for p in build_timings.TIMINGS.glob("*.json") if before.get(p) != p.read_text()) + if not changed: + return Result("Build times are up to date", "No new build times in the last week's build-branch runs.") + return Result("Update the build times", f"{count} build times from the last week's build-branch runs, in " + + ", ".join(f"`shared/build_timings/{name}`" for name in changed) + + ". `rs gha` packs each stage's recipes into jobs by them.", changed=True) + + def _default_ci_yaml() -> str: return (rs.TOOLS / "ci_default.yaml").read_text() @@ -1340,6 +1355,8 @@ def main(command: str, argv: list[str], distro: str | None = None) -> int: return report(update_vinca(args.ref), args.summary) if command == "dependency-report": return report(dependency_report(distro), args.summary) + if command == "update-build-timings": + return report(update_build_timings(), args.summary) if command == "add-package": known = set(rs.distros()) distros = [n for n in args.names if n in known] diff --git a/tools/robostack.py b/tools/robostack.py index 27f4fa887..4d3d7a2e7 100644 --- a/tools/robostack.py +++ b/tools/robostack.py @@ -404,7 +404,14 @@ def task(distro: str, name: str, args: list[str]) -> int: if name == "build": return run_build(rattler_build(distro, ["--recipe-dir", "./recipes", *args]), w) if name == "build-ci": # used by .scripts/build_unix.sh / build_win.bat - return run_build(rattler_build(distro, args, skip_existing=False), w) + started = time.monotonic() + rc = run_build(rattler_build(distro, args, skip_existing=False), w) + if rc == 0 and "--recipe" in args[:-1] and os.environ.get("GITHUB_ACTIONS"): + import build_timings # for packing the build jobs by time + + recipe = Path(args[args.index("--recipe") + 1].rstrip("\\/")).name + build_timings.note(distro, recipe, time.monotonic() - started) + return rc if name == "build-one": pkg = args[0] if args else "ros2-ros-workspace" for patch in (d / "patch").glob(f"{pkg}.*patch"): @@ -429,10 +436,26 @@ def generate_gha(distro: str, args: list[str]) -> int: # GitHub refuses workflow files over 500 KiB, which a full rebuild's many batches can # reach (jazzy's win-64 at 25): then retry with larger, hence fewer, batches. batch_size = int(ns.batch_size) + # each job keeps the build times it records (build-ci); with times for the platform, + # vinca-gha packs each stage's recipes into jobs by time instead of by count + import build_timings + + extra = ["--keep-file", f"distros/{distro}/work/{build_timings.JOURNAL}"] + timed = build_timings.vinca_timings(ns.platform, d / "build_timings.json") + if timed: + extra += ["--timings", "build_timings.json"] + runners = 20 while True: rc, output = run_tee(["vinca-gha", "--platform", ns.platform, "--trigger-branch", branch, "-d", - "./recipes", "--batch_size", str(batch_size)], d) - if not rc or WORKFLOW_SIZE_MESSAGE not in output or batch_size >= MAX_BATCH_SIZE: + "./recipes", "--batch_size", str(batch_size), *extra], d) + if not rc or WORKFLOW_SIZE_MESSAGE not in output: + break + if timed and runners > 2: # planning for fewer runners makes fewer jobs + runners //= 2 + extra = [a for a in extra if not a.startswith("--runners")] + [f"--runners={runners}"] + print(f"The workflow is too large; retrying with --runners {runners}", flush=True) + continue + if batch_size >= MAX_BATCH_SIZE: break batch_size = min(batch_size * 3 // 2, MAX_BATCH_SIZE) print(f"The workflow is too large; retrying with --batch_size {batch_size}", flush=True) @@ -506,7 +529,7 @@ def changed_distros(base: str) -> list[str]: REPO_COMMANDS = ("check", "update-pinning", "new-distro", "parse-command", "add-package", "rosdistro-syncs", - "rebuild", "rebuild-dependents", "update-vinca") + "rebuild", "rebuild-dependents", "update-vinca", "update-build-timings") def main() -> int: diff --git a/tools/tests/test_build_timings.py b/tools/tests/test_build_timings.py new file mode 100644 index 000000000..77a243867 --- /dev/null +++ b/tools/tests/test_build_timings.py @@ -0,0 +1,87 @@ +import json +import subprocess +from pathlib import Path + +import build_timings + + +def test_note_records_build_times(tmp_path, monkeypatch): + monkeypatch.setattr(build_timings.rs, "work_dir", lambda distro: tmp_path) + build_timings.note("jazzy", "ros2-a", 12.34) + build_timings.note("jazzy", "ros-jazzy-a", 1.0) + entries = [json.loads(line) for line in (tmp_path / build_timings.JOURNAL).read_text().splitlines()] + assert [(e["recipe"], e["seconds"]) for e in entries] == [("ros2-a", 12.3), ("ros-jazzy-a", 1.0)] + assert all(e["platform"] for e in entries) + + +def test_collect_merges_the_runs_journals(tmp_path, monkeypatch): + monkeypatch.setattr(build_timings, "TIMINGS", tmp_path / "timings") + runs = [{"databaseId": 2, "headBranch": "buildbranch_jazzy_linux_64", "createdAt": "2026-10-22T02:00:00Z"}, + {"databaseId": 1, "headBranch": "buildbranch_jazzy_linux_64", "createdAt": "2026-10-22T01:00:00Z"}, + {"databaseId": 3, "headBranch": "buildbranch_lyrical_linux_64", "createdAt": "2026-10-22T03:00:00Z"}] + journals = {1: [("ros2-a", 10.0), ("ros2-b", 5.0)], 2: [("ros2-a", 12.0)], 3: [("ros2-a", 20.0)]} + + def run(cmd, **kwargs): + if cmd[1:3] == ["run", "list"]: + return subprocess.CompletedProcess(cmd, 0, json.dumps(runs), "") + run_id = int(cmd[3]) + job = Path(cmd[cmd.index("--dir") + 1]) / "stage_0_job_0-1" + job.mkdir(parents=True) + (job / build_timings.JOURNAL).write_text("".join( + json.dumps({"platform": "linux-64", "recipe": r, "seconds": s}) + "\n" for r, s in journals[run_id])) + return subprocess.CompletedProcess(cmd, 0, b"", b"") + + monkeypatch.setattr(build_timings.subprocess, "run", run) + assert build_timings.collect() == 4 + # the newer jazzy run wins; each distribution keeps its own time + assert json.loads((tmp_path / "timings" / "linux-64.json").read_text()) == { + "ros2-a": {"jazzy": 12.0, "lyrical": 20.0}, "ros2-b": {"jazzy": 5.0}} + # vinca-gha gets the average over the distributions + out = tmp_path / "vinca.json" + assert build_timings.vinca_timings("linux-64", out) + assert json.loads(out.read_text()) == {"recipes": {"ros2-a": 16.0, "ros2-b": 5.0}} + assert not build_timings.vinca_timings("win-64", out) + + +def test_update_build_timings_command(tmp_path, monkeypatch): + import maintenance + + assert maintenance.parse_command("@robostack-bot update-build-timings", "MEMBER") == [ + {"command": "update-build-timings", "distro": "", "args": "", "preview": False}] + assert maintenance.parse_command("@robostack-bot update-build-timings", "NONE") == [] + monkeypatch.setattr(build_timings, "TIMINGS", tmp_path) + + def collect(): + (tmp_path / "linux-64.json").write_text('{"ros2-a": {"jazzy": 1.0}}\n') + return 1 + + monkeypatch.setattr(build_timings, "collect", collect) + result = maintenance.update_build_timings() + assert result.changed and "linux-64.json" in result.summary + assert not maintenance.update_build_timings().changed # nothing new: no PR + + +def test_gha_keeps_the_journal_and_uses_the_times(tmp_path, monkeypatch): + import robostack as rs + + work, root = tmp_path / "work", tmp_path / "root" + work.mkdir() + (root / ".github" / "workflows").mkdir(parents=True) + calls = [] + + def fake_run_tee(cmd, cwd): + calls.append(cmd) + (work / "linux.yml").write_text("name: build\njobs:\n stage_0_job_0:\n steps: []\n") + return 0, b"" + + monkeypatch.setattr(rs, "run_tee", fake_run_tee) + monkeypatch.setattr(rs, "work_dir", lambda distro: work) + monkeypatch.setattr(rs, "ROOT", root) + monkeypatch.setattr(build_timings, "TIMINGS", tmp_path / "timings") + assert rs.generate_gha("jazzy", ["--platform", "linux-64"]) == 0 + assert "--keep-file" in calls[-1] and "--timings" not in calls[-1] # no times yet + (tmp_path / "timings").mkdir() + (tmp_path / "timings" / "linux-64.json").write_text('{"ros2-a": {"jazzy": 3.0}}\n') + assert rs.generate_gha("jazzy", ["--platform", "linux-64"]) == 0 + assert calls[-1][calls[-1].index("--timings") + 1] == "build_timings.json" + assert json.loads((work / "build_timings.json").read_text()) == {"recipes": {"ros2-a": 3.0}}