Record each recipe's build time in the build jobs; pack the build jobs by time

.github/ISSUE_TEMPLATE/3_robostack_bot.yaml

@@ -18,14 +18,15 @@ body:
         | `rebuild` | maintainers | New builds of the given ROS packages (per-package build numbers above every released build); add `--with-dependents` to the packages to also rebuild everything that depends on them. The PR lists the packages. |
         | `update-vinca` | maintainers | Move the vinca pin in `pixi.toml` to the newest commit of its branch (or the ref given under Packages), lock and check. The PR lists the vinca commits. |
         | `dependency-report` | maintainers | Check the dependencies against the pins and update the distribution's dependency-report issue (closed when nothing conflicts). Runs weekly. |
+        | `update-build-timings` | maintainers | Record the build times of the last week's build-branch runs in `shared/build_timings/`, which the build workflows use to pack each stage's recipes into jobs by time. Runs weekly. |
 
         On a pull request, `@robostack-bot rebuild-dependents` (maintainers) commits the rebuilds its ABI check suggests to the pull request's branch.
   - type: dropdown
     id: command
     attributes:
       label: Command
       options: [add-package, update-rosdistro-snapshot, find-stale-packages, update-conda-forge-pinning, rebuild, update-vinca,
-                dependency-report]
+                dependency-report, update-build-timings]
     validations:
       required: true
   - type: input

.github/workflows/bot.yaml

@@ -8,14 +8,15 @@
 #   new pins can't be installed together).
 # - weekly (Mondays): a dependency check of every distribution, kept in one issue per
 #   distribution ("<distro>: dependency report", label dependency-report), closed when
-#   nothing conflicts.
+#   nothing conflicts; and a PR with the build times the week's build-branch runs
+#   recorded (shared/build_timings/, for packing the build jobs by time).
 # - on demand (owners, members and collaborators):
 #     Actions > robostack-bot > Run workflow, an issue from the "robostack-bot command"
 #     template, or a comment: @robostack-bot <command> [<distro>]
 #   commands: update-rosdistro-snapshot [<distro>...|all] | find-stale-packages [<distro>...|all] |
 #             update-conda-forge-pinning [<distro>...|all] | add-package <pkg>... [<distro>...] |
 #             rebuild <pkg>... [<distro>...|all] [--with-dependents] | update-vinca [<ref>] |
-#             dependency-report [<distro>...|all] |
+#             dependency-report [<distro>...|all] | update-build-timings |
 #             rebuild-dependents (a comment on a pull request: commits the rebuilds its ABI
 #             check suggests to its branch; not possible for pull requests from forks)
 # - package requests (the "Package request" issue form or `@robostack-bot add-package`,
@@ -35,7 +36,7 @@ on:
       command:
         type: choice
         options: [add-package, update-rosdistro-snapshot, find-stale-packages, update-conda-forge-pinning, rebuild,
-                  update-vinca, dependency-report]
+                  update-vinca, dependency-report, update-build-timings]
       distro:
         description: Distribution (for the per-distribution commands; empty = all)
         required: false
@@ -90,7 +91,9 @@ jobs:
           case "$EVENT" in
             schedule)
               if [ "$SCHEDULE" = "43 4 * * 1" ]; then
-                jobs=$(ROBOSTACK_BOT_BODY="@robostack-bot dependency-report all" ROBOSTACK_BOT_ASSOCIATION=OWNER pixi run -q rs parse-command)
+                # weekly: the dependency reports, and the build times of the week's builds
+                jobs=$(ROBOSTACK_BOT_BODY="@robostack-bot dependency-report all" ROBOSTACK_BOT_ASSOCIATION=OWNER pixi run -q rs parse-command |
+                       python3 -c 'import json,sys; print(json.dumps(json.load(sys.stdin) + [{"command":"update-build-timings","distro":"","args":"","preview":False}]))')
                 echo "jobs=$jobs" | tee -a "$GITHUB_OUTPUT"
                 exit 0
               fi
@@ -113,6 +116,7 @@ jobs:
       matrix:
         include: ${{ fromJSON(needs.plan.outputs.jobs) }}
     permissions:
+      actions: read  # update-build-timings: the build-branch runs' artifacts
       contents: write
       pull-requests: write
       issues: write
@@ -160,6 +164,8 @@ jobs:
             # pushes to the pull request's branch itself (with this job's checkout token)
             rebuild-dependents) pixi run rs rebuild-dependents "$ARGS" --summary "$RUNNER_TEMP/summary.md" ;;
             update-vinca) pixi run rs update-vinca $ARGS --summary "$RUNNER_TEMP/summary.md" ;;
+            # the build-branch runs' build-times artifacts (actions: read)
+            update-build-timings) pixi run rs update-build-timings --summary "$RUNNER_TEMP/summary.md" ;;
             # creates / updates the distribution's dependency-report issue
             dependency-report) pixi run rs "$DISTRO" dependency-report --summary "$RUNNER_TEMP/summary.md" ;;
           esac

README.md

@@ -119,6 +119,7 @@ label doesn't start one by itself. Nothing has to be reset before merging.
   - Each selected distribution is built on all five platforms, with its own build cache. The `check` job runs `pixi run rs check`.
 - **After a merge** (`main.yaml`): for every changed distribution and platform, the job regenerates the recipes and the staged build workflow, and pushes them to `buildbranch_<distro>_<platform>`.
   - Each build branch carries its workflow as `.github/workflows/build.yaml`, which runs `.scripts/build_*.sh` in `distros/<distro>/work`.
+  - The workflow builds in stages (by dependency depth), each stage in parallel jobs. Each job records how long its recipes took to build and keeps that as an artifact; `robostack-bot update-build-timings` merges them weekly into `shared/build_timings/<platform>.json` (each recipe's time per distribution). With those times, each stage's recipes are packed into jobs by time (vinca-gha `--timings`, each recipe's average over the distributions), so that no single job holds up its stage; until a platform has times, its jobs are cut by count as before.
   - Those workflows build the packages and upload them to the distribution's channel on the servers `upload_to` in `distro.yaml` lists (prefix.dev and/or anaconda.org, the same channel name on each; packages already there are skipped). `read_from` says where the builds look up what's already published.
 - **Repository variables:**
   - `ROBOSTACK_UPLOAD_CHANNEL`: upload only to this prefix.dev channel instead (for example a test channel), or `none` to build without uploading. Use the canonical reference: `<namespace>/<channel>` for a channel that isn't the namespace's primary one (e.g. `tobias-fischer/robostack-distro-test`).
@@ -143,6 +144,7 @@ label doesn't start one by itself. Nothing has to be reset before merging.
 | `rebuild-dependents` | maintainers | Only as a comment on a pull request: commits the build-number entries the ABI check's comment suggests (its `yaml` blocks per distribution) to the pull request's branch, keeping existing entries' other keys. Pull requests from forks can't be pushed to: the bot replies with the entries to add instead. |
 | `update-vinca [<ref>]` | maintainers | Moves the vinca pin in `pixi.toml` to the newest commit of the pinned repository's branch (`robostack-integration` for Tobias-Fischer/vinca, else its default branch) or of `<ref>`, runs `pixi lock`, re-renders the pinning and runs `rs check`. The PR links the comparison and lists the vinca commits. |
 | `dependency-report [<distro>... \| all]` | maintainers | Runs the dependency check (`check-deps`, linux-64) and keeps one issue per distribution, *`<distro>`: dependency report* (label `dependency-report`), up to date with the conflicts, notes and conda-forge migration status; the issue is closed when nothing conflicts. Runs weekly (Mondays). |
+| `update-build-timings` | maintainers | Merges the build times the last week's build-branch runs recorded (`rs <distro> build-ci`, kept as each job's artifact) into `shared/build_timings/<platform>.json` and opens a PR. The build workflows pack each stage's recipes into jobs by these times. Runs weekly (Mondays). |
 
 There are three ways to trigger a command:
 - a *Package request* or *robostack-bot command* issue;

pixi.toml

@@ -36,7 +36,7 @@ git = "*"
 
 [pypi-dependencies]
 # One vinca for all distributions.
-vinca = { git = "https://github.com/RoboStack/vinca.git", rev = "3d552f4001634fd26376b2bf5fe37fe81ef3dc22" }
+vinca = { git = "https://github.com/RoboStack/vinca.git", rev = "c52ce0cc30bc9f121dae0980b1f947debf1d8c1c" }
 # For local development, temporarily use (and regenerate pixi.lock):
 # vinca = { path = "../vinca", editable = true }
 

tools/build_timings.py

@@ -0,0 +1,78 @@
+"""Measured build times, for packing the build jobs by time (vinca-gha --timings).
+
+Each build-branch job records how long each of its recipes took to build (`rs <distro>
+build-ci`, in distros/<distro>/work/build_timings.jsonl) and keeps that file as an artifact
+(vinca-gha --keep-file). `robostack-bot update-build-timings` merges the last week's into
+shared/build_timings/<platform>.json, {recipe: {distro: seconds}}, and `rs <distro> gha`
+hands vinca-gha each recipe's average over the distributions: a recipe new to one
+distribution still has a time (the names are the same, ros2-<package>).
+
+    pixi run python tools/build_timings.py   # what the bot command does
+"""
+
+from __future__ import annotations
+
+import json
+import statistics
+import subprocess
+import sys
+import tempfile
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+import robostack as rs
+
+TIMINGS = rs.ROOT / "shared" / "build_timings"
+JOURNAL = "build_timings.jsonl"
+
+
+def note(distro: str, recipe: str, seconds: float) -> None:
+    """Record a recipe's build time in the work directory's journal."""
+    from vinca.platforms import get_conda_subdir
+
+    entry = {"platform": get_conda_subdir(), "recipe": recipe, "seconds": round(seconds, 1)}
+    with (rs.work_dir(distro) / JOURNAL).open("a", encoding="utf-8") as fh:
+        fh.write(json.dumps(entry) + "\n")
+
+
+def collect(days: int = 8) -> int:
+    """Merge the build times the build-branch runs of the last `days` days recorded into
+    shared/build_timings/ (newer runs win); returns how many times were recorded."""
+    since = (datetime.now(timezone.utc) - timedelta(days=days)).strftime("%Y-%m-%d")
+    runs = json.loads(subprocess.run(
+        ["gh", "run", "list", "--repo", "RoboStack/robostack", "--workflow", "build.yaml", "--status", "completed",
+         "--created", f">={since}", "--limit", "1000", "--json", "databaseId,headBranch,createdAt"],
+        capture_output=True, text=True, check=True).stdout)
+    files = {p.stem: json.loads(p.read_text()) for p in TIMINGS.glob("*.json")}
+    count = 0
+    with tempfile.TemporaryDirectory() as tmp:
+        for run in sorted(runs, key=lambda r: r["createdAt"]):
+            distro = run["headBranch"].removeprefix("buildbranch_").split("_")[0]
+            out = Path(tmp) / str(run["databaseId"])
+            subprocess.run(["gh", "run", "download", str(run["databaseId"]), "--repo", "RoboStack/robostack",
+                            "--dir", str(out)], capture_output=True, check=False)
+            for journal in sorted(out.rglob(JOURNAL)):
+                for line in journal.read_text().splitlines():
+                    entry = json.loads(line)
+                    files.setdefault(entry["platform"], {}).setdefault(entry["recipe"], {})[distro] = entry["seconds"]
+                    count += 1
+    TIMINGS.mkdir(parents=True, exist_ok=True)
+    for platform, recipes in files.items():
+        (TIMINGS / f"{platform}.json").write_text(json.dumps(recipes, indent=0, sort_keys=True) + "\n")
+    return count
+
+
+def vinca_timings(platform: str, path: Path) -> bool:
+    """Write vinca-gha's --timings file for the platform: each recipe's average over the
+    distributions. False when there are no times yet."""
+    source = TIMINGS / f"{platform}.json"
+    if not source.is_file():
+        return False
+    recipes = {r: round(statistics.fmean(by.values()), 1) for r, by in json.loads(source.read_text()).items()}
+    path.write_text(json.dumps({"recipes": recipes}) + "\n")
+    return True
+
+
+if __name__ == "__main__":
+    print(f"{collect()} build times recorded")

tools/maintenance.py

@@ -43,6 +43,7 @@
     "rebuild-dependents": "on a pull request: add the rebuilds its ABI check suggests (pushes to its branch)",
     "update-vinca": "move the vinca pin to the newest commit of its branch (opens a PR)",
     "dependency-report": "check the dependencies against the pins, in the distribution's dependency-report issue",
+    "update-build-timings": "record the build times of the last week's build-branch runs, for batching by time (opens a PR)",
 }
 PER_DISTRO_COMMANDS = ("update-rosdistro-snapshot", "find-stale-packages", "update-conda-forge-pinning",
                        "dependency-report")
@@ -1127,6 +1128,20 @@ def dependency_report(distro: str) -> Result:
     return Result(title, f"Report: {url}" if url else "No conflicts, and no report issue to close.")
 
 
+def update_build_timings() -> Result:
+    """Merge the build times of the last week's build-branch runs into shared/build_timings/."""
+    import build_timings
+
+    before = {p: p.read_text() for p in build_timings.TIMINGS.glob("*.json")}
+    count = build_timings.collect()
+    changed = sorted(p.name for p in build_timings.TIMINGS.glob("*.json") if before.get(p) != p.read_text())
+    if not changed:
+        return Result("Build times are up to date", "No new build times in the last week's build-branch runs.")
+    return Result("Update the build times", f"{count} build times from the last week's build-branch runs, in "
+                  + ", ".join(f"`shared/build_timings/{name}`" for name in changed)
+                  + ". `rs <distro> gha` packs each stage's recipes into jobs by them.", changed=True)
+
+
 def _default_ci_yaml() -> str:
     return (rs.TOOLS / "ci_default.yaml").read_text()
 
@@ -1340,6 +1355,8 @@ def main(command: str, argv: list[str], distro: str | None = None) -> int:
         return report(update_vinca(args.ref), args.summary)
     if command == "dependency-report":
         return report(dependency_report(distro), args.summary)
+    if command == "update-build-timings":
+        return report(update_build_timings(), args.summary)
     if command == "add-package":
         known = set(rs.distros())
         distros = [n for n in args.names if n in known]