From 290a9034b3df4adab93b5936412afa87d8b7d262 Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Thu, 1 Oct 2026 19:59:25 +0200 Subject: [PATCH] dashboard: read the registry from docs/packages/*.yaml The /simple/ index is generated from that same YAML and published by the same job, so probing it over HTTP answered a deploy late, and requests-cache could pin a 404 for a newly added package for 30 days. --- .github/workflows/website.yml | 9 +- .gitignore | 6 + ci_scripts/dashboard/README.md | 53 +++++++-- ci_scripts/dashboard/build.py | 195 ++++++++++++------------------- ci_scripts/dashboard/generate.py | 23 ---- ci_scripts/dashboard/utils.py | 19 +-- 6 files changed, 131 insertions(+), 174 deletions(-) create mode 100644 .gitignore delete mode 100644 ci_scripts/dashboard/generate.py diff --git a/.github/workflows/website.yml b/.github/workflows/website.yml index 644cc8a1423..283b49acd92 100644 --- a/.github/workflows/website.yml +++ b/.github/workflows/website.yml @@ -115,19 +115,14 @@ jobs: - name: Restore the dashboard's pypi.org response cache uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: - path: ${{ runner.temp }}/dashboard/requests-cache.sqlite + path: requests-cache.sqlite key: requests-cache-${{ github.ref }} restore-keys: requests-cache-main - # build.py stages nothing unless the result is complete, and keep_files - # leaves the previous dashboard serving, so an upstream data outage must - # not block a docs deploy. - name: Generate the dashboard continue-on-error: true run: | - python ci_scripts/dashboard/build.py \ - --build-dir "${{ runner.temp }}/dashboard" \ - --output-dir docs/dashboard + python ci_scripts/dashboard/build.py --output-dir docs/dashboard - name: Build with Jekyll uses: actions/jekyll-build-pages@44a6e6beabd48582f863aeeb6cb2151cc1716697 # v1.0.13 diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000000..919760e463a --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +.bundle/ +.venv/ +requests-cache.sqlite +top-pypi-packages.json +results.json +wheel.svg diff --git a/ci_scripts/dashboard/README.md b/ci_scripts/dashboard/README.md index 9aa87d5abb8..88ff14cf2ff 100644 --- a/ci_scripts/dashboard/README.md +++ b/ci_scripts/dashboard/README.md @@ -24,41 +24,72 @@ rest of this repo. | File | Origin | |---|---| | `LICENSE` | verbatim from upstream | -| `generate.py`, `utils.py`, `svg_wheel.py` | upstream, SPDX header prepended | +| `svg_wheel.py` | upstream, SPDX header prepended | +| `utils.py` | upstream, SPDX header prepended, plus the registry change below | | `wheel.css`, `favicon.ico` | verbatim from upstream | | `index.html` | upstream, with the text edits listed below | | `build.py` | **new**, RISE-authored — not upstream | +Upstream's `generate.py` is **not** vendored: it is a four-line `main()`, and +`build.py` calls those same four functions itself. + The generator's own dependencies (`requests`, `requests-cache`) live in `ci_scripts/requirements.txt` alongside the rest of this repo's CI Python deps; upstream's `pre-commit` pin is dropped, as there are no local hooks here. -`build.py` holds all python-wheels-specific logic so that the upstream files stay -a clean copy. It fetches and sanity-checks the package list, runs `generate.py` in -a scratch directory, validates the result, and only then stages five files into -the Jekyll source tree. +`build.py` holds all python-wheels-specific logic. It fetches and sanity-checks the +package list, collects the registry contents from `docs/packages/*.yaml`, imports +`utils` and `svg_wheel` to produce `results.json` and `wheel.svg` in a scratch +directory, validates the result, and only then stages five files into the Jekyll +source tree. + +Because it imports the vendored modules rather than spawning them, two cwd-bound +details matter. `utils` builds its `CachedSession` from a relative path **at import +time**, and `get_top_packages()`/`generate_svg_wheel()` read and write relative to +the cwd. So `build.py` resolves its path arguments to absolute, `chdir`s into the +build dir, and rebinds `utils.SESSION` there — otherwise a multi-GB +`requests-cache.sqlite` lands wherever the script was invoked from. + +## The registry lookup + +Upstream's `utils.in_rise_registry()` asks +`https://pypi.riseproject.dev/simple//` whether the registry has a package, +one HTTP request per candidate. In this repo the answer is already on disk: +`docs/packages/*.yaml` is what the registry index is generated from, and +`generate_packages_doc.py` publishes that index in the same job that builds this +dashboard — so the HTTP answer is a deploy behind, and `requests-cache` can pin a +404 for a newly added package for up to 30 days. + +So `in_rise_registry()` is gone, along with `RISE_REGISTRY_URL`, and +`annotate_wheels(packages, registry)` takes the set of normalised names that +`build.py` collected and tests membership directly. ## Resyncing from upstream ```bash cd ci_scripts/dashboard UP=/path/to/python-wheels-dashboard -cp "$UP"/{LICENSE,generate.py,utils.py,svg_wheel.py,index.html,wheel.css,favicon.ico} . +cp "$UP"/{LICENSE,utils.py,svg_wheel.py,index.html,wheel.css,favicon.ico} . ``` Then re-apply the local deltas: -1. Prepend the BSD-2-Clause SPDX header to `generate.py`, `utils.py` and - `svg_wheel.py` (copy it from a sibling file). -2. Re-apply the `index.html` edits: +1. Prepend the BSD-2-Clause SPDX header to `utils.py` and `svg_wheel.py` (copy it + from a sibling file). +2. Re-apply the registry change described above: drop `RISE_REGISTRY_URL` and + `in_rise_registry()`, give `annotate_wheels()` a `registry` parameter, and use + `normalize(package["name"]) in registry` where the HTTP probe was. +3. Re-apply the `index.html` edits: - drop the stale "top 360" figure from the "What is this list?" paragraph and the "Thanks" paragraph — the generator applies no slice, so the real count moves every run; - change the footer cadence from "Updated daily." to the real cadence; - keep the absolute link back to near the `

`. -3. If upstream changed its `requests`/`requests-cache` pins, update them in +4. If upstream changed its `requests`/`requests-cache` pins, update them in `ci_scripts/requirements.txt`. -4. Update the vendored commit SHA above. +5. If upstream's `generate.py` grew a step beyond its four calls, mirror it in + `build.py`'s `main()`. +6. Update the vendored commit SHA above. Leave `build.py` alone — it is not upstream. diff --git a/ci_scripts/dashboard/build.py b/ci_scripts/dashboard/build.py index 958c4aafe4b..6bbf1d582d3 100644 --- a/ci_scripts/dashboard/build.py +++ b/ci_scripts/dashboard/build.py @@ -3,9 +3,9 @@ """Build the RISC-V Wheels dashboard into the Jekyll source tree. -Wraps the vendored generator (see README.md) with the sanity checks the website -build needs: the upstream package list has served HTTP 200 with zero rows, and -publishing that would replace a working dashboard with a blank one. +Runs the vendored generator's steps (see README.md) with the sanity checks the +website build needs: the upstream package list has served HTTP 200 with zero rows, +and publishing that would replace a working dashboard with a blank one. Nothing is written to --output-dir unless a complete result was produced, so a failed run leaves the previously published dashboard serving untouched. @@ -15,21 +15,25 @@ import json import os from pathlib import Path +import re import shutil -import subprocess import sys import urllib.request +import requests_cache +import yaml + VENDOR_DIR = Path(__file__).resolve().parent +REPO_ROOT = VENDOR_DIR.parents[1] + +sys.path.insert(0, str(VENDOR_DIR)) -UPSTREAM_LIST_URL = "https://hugovk.dev/top-pypi-packages/top-pypi-packages.min.json" -# The upstream endpoint is a live query and has returned HTTP 200 carrying an -# empty `rows` plus a ClickHouse row-limit `exception`. This pinned copy of the -# same file is committed upstream and regenerated monthly, so falling back to it -# costs at most one month of ranking drift. -FALLBACK_LIST_URL = ( - "https://raw.githubusercontent.com/hugovk/top-pypi-packages/" - "6becf8c3b/top-pypi-packages.min.json" +from svg_wheel import generate_svg_wheel # noqa: E402 +import utils # noqa: E402 +from utils import ( # noqa: E402 + annotate_wheels, + get_top_packages, + save_to_file, ) # Loose floors: enough to catch an empty or truncated upstream response without @@ -41,7 +45,7 @@ # Copied from the vendored directory as-is. STATIC_FILES = ("index.html", "wheel.css", "favicon.ico") -# Produced by generate.py in the build directory. +# Produced by generate.py. GENERATED_FILES = ("results.json", "wheel.svg") @@ -83,93 +87,55 @@ def validate_package_list(payload): return rows, None -def get_package_list(candidates): +def get_package_list(): """Fetch the package list from the first usable candidate. Exits on failure.""" failures = [] - for label, url in candidates: - print(f"Fetching the {label} package list: {url}") - payload = fetch(url) - if payload is not None: - rows, reason = validate_package_list(payload) - if rows is not None: - print(f" accepted: {len(rows)} rows") - if label == "pinned": - print( - "::warning::The upstream package list was unusable; " - "using the pinned copy, so the download rankings may be " - "up to a month old." - ) - return payload - print(f" rejected: {reason}") - failures.append(f"{label}: {reason}") - else: - failures.append(f"{label}: could not be fetched") - - sys.exit( - "Could not obtain a usable package list, refusing to publish:\n " - + "\n ".join(failures) - ) + url = "https://hugovk.dev/top-pypi-packages/top-pypi-packages.min.json" + print(f"Fetching the package list: {url}") + payload = fetch(url) + if payload is not None: + rows, reason = validate_package_list(payload) + if rows is not None: + print(f" accepted: {len(rows)} rows") + return payload + sys.exit(f"Could not obtain a usable package list, refusing to publish: {reason}") + else: + sys.exit(f"Could not obtain a usable package list, refusing to publish: could not be fetched") -def validate_output(build_dir): - """Exit unless the generator produced a complete, sane result.""" - results = build_dir / "results.json" - try: - with open(results, encoding="utf-8") as f: - data = json.load(f) - except (OSError, json.JSONDecodeError) as e: - sys.exit(f"{results} is unusable: {e}") - - packages = data.get("data") - if not isinstance(packages, list): - sys.exit(f"{results} has no `data` list") - if len(packages) < MIN_OUTPUT_PACKAGES: - sys.exit( - f"{results} has only {len(packages)} packages, expected at least " - f"{MIN_OUTPUT_PACKAGES}" - ) - if not isinstance(data.get("last_update"), str) or not data["last_update"]: - sys.exit(f"{results} has no `last_update`") - - wheel = build_dir / "wheel.svg" - if not wheel.is_file(): - sys.exit(f"{wheel} was not generated") - size = wheel.stat().st_size - if size < MIN_WHEEL_SVG_BYTES: - # One per package, so a near-empty wheel is a few hundred bytes. - sys.exit(f"{wheel} is only {size} bytes, expected at least " - f"{MIN_WHEEL_SVG_BYTES}") - - return len(packages) - - -def check_no_front_matter(): - """Exit if index.html gained YAML front matter. - - The page's AngularJS bindings (``{{ package.name }}``) are also valid Liquid. - Jekyll only leaves them alone because a file without front matter is a static - file, copied byte-for-byte. Add front matter and the package list renders - empty in production with no build error. +def collect_registry_packages(packages_dir): + """Return the normalised names of every package the RISE registry serves. + + These are read from this repo's own docs/packages/*.yaml rather than probed + over HTTP against pypi.riseproject.dev: the YAML is what the registry is + generated from, so it is both authoritative and free. """ - index = VENDOR_DIR / "index.html" - with open(index, encoding="utf-8") as f: - if f.read(3) == "---": - sys.exit( - f"{index} starts with YAML front matter. Jekyll would then run " - "Liquid over it and eat the AngularJS bindings, silently " - "emptying the package list. Remove the front matter." - ) + names = set() + unreleased = 0 + for path in sorted(packages_dir.glob("*.yaml")): + try: + with open(path, encoding="utf-8") as f: + data = yaml.safe_load(f) + except (OSError, yaml.YAMLError) as e: + sys.exit(f"{path} is unusable: {e}") + + name = (data or {}).get("package-name") + if not name: + sys.exit(f"{path} has no `package-name`") + if not (data.get("versions") or []): + # Nothing published yet, so the registry serves no wheels for it. + unreleased += 1 + continue + # Same normalisation as utils.normalize(). + names.add(re.sub(r"[-_.]+", "-", name).lower()) + + print(f"Read {len(names)} registry packages from {packages_dir}", end="") + print(f" ({unreleased} with no release yet)" if unreleased else "") + return frozenset(names) def main(): parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--build-dir", - required=True, - type=Path, - help="scratch directory for the generator; must be outside docs/, as it " - "collects a multi-GB requests-cache.sqlite", - ) parser.add_argument( "--output-dir", required=True, @@ -177,44 +143,37 @@ def main(): help="where to stage the dashboard, e.g. docs/dashboard", ) parser.add_argument( - "--list-url", - help="override the upstream package list URL (for testing the input gate)", + "--packages-dir", + default=REPO_ROOT / "docs" / "packages", + type=Path, + help="directory of per-package YAML describing the RISE registry (default: docs/packages)", ) args = parser.parse_args() - if args.list_url: - candidates = [("override", args.list_url)] - else: - candidates = [("upstream", UPSTREAM_LIST_URL), ("pinned", FALLBACK_LIST_URL)] + output_dir = args.output_dir.resolve() + packages_dir = args.packages_dir.resolve() - check_no_front_matter() + payload = get_package_list() + registry = collect_registry_packages(packages_dir) - build_dir = args.build_dir.resolve() - build_dir.mkdir(parents=True, exist_ok=True) - - payload = get_package_list(candidates) - (build_dir / "top-pypi-packages.json").write_bytes(payload) - - # A subprocess, not an import: utils.py builds its CachedSession at module - # import time, so the sqlite path binds to the cwd of whoever imports it - # first. Running with cwd=build_dir keeps every cwd-relative read and write - # out of the Jekyll source tree. - print("Running the vendored generator...") - subprocess.run( - [sys.executable, str(VENDOR_DIR / "generate.py")], - cwd=build_dir, - check=True, + utils.SESSION = requests_cache.CachedSession( + "requests-cache", expire_after=utils.SESSION.settings.expire_after ) - count = validate_output(build_dir) + Path("top-pypi-packages.json").write_bytes(payload) + + packages = get_top_packages() + packages = annotate_wheels(packages, registry) + save_to_file(packages, "results.json") + generate_svg_wheel(packages) - os.makedirs(args.output_dir, exist_ok=True) + os.makedirs(output_dir, exist_ok=True) for name in STATIC_FILES: - shutil.copy2(VENDOR_DIR / name, args.output_dir / name) + shutil.copy2(VENDOR_DIR / name, output_dir / name) for name in GENERATED_FILES: - shutil.copy2(build_dir / name, args.output_dir / name) + shutil.copy2(name, output_dir / name) - print(f"Staged the dashboard for {count} packages in {args.output_dir}") + print(f"Staged the dashboard for {len(packages)} packages in {output_dir}") if __name__ == "__main__": diff --git a/ci_scripts/dashboard/generate.py b/ci_scripts/dashboard/generate.py deleted file mode 100644 index 714b27e3a84..00000000000 --- a/ci_scripts/dashboard/generate.py +++ /dev/null @@ -1,23 +0,0 @@ -# SPDX-FileCopyrightText: 2013 Charlie Denton -# SPDX-FileCopyrightText: 2026 Stan Ulbrych -# SPDX-License-Identifier: BSD-2-Clause -# -# Vendored from riseproject-dev/python-wheels-dashboard. See README.md. - -from svg_wheel import generate_svg_wheel -from utils import ( - annotate_wheels, - get_top_packages, - save_to_file, -) - - -def main() -> None: - packages = get_top_packages() - packages = annotate_wheels(packages) - save_to_file(packages, "results.json") - generate_svg_wheel(packages) - - -if __name__ == "__main__": - main() diff --git a/ci_scripts/dashboard/utils.py b/ci_scripts/dashboard/utils.py index 712595b825a..064397198ae 100644 --- a/ci_scripts/dashboard/utils.py +++ b/ci_scripts/dashboard/utils.py @@ -13,7 +13,6 @@ BASE_URL = "https://pypi.org/pypi" -RISE_REGISTRY_URL = "https://pypi.riseproject.dev/simple" DEPRECATED_PACKAGES = { "BeautifulSoup", @@ -225,19 +224,9 @@ def normalize(package_name: str) -> str: return re.sub(r"[-_.]+", "-", package_name).lower() -def in_rise_registry(package_name: str) -> bool: - """Whether the RISE riscv64 registry has wheels for this package.""" - url = f"{RISE_REGISTRY_URL}/{normalize(package_name)}/" - try: - response = SESSION.get(url, allow_redirects=False) - except Exception as e: - print(f" ! Could not check the RISE registry for {package_name}: {e}") - return False - - return response.status_code == 200 - - -def annotate_wheels(packages) -> list[dict]: +def annotate_wheels(packages, registry) -> list[dict]: + """Classify each package. ``registry`` holds the normalised names of the + packages the RISE riscv64 registry serves.""" print("Getting wheel data...") keep = [] @@ -272,7 +261,7 @@ def _handle_package(package) -> tuple[dict, bool]: package["css_class"] = "success" package["icon"] = "\u2713" # Check mark elif has_other_binary_wheel: - in_registry = in_rise_registry(package["name"]) + in_registry = normalize(package["name"]) in registry if in_registry: package["css_class"] = "rise" package["icon"] = "📦"