diff --git a/.github/workflows/website.yml b/.github/workflows/website.yml index d9e6e79018c..18c82a98606 100644 --- a/.github/workflows/website.yml +++ b/.github/workflows/website.yml @@ -8,10 +8,12 @@ on: branches: [main] paths: - "docs/**" + - "ci_scripts/dashboard/**" pull_request: types: [opened, synchronize, reopened, closed] paths: - "docs/**" + - "ci_scripts/dashboard/**" workflow_dispatch: concurrency: @@ -40,7 +42,7 @@ jobs: with: python-version: '3' - - name: Install package doc generator dependencies + - name: Install site generator dependencies if: github.event.action != 'closed' run: pip install -r ci_scripts/requirements.txt @@ -103,12 +105,29 @@ jobs: with: python-version: '3' - - name: Install package doc generator dependencies + - name: Install site generator dependencies run: pip install -r ci_scripts/requirements.txt - name: Generate package pages from YAML run: python ci_scripts/generate_packages_doc.py + - name: Restore the dashboard's pypi.org response cache + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ runner.temp }}/dashboard/requests-cache.sqlite + key: requests-cache-${{ github.ref }} + restore-keys: requests-cache-main + + # build.py stages nothing unless the result is complete, and keep_files + # leaves the previous dashboard serving, so an upstream data outage must + # not block a docs deploy. + - name: Generate the dashboard + continue-on-error: true + run: | + python ci_scripts/dashboard/build.py \ + --build-dir "${{ runner.temp }}/dashboard" \ + --output-dir docs/dashboard + - name: Build with Jekyll uses: actions/jekyll-build-pages@v1 with: diff --git a/ci_scripts/dashboard/.gitignore b/ci_scripts/dashboard/.gitignore new file mode 100644 index 00000000000..c18dd8d83ce --- /dev/null +++ b/ci_scripts/dashboard/.gitignore @@ -0,0 +1 @@ +__pycache__/ diff --git a/ci_scripts/dashboard/LICENSE b/ci_scripts/dashboard/LICENSE new file mode 100644 index 00000000000..e20b1cbfda2 --- /dev/null +++ b/ci_scripts/dashboard/LICENSE @@ -0,0 +1,24 @@ +Copyright (c) 2013, Charlie Denton + 2026, Stan Ulbrych +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/ci_scripts/dashboard/README.md b/ci_scripts/dashboard/README.md new file mode 100644 index 00000000000..9aa87d5abb8 --- /dev/null +++ b/ci_scripts/dashboard/README.md @@ -0,0 +1,80 @@ +# Vendored RISC-V Wheels dashboard generator + +This directory is a **vendored copy** of the dashboard generator from +, used to publish the +dashboard at as part of this repo's +website build (`.github/workflows/website.yml`, `deploy` job). + +The dashboard repo remains the **upstream source of truth**. It keeps its own +GitHub Pages site and its own 6-hourly refresh; this copy is regenerated on each +publish from `main`. The two sites can legitimately disagree by a few hours. + +Vendored from commit **`edcf127`** ("Persist pypi.org request cache across CI +runs, pin actions to SHAs"). + +## Licensing + +The upstream project is BSD-2-Clause, whose clause 1 requires retaining the +copyright notice, so `LICENSE` is shipped verbatim and the vendored files carry +BSD-2-Clause SPDX headers. Only `build.py` is RISE-authored and MIT, matching the +rest of this repo. + +## Files + +| File | Origin | +|---|---| +| `LICENSE` | verbatim from upstream | +| `generate.py`, `utils.py`, `svg_wheel.py` | upstream, SPDX header prepended | +| `wheel.css`, `favicon.ico` | verbatim from upstream | +| `index.html` | upstream, with the text edits listed below | +| `build.py` | **new**, RISE-authored — not upstream | + +The generator's own dependencies (`requests`, `requests-cache`) live in +`ci_scripts/requirements.txt` alongside the rest of this repo's CI Python deps; +upstream's `pre-commit` pin is dropped, as there are no local hooks here. + +`build.py` holds all python-wheels-specific logic so that the upstream files stay +a clean copy. It fetches and sanity-checks the package list, runs `generate.py` in +a scratch directory, validates the result, and only then stages five files into +the Jekyll source tree. + +## Resyncing from upstream + +```bash +cd ci_scripts/dashboard +UP=/path/to/python-wheels-dashboard +cp "$UP"/{LICENSE,generate.py,utils.py,svg_wheel.py,index.html,wheel.css,favicon.ico} . +``` + +Then re-apply the local deltas: + +1. Prepend the BSD-2-Clause SPDX header to `generate.py`, `utils.py` and + `svg_wheel.py` (copy it from a sibling file). +2. Re-apply the `index.html` edits: + - drop the stale "top 360" figure from the "What is this list?" paragraph and + the "Thanks" paragraph — the generator applies no slice, so the real count + moves every run; + - change the footer cadence from "Updated daily." to the real cadence; + - keep the absolute link back to near the `

`. +3. If upstream changed its `requests`/`requests-cache` pins, update them in + `ci_scripts/requirements.txt`. +4. Update the vendored commit SHA above. + +Leave `build.py` alone — it is not upstream. + +**Never add YAML front matter to `index.html`.** It contains AngularJS bindings +(`{{ package.name }}`) that are also valid Liquid. The file is correct only +because Jekyll treats it as a static file and copies it byte-for-byte; front +matter would make Liquid eat the bindings and the package list would render empty +with no build error. `build.py` asserts this. + +## Running it locally + +```bash +pip install -r ../requirements.txt +python build.py --build-dir /tmp/dash --output-dir ../../docs/dashboard +``` + +A cold run makes one pypi.org request per package (~15000 at 8 concurrency) and +writes a multi-GB `requests-cache.sqlite` into the build dir, which is why the +build dir is kept outside `docs/`. diff --git a/ci_scripts/dashboard/build.py b/ci_scripts/dashboard/build.py new file mode 100644 index 00000000000..958c4aafe4b --- /dev/null +++ b/ci_scripts/dashboard/build.py @@ -0,0 +1,221 @@ +# SPDX-FileCopyrightText: 2026 The RISE Project +# SPDX-License-Identifier: MIT + +"""Build the RISC-V Wheels dashboard into the Jekyll source tree. + +Wraps the vendored generator (see README.md) with the sanity checks the website +build needs: the upstream package list has served HTTP 200 with zero rows, and +publishing that would replace a working dashboard with a blank one. + +Nothing is written to --output-dir unless a complete result was produced, so a +failed run leaves the previously published dashboard serving untouched. +""" + +import argparse +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import urllib.request + +VENDOR_DIR = Path(__file__).resolve().parent + +UPSTREAM_LIST_URL = "https://hugovk.dev/top-pypi-packages/top-pypi-packages.min.json" +# The upstream endpoint is a live query and has returned HTTP 200 carrying an +# empty `rows` plus a ClickHouse row-limit `exception`. This pinned copy of the +# same file is committed upstream and regenerated monthly, so falling back to it +# costs at most one month of ranking drift. +FALLBACK_LIST_URL = ( + "https://raw.githubusercontent.com/hugovk/top-pypi-packages/" + "6becf8c3b/top-pypi-packages.min.json" +) + +# Loose floors: enough to catch an empty or truncated upstream response without +# tripping on normal drift (the list is ~15000 rows, of which ~1900 survive the +# extension filter). +MIN_INPUT_ROWS = 5000 +MIN_OUTPUT_PACKAGES = 500 +MIN_WHEEL_SVG_BYTES = 10 * 1024 + +# Copied from the vendored directory as-is. +STATIC_FILES = ("index.html", "wheel.css", "favicon.ico") +# Produced by generate.py in the build directory. +GENERATED_FILES = ("results.json", "wheel.svg") + + +def fetch(url): + """Return the payload at ``url``, or ``None`` after printing why not.""" + try: + with urllib.request.urlopen(url, timeout=120) as response: + # file:// responses carry status None; only HTTP has one to check. + status = getattr(response, "status", None) + if status is not None and status != 200: + print(f" rejected: HTTP {status}") + return None + return response.read() + except Exception as e: + print(f" rejected: {e}") + return None + + +def validate_package_list(payload): + """Return ``(rows, None)`` if the payload is usable, else ``(None, reason)``.""" + try: + data = json.loads(payload) + except (json.JSONDecodeError, UnicodeDecodeError) as e: + return None, f"not valid JSON: {e}" + + if not isinstance(data, dict): + return None, "payload is not a JSON object" + + # The upstream query reports failure in-band, with a 200 and empty rows. + if "exception" in data: + return None, f"upstream reported an error: {data['exception']}" + + rows = data.get("rows") + if not isinstance(rows, list): + return None, "payload has no `rows` list" + if len(rows) < MIN_INPUT_ROWS: + return None, f"only {len(rows)} rows, expected at least {MIN_INPUT_ROWS}" + + return rows, None + + +def get_package_list(candidates): + """Fetch the package list from the first usable candidate. Exits on failure.""" + failures = [] + for label, url in candidates: + print(f"Fetching the {label} package list: {url}") + payload = fetch(url) + if payload is not None: + rows, reason = validate_package_list(payload) + if rows is not None: + print(f" accepted: {len(rows)} rows") + if label == "pinned": + print( + "::warning::The upstream package list was unusable; " + "using the pinned copy, so the download rankings may be " + "up to a month old." + ) + return payload + print(f" rejected: {reason}") + failures.append(f"{label}: {reason}") + else: + failures.append(f"{label}: could not be fetched") + + sys.exit( + "Could not obtain a usable package list, refusing to publish:\n " + + "\n ".join(failures) + ) + + +def validate_output(build_dir): + """Exit unless the generator produced a complete, sane result.""" + results = build_dir / "results.json" + try: + with open(results, encoding="utf-8") as f: + data = json.load(f) + except (OSError, json.JSONDecodeError) as e: + sys.exit(f"{results} is unusable: {e}") + + packages = data.get("data") + if not isinstance(packages, list): + sys.exit(f"{results} has no `data` list") + if len(packages) < MIN_OUTPUT_PACKAGES: + sys.exit( + f"{results} has only {len(packages)} packages, expected at least " + f"{MIN_OUTPUT_PACKAGES}" + ) + if not isinstance(data.get("last_update"), str) or not data["last_update"]: + sys.exit(f"{results} has no `last_update`") + + wheel = build_dir / "wheel.svg" + if not wheel.is_file(): + sys.exit(f"{wheel} was not generated") + size = wheel.stat().st_size + if size < MIN_WHEEL_SVG_BYTES: + # One per package, so a near-empty wheel is a few hundred bytes. + sys.exit(f"{wheel} is only {size} bytes, expected at least " + f"{MIN_WHEEL_SVG_BYTES}") + + return len(packages) + + +def check_no_front_matter(): + """Exit if index.html gained YAML front matter. + + The page's AngularJS bindings (``{{ package.name }}``) are also valid Liquid. + Jekyll only leaves them alone because a file without front matter is a static + file, copied byte-for-byte. Add front matter and the package list renders + empty in production with no build error. + """ + index = VENDOR_DIR / "index.html" + with open(index, encoding="utf-8") as f: + if f.read(3) == "---": + sys.exit( + f"{index} starts with YAML front matter. Jekyll would then run " + "Liquid over it and eat the AngularJS bindings, silently " + "emptying the package list. Remove the front matter." + ) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--build-dir", + required=True, + type=Path, + help="scratch directory for the generator; must be outside docs/, as it " + "collects a multi-GB requests-cache.sqlite", + ) + parser.add_argument( + "--output-dir", + required=True, + type=Path, + help="where to stage the dashboard, e.g. docs/dashboard", + ) + parser.add_argument( + "--list-url", + help="override the upstream package list URL (for testing the input gate)", + ) + args = parser.parse_args() + + if args.list_url: + candidates = [("override", args.list_url)] + else: + candidates = [("upstream", UPSTREAM_LIST_URL), ("pinned", FALLBACK_LIST_URL)] + + check_no_front_matter() + + build_dir = args.build_dir.resolve() + build_dir.mkdir(parents=True, exist_ok=True) + + payload = get_package_list(candidates) + (build_dir / "top-pypi-packages.json").write_bytes(payload) + + # A subprocess, not an import: utils.py builds its CachedSession at module + # import time, so the sqlite path binds to the cwd of whoever imports it + # first. Running with cwd=build_dir keeps every cwd-relative read and write + # out of the Jekyll source tree. + print("Running the vendored generator...") + subprocess.run( + [sys.executable, str(VENDOR_DIR / "generate.py")], + cwd=build_dir, + check=True, + ) + + count = validate_output(build_dir) + + os.makedirs(args.output_dir, exist_ok=True) + for name in STATIC_FILES: + shutil.copy2(VENDOR_DIR / name, args.output_dir / name) + for name in GENERATED_FILES: + shutil.copy2(build_dir / name, args.output_dir / name) + + print(f"Staged the dashboard for {count} packages in {args.output_dir}") + + +if __name__ == "__main__": + main() diff --git a/ci_scripts/dashboard/favicon.ico b/ci_scripts/dashboard/favicon.ico new file mode 100644 index 00000000000..e803ae04267 Binary files /dev/null and b/ci_scripts/dashboard/favicon.ico differ diff --git a/ci_scripts/dashboard/generate.py b/ci_scripts/dashboard/generate.py new file mode 100644 index 00000000000..714b27e3a84 --- /dev/null +++ b/ci_scripts/dashboard/generate.py @@ -0,0 +1,23 @@ +# SPDX-FileCopyrightText: 2013 Charlie Denton +# SPDX-FileCopyrightText: 2026 Stan Ulbrych +# SPDX-License-Identifier: BSD-2-Clause +# +# Vendored from riseproject-dev/python-wheels-dashboard. See README.md. + +from svg_wheel import generate_svg_wheel +from utils import ( + annotate_wheels, + get_top_packages, + save_to_file, +) + + +def main() -> None: + packages = get_top_packages() + packages = annotate_wheels(packages) + save_to_file(packages, "results.json") + generate_svg_wheel(packages) + + +if __name__ == "__main__": + main() diff --git a/ci_scripts/dashboard/index.html b/ci_scripts/dashboard/index.html new file mode 100644 index 00000000000..668552cf382 --- /dev/null +++ b/ci_scripts/dashboard/index.html @@ -0,0 +1,147 @@ + + + + + + + + + ⚙️ RISC-V Wheels + + +
+
+
+

⚙️ RISC-V Wheels

+

RISE Python Wheels — riscv64 binary wheels for popular Python projects

+ +

What are wheels?

+

Wheels are the standard binary format for distributing Python packages. See pythonwheels.com.

+

What are RISC-V wheels?

+

RISC-V is an open standard instruction set architecture, and 64-bit RISC-V (riscv64) hardware and Linux distributions are now widely available. + Pure-Python wheels already install on riscv64, but wheels with extensions need to be built for the architecture. + Without them, pip install falls back to building from source, which is slow at best and fails at worst. + This site shows which packages with extensions ship riscv64 wheels. + See Packaging support for riscv64. +

+

PyPI accepts riscv64 wheels, and manylinux has provided riscv64 images since July 2025. Installing them needs pip 24.1 or later.

+

What is this list?

+

This site shows the most-downloaded packages with extensions on PyPI:

+
    +
  • Dark green packages with a ✓ offer riscv64 wheels on PyPI
    • +
  • Light green packages with a 📦 have no riscv64 wheels on PyPI, but are built for riscv64 in the RISE registry
    • +
  • Light blue packages offer generic pure Python wheels, but no compiled wheels for riscv64 (yet!)
    • +
  • Orange packages offer wheels, but nothing that installs on riscv64 (yet!)
    • +
+

The number in the wheel counts the packages a riscv64 wheel exists for, whether on PyPI or in the RISE registry.

+

RISC-V wheels have a platform tag containing riscv64, for example, manylinux_2_39_riscv64.

+

Packages that are known to be deprecated are not included (for example, distribute). If your package is incorrectly listed, please create a ticket.

+

My package is orange. What can I do?

+

cibuildwheel can build riscv64 wheels: add riscv64 to your Linux architectures. + See the RISE wheel builder docs and riseproject-dev/python-wheels, where riscv64 packaging gaps are tracked and patched.

+

Something's wrong with this page!

+

Fantastic, a problem found is a problem fixed. Please create a ticket!

+

You can also submit a pull-request.

+

Note: Requests for behavioural changes in the packaging tools themselves should be directed to discuss.python.org and the Python Packaging Authority.

+

Thanks

+

This is a derivative work of Free-Threaded Wheels, itself a derivative of Python Wheels, a site that tracks which Python distributions ship the wheel distribution. + The rankings come from Top PyPI Packages. + The riscv64 registry comes from RISE.

+

Thanks also to the many contributors.

+
+
+
+ This page requires javascript to be enabled to display the list of packages. + + + + +
+
+
+ +
+ + + + + + diff --git a/ci_scripts/dashboard/svg_wheel.py b/ci_scripts/dashboard/svg_wheel.py new file mode 100644 index 00000000000..e7609e5edc8 --- /dev/null +++ b/ci_scripts/dashboard/svg_wheel.py @@ -0,0 +1,148 @@ +# SPDX-FileCopyrightText: 2013 Charlie Denton +# SPDX-FileCopyrightText: 2026 Stan Ulbrych +# SPDX-License-Identifier: BSD-2-Clause +# +# Vendored from riseproject-dev/python-wheels-dashboard. See README.md. + +import math +from math import tau as 𝜏 +import xml.etree.ElementTree as et + +HEADERS = b""" + + +""" + +PATH_TEMPLATE = """ +M {start_outer_x},{start_outer_y} +A{outer_radius},{outer_radius} 0 0 1 {end_outer_x},{end_outer_y} +L {start_inner_x},{start_inner_y} +A{inner_radius},{inner_radius} 0 0 0 {end_inner_x},{end_inner_y} +Z +""" + +FRACTION_LINE = 80 +OFFSET = 20 +PADDING = 10 +OUTER_RADIUS = 180 +INNER_RADIUS = OUTER_RADIUS / 2 +CENTER = PADDING + OUTER_RADIUS + + +def annular_sector_path(start, stop): + cos_stop = math.cos(stop) + cos_start = math.cos(start) + sin_stop = math.sin(stop) + sin_start = math.sin(start) + + points = { + "inner_radius": INNER_RADIUS, + "outer_radius": OUTER_RADIUS, + "start_outer_x": CENTER + OUTER_RADIUS * cos_start, + "start_outer_y": CENTER + OUTER_RADIUS * sin_start, + "end_outer_x": CENTER + OUTER_RADIUS * cos_stop, + "end_outer_y": CENTER + OUTER_RADIUS * sin_stop, + "start_inner_x": CENTER + INNER_RADIUS * cos_stop, + "start_inner_y": CENTER + INNER_RADIUS * sin_stop, + "end_inner_x": CENTER + INNER_RADIUS * cos_start, + "end_inner_y": CENTER + INNER_RADIUS * sin_start, + } + return PATH_TEMPLATE.format(**points) + + +def add_annular_sectors(wheel, packages): + total = len(packages) + for index, result in enumerate(packages): + sector = et.SubElement( + wheel, + "path", + d=annular_sector_path(*angles(index, total)), + attrib={"class": result["css_class"]}, + ) + title = et.SubElement(sector, "title") + title.text = f"{result['name']} {result['icon']}" + + +def angles(index, total): + # Angle, in radians, of one wedge of the wheel. + angle_per_wedge = 𝜏 / total + # Used to turn the start of the wheel from east to north. + quarter_circle = 𝜏 / 4 + + # Angle of the beginning of the wedge. + start = (index * angle_per_wedge) - quarter_circle + # Angle of the end of the wedge. + stop = start + angle_per_wedge + + return start, stop + + +def add_fraction(wheel, packages): + total = len(packages) + + text_attributes = { + "class": "wheel-text", + "text-anchor": "middle", + "dominant-baseline": "central", + "font-size": str(2 * OFFSET), + "font-family": '"Helvetica Neue",Helvetica,Arial,sans-serif', + } + + # Packages with riscv64 wheels, on PyPI or in the RISE registry + wheel_packages = sum( + package["riscv64_wheel"] or package["rise_registry"] for package in packages + ) + + packages_with_wheels = et.SubElement( + wheel, + "text", + x=str(CENTER), + y=str(CENTER - OFFSET), + attrib=text_attributes, + ) + packages_with_wheels.text = f"{wheel_packages}" + + title = et.SubElement(packages_with_wheels, "title") + percentage = f"{wheel_packages / float(total):.0%}" + title.text = percentage + + # Dividing line + et.SubElement( + wheel, + "line", + x1=str(CENTER - FRACTION_LINE // 2), + y1=str(CENTER), + x2=str(CENTER + FRACTION_LINE // 2), + y2=str(CENTER), + attrib={"class": "wheel-line", "stroke-width": "2"}, + ) + + # Total packages + total_packages = et.SubElement( + wheel, + "text", + x=str(CENTER), + y=str(CENTER + OFFSET), + attrib=text_attributes, + ) + total_packages.text = f"{total}" + + title = et.SubElement(total_packages, "title") + title.text = percentage + + +def generate_svg_wheel(packages): + wheel = et.Element( + "svg", + viewBox=f"0 0 {2 * CENTER} {2 * CENTER}", + version="1.1", + xmlns="http://www.w3.org/2000/svg", + ) + add_annular_sectors(wheel, packages) + + add_fraction(wheel, packages) + + with open("wheel.svg", "wb") as svg: + svg.write(HEADERS) + svg.write(et.tostring(wheel)) diff --git a/ci_scripts/dashboard/utils.py b/ci_scripts/dashboard/utils.py new file mode 100644 index 00000000000..712595b825a --- /dev/null +++ b/ci_scripts/dashboard/utils.py @@ -0,0 +1,329 @@ +# SPDX-FileCopyrightText: 2013 Charlie Denton +# SPDX-FileCopyrightText: 2026 Stan Ulbrych +# SPDX-License-Identifier: BSD-2-Clause +# +# Vendored from riseproject-dev/python-wheels-dashboard. See README.md. + +import datetime as dt +import json +import re +import requests_cache + +from concurrent.futures import ThreadPoolExecutor + + +BASE_URL = "https://pypi.org/pypi" +RISE_REGISTRY_URL = "https://pypi.riseproject.dev/simple" + +DEPRECATED_PACKAGES = { + "BeautifulSoup", + "bs4", + "distribute", + "django-social-auth", + "nose", + "pep8", + "pycrypto", + "pypular", + "sklearn", + "y-py", +} + +# Packages we deliberately do not port to riscv64, and why. +IGNORED_PACKAGES = { + # Proprietary GPU/vendor blobs, x86_64 and aarch64 only + "cuda-bindings", + "cuda-core", + "nvidia-cublas-cu11", + "nvidia-cublas-cu12", + "nvidia-cublas", + "nvidia-cuda-cccl-cu12", + "nvidia-cuda-cccl", + "nvidia-cuda-crt", + "nvidia-cuda-cupti-cu11", + "nvidia-cuda-cupti-cu12", + "nvidia-cuda-cupti", + "nvidia-cuda-nvcc-cu12", + "nvidia-cuda-nvcc", + "nvidia-cuda-nvdisasm", + "nvidia-cuda-nvrtc-cu11", + "nvidia-cuda-nvrtc-cu12", + "nvidia-cuda-nvrtc", + "nvidia-cuda-runtime-cu11", + "nvidia-cuda-runtime-cu12", + "nvidia-cuda-runtime", + "nvidia-cuda-tileiras", + "nvidia-cudnn-cu11", + "nvidia-cudnn-cu12", + "nvidia-cudnn-cu13", + "nvidia-cudnn-frontend", + "nvidia-cufft-cu11", + "nvidia-cufft-cu12", + "nvidia-cufft", + "nvidia-cufile-cu12", + "nvidia-cufile", + "nvidia-curand-cu11", + "nvidia-curand-cu12", + "nvidia-curand", + "nvidia-cusolver-cu11", + "nvidia-cusolver-cu12", + "nvidia-cusolver", + "nvidia-cusparse-cu11", + "nvidia-cusparse-cu12", + "nvidia-cusparse", + "nvidia-cusparselt-cu12", + "nvidia-cusparselt-cu13", + "nvidia-cutlass-dsl-libs-base", + "nvidia-cutlass-dsl-libs-cu12", + "nvidia-cutlass-dsl-libs-cu13", + "nvidia-libnvcomp-cu12", + "nvidia-matmul-heuristics", + "nvidia-nccl-cu11", + "nvidia-nccl-cu12", + "nvidia-nccl-cu13", + "nvidia-nvfatbin-cu12", + "nvidia-nvfatbin", + "nvidia-nvimgcodec-cu12", + "nvidia-nvjitlink-cu12", + "nvidia-nvjitlink", + "nvidia-nvshmem-cu12", + "nvidia-nvshmem-cu13", + "nvidia-nvtx-cu11", + "nvidia-nvtx-cu12", + "nvidia-nvtx", + "nvidia-nvvm", + "nvidia-resiliency-ext", + "triton", + # Windows only + "pywin32", + "pywinpty", + "triton-windows", + "winkerberos", + "winrt-runtime", + "winrt-windows-foundation", + "winrt-windows-foundation-collections", + "winrt-windows-storage-streams", + "winrt-windows-devices-enumeration", + "winrt-windows-devices-bluetooth", + "winrt-windows-devices-bluetooth-advertisement", + "winrt-windows-devices-bluetooth-genericattributeprofile", + "winrt-windows-devices-radios", + "winrt-windows-graphics-imaging", + "winrt-windows-media-ocr", + "winrt-windows-globalization", + # macOS only + "pyobjc-core", + "pyobjc-framework-cocoa", + "pyobjc-framework-quartz", + "pyobjc-framework-applicationservices", + "pyobjc-framework-coretext", + "pyobjc-framework-security", + "pyobjc-framework-webkit", + "pyobjc-framework-corebluetooth", + "pyobjc-framework-coreml", + "pyobjc-framework-vision", + "pyobjc-framework-libdispatch", + "pyobjc-framework-coremedia", + "pyobjc-framework-coreaudio", + "pyobjc-framework-fsevents", + "pyobjc-framework-avfoundation", + "pyobjc-framework-corelocation", + "pyobjc-framework-metal", + "pyobjc-framework-systemconfiguration", + "pyobjc-framework-contacts", + "pyobjc-framework-coreservices", + "pyobjc-framework-photos", + "pyobjc-framework-spritekit", + "pyobjc-framework-cfnetwork", + "pyobjc-framework-addressbook", + "pyobjc-framework-localauthentication", + "pyobjc-framework-coredata", + "pyobjc-framework-automator", + "pyobjc-framework-discrecording", + "pyobjc-framework-screensaver", + "pyobjc-framework-syncservices", + "pyobjc-framework-coreaudiokit", + "pyobjc-framework-corewlan", + "pyobjc-framework-securityinterface", + "pyobjc-framework-speech", + "pyobjc-framework-coremidi", + "pyobjc-framework-screencapturekit", + "pyobjc-framework-avkit", + "pyobjc-framework-usernotifications", + "pyobjc-framework-corespotlight", + "pyobjc-framework-imagecapturecore", + "pyobjc-framework-cryptotokenkit", + "pyobjc-framework-scriptingbridge", + "pyobjc-framework-gamecenter", + "pyobjc-framework-contactsui", + "pyobjc-framework-metalperformanceshaders", + "pyobjc-framework-gamecontroller", + "pyobjc-framework-coremediaio", + "pyobjc-framework-intents", + "pyobjc-framework-storekit", + "pyobjc-framework-mapkit", + "pyobjc-framework-accessibility", + "pyobjc-framework-photosui", + "pyobjc-framework-multipeerconnectivity", + "pyobjc-framework-authenticationservices", + "pyobjc-framework-externalaccessory", + "pyobjc-framework-modelio", + "pyobjc-framework-fileprovider", + "pyobjc-framework-networkextension", + "pyobjc-framework-scenekit", + "pyobjc-framework-mediatoolbox", + "pyobjc-framework-audiovideobridging", + "pyobjc-framework-videotoolbox", + "pyobjc-framework-coremotion", + "pyobjc-framework-notificationcenter", + "pyobjc-framework-gamekit", + "pyobjc-framework-network", + "pyobjc-framework-gameplaykit", + "pyobjc-framework-automaticassessmentconfiguration", + "pyobjc-framework-safariservices", + "pyobjc-framework-metalkit", + "pyobjc-framework-callkit", + "pyobjc-framework-pushkit", + "pyobjc-framework-classkit", + "pyobjc-framework-oslog", + "pyobjc-framework-systemextensions", + "pyobjc-framework-passkit", + "pyobjc-framework-replaykit", + "pyobjc-framework-virtualization", + "pyobjc-framework-intentsui", + "pyobjc-framework-metrickit", + "pyobjc-framework-shazamkit", + "pyobjc-framework-sharedwithyoucore", + "pyobjc-framework-iobluetooth", + "pyobjc-framework-libxpc", + "pyobjc-framework-backgroundassets", + "pyobjc-framework-avrouting", + "pyobjc-framework-healthkit", + "pyobjc-framework-metalfx", + "pyobjc-framework-extensionkit", + "pyobjc-framework-safetykit", + "pyobjc-framework-sharedwithyou", + "pyobjc-framework-browserenginekit", + "pyobjc-framework-inputmethodkit", + "pyobjc-framework-mediaextension", + "pyobjc-framework-fskit", + "mlx-metal", +} + +# Keep responses for one hour +SESSION = requests_cache.CachedSession("requests-cache", expire_after=dt.timedelta(days=30)) + + +def get_json_url(package_name): + return BASE_URL + "/" + package_name + "/json" + + +def normalize(package_name: str) -> str: + """Normalise a package name for a simple index URL. + + https://packaging.python.org/en/latest/specifications/name-normalization/ + """ + return re.sub(r"[-_.]+", "-", package_name).lower() + + +def in_rise_registry(package_name: str) -> bool: + """Whether the RISE riscv64 registry has wheels for this package.""" + url = f"{RISE_REGISTRY_URL}/{normalize(package_name)}/" + try: + response = SESSION.get(url, allow_redirects=False) + except Exception as e: + print(f" ! Could not check the RISE registry for {package_name}: {e}") + return False + + return response.status_code == 200 + + +def annotate_wheels(packages) -> list[dict]: + print("Getting wheel data...") + keep = [] + + def _handle_package(package) -> tuple[dict, bool]: + if package["name"] in DEPRECATED_PACKAGES | IGNORED_PACKAGES: + return (package, False) + + has_other_binary_wheel = False + has_riscv64_wheel = False + has_pure_python_wheel = False + in_registry = False + url = get_json_url(package["name"]) + response = SESSION.get(url) + if response.status_code != 200: + print(" ! Skipping " + package["name"]) + return (package, False) + + data = response.json() + + for download in data["urls"]: + if download["packagetype"] == "bdist_wheel": + platform_tag = download["filename"].removesuffix(".whl").split("-")[-1] + + if "riscv64" in platform_tag: + has_riscv64_wheel = True + elif platform_tag != "any": + has_other_binary_wheel = True + else: + has_pure_python_wheel = True + + if has_riscv64_wheel: + package["css_class"] = "success" + package["icon"] = "\u2713" # Check mark + elif has_other_binary_wheel: + in_registry = in_rise_registry(package["name"]) + if in_registry: + package["css_class"] = "rise" + package["icon"] = "📦" + elif has_pure_python_wheel: + package["css_class"] = "pure-py" + package["icon"] = "🐍" + else: + package["css_class"] = "warning" + package["icon"] = "\u2717" # Ballot X + else: + return (package, False) + + package["riscv64_wheel"] = has_riscv64_wheel + package["rise_registry"] = in_registry + + return (package, True) + + with ThreadPoolExecutor(max_workers=8) as executor: + for index, (package, should_keep) in enumerate( + executor.map(_handle_package, packages) + ): + print(f"{index + 1}/{len(packages)} {package['name']}") + if should_keep: + keep.append(package) + + return keep + + +def get_top_packages(): + print("Getting packages...") + + with open("top-pypi-packages.json") as data_file: + packages = json.load(data_file)["rows"] + + # Rename keys + for package in packages: + package["downloads"] = package.pop("download_count") + package["name"] = package.pop("project") + + return packages + + +def save_to_file(packages, file_name): + now = dt.datetime.now(tz=dt.timezone.utc) + with open(file_name, "w") as f: + f.write( + json.dumps( + { + "data": packages, + "last_update": now.strftime("%A, %d %B %Y, %X %Z"), + }, + indent=1, + ) + ) diff --git a/ci_scripts/dashboard/wheel.css b/ci_scripts/dashboard/wheel.css new file mode 100644 index 00000000000..689f8249ff1 --- /dev/null +++ b/ci_scripts/dashboard/wheel.css @@ -0,0 +1,41 @@ +.success { + stroke: #198754; + stroke-width: 1; + fill: #198754; +} + +.warning { + stroke: #ffca2c; + stroke-width: 1; + fill: #ffc107; +} + +.rise { + stroke: #90EE90; + stroke-width: 1; + fill: #90EE90; +} + +.pure-py { + stroke: #9ec5fe; + stroke-width: 1; + fill: #9ec5fe; +} + +line.wheel-line { + stroke: #333; +} + +text.wheel-text { + fill: #333; +} + +@media (prefers-color-scheme: dark) { + line.wheel-line { + stroke: #ccc; + } + text.wheel-text { + fill: #ccc; + } + +} diff --git a/ci_scripts/requirements.txt b/ci_scripts/requirements.txt index 5e09fee6c7d..8f45fc70f24 100644 --- a/ci_scripts/requirements.txt +++ b/ci_scripts/requirements.txt @@ -3,3 +3,6 @@ pyyaml sphinx-rtd-theme sphinx-copybutton sphinx-inline-tabs + +requests==2.34.2 +requests-cache==1.3.3 diff --git a/docs/.gitignore b/docs/.gitignore index c93c057f9bd..b246c904fe5 100644 --- a/docs/.gitignore +++ b/docs/.gitignore @@ -1,2 +1,3 @@ packages/*.md simple/ +dashboard/ diff --git a/docs/_config.yml b/docs/_config.yml index d88cb1f0f86..553a6c8bab9 100644 --- a/docs/_config.yml +++ b/docs/_config.yml @@ -27,6 +27,15 @@ aux_links: "GitHub": - "https://github.com/riseproject-dev/python-wheels" +# The dashboard is a static file (no front matter, so Liquid never touches its +# AngularJS bindings) and therefore cannot carry nav_order. The theme pipes this +# through absolute_url, so a plain path also resolves correctly under a PR +# preview's baseurl. hide_icon because the target is on this same site. +nav_external_links: + - title: Dashboard + url: /dashboard/ + hide_icon: true + exclude: - packages/generate_packages_doc.py - packages/*.yaml diff --git a/docs/index.md b/docs/index.md index caf2ae34ba5..85f4e6276c4 100644 --- a/docs/index.md +++ b/docs/index.md @@ -27,3 +27,4 @@ website](https://riseproject.dev/). {% if p.url == page.url %}{% continue %}{% endif -%} - [{{ p.title }}]({{ p.url | relative_url }}) {% endfor %} +- [Dashboard]({{ '/dashboard/' | relative_url }}) diff --git a/docs/workflows.md b/docs/workflows.md index 273d3bacf76..afa2f3abef5 100644 --- a/docs/workflows.md +++ b/docs/workflows.md @@ -112,10 +112,14 @@ following workflows beyond those used for specific package builds: ## The RISC-V Wheels Dashboard -RISE makes use of the [RISC-V -Wheels](https://stanfromireland.github.io/riscv-wheels/) dashboard, which tracks -the status of riscv64 compatibility for 360 binary Python wheels. This provides -a detailed look at ecosystem-wide support, including wheels which are available -in the RISE package index but not yet upstream. It should be the first reference when -determining which packages to contribute support for and at what level (i.e. -upstream versus `python-wheels`). +RISE maintains the [RISC-V Wheels]({{ '/dashboard/' | relative_url }}) dashboard, +which tracks the status of riscv64 compatibility across the most-downloaded +binary Python wheels. This provides a detailed look at ecosystem-wide support, +including wheels which are available in the RISE package index but not yet +upstream. It should be the first reference when determining which packages to +contribute support for and at what level (i.e. upstream versus `python-wheels`). + +The dashboard is regenerated with each publish of this site. Its generator is +vendored in `ci_scripts/dashboard/` from +[riseproject-dev/python-wheels-dashboard](https://github.com/riseproject-dev/python-wheels-dashboard), +which remains its upstream source of truth.