Repository navigation
docs: Update projects #18
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # SPDX-FileCopyrightText: 2026 The RISE Project | |
| # SPDX-License-Identifier: MIT | |
| --- | |
| # This workflow is based on: https://github.com/PyAV-Org/PyAV/blob/v18.1.0/.github/workflows/tests.yml | |
| name: Build av wheels (riscv64) | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| version: | |
| description: 'Version glob to (re)build; empty builds every version of docs/packages/av.yaml not released yet' | |
| required: false | |
| default: '' | |
| pull_request: | |
| branches: [main] | |
| paths: | |
| - '.github/workflows/build-av.yml' | |
| - 'docs/packages/av.yaml' | |
| push: | |
| branches: [main] | |
| paths: | |
| - '.github/workflows/build-av.yml' | |
| - 'docs/packages/av.yaml' | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} | |
| cancel-in-progress: true | |
| permissions: | |
| contents: read # to fetch code (actions/checkout) | |
| env: | |
| MANYLINUX_RISCV64_IMAGE: quay.io/pypa/manylinux_2_39_riscv64 | |
| jobs: | |
| setup: | |
| uses: $/.github/workflows/_setup.yml | |
| with: | |
| package: av | |
| version: ${{ inputs.version }} | |
| # The wheels bundle prebuilt FFmpeg libraries from a pyav-ffmpeg release, | |
| # including GPL x264/x265 and LGPL FFmpeg, GnuTLS, Nettle, GMP, libunistring, | |
| # alsa-lib and LAME. Publishing them obliges us to ship their licence texts | |
| # and to make the corresponding sources permanently available. | |
| vendor_sources: | |
| name: Collect av ${{ matrix.version }} vendored FFmpeg sources | |
| needs: [setup] | |
| if: needs.setup.outputs.versions != '[]' | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| version: ${{ fromJSON(needs.setup.outputs.versions) }} | |
| env: | |
| AV_VERSION: ${{ matrix.version }} | |
| steps: | |
| - name: Checkout PyAV v${{ env.AV_VERSION }} | |
| uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| repository: PyAV-Org/PyAV | |
| ref: v${{ env.AV_VERSION }} | |
| persist-credentials: false | |
| path: pyav | |
| - name: Download the vendored sources and extract their licences | |
| run: | | |
| cat > "$RUNNER_TEMP/collect-vendor-sources.py" <<'PY' | |
| # SPDX-FileCopyrightText: 2026 The RISE Project | |
| # SPDX-License-Identifier: MIT | |
| """Collect the sources and licence texts of the FFmpeg stack PyAV vendors. | |
| PyAV's wheels bundle prebuilt shared libraries fetched from a pyav-ffmpeg | |
| release. Several of them are GPL (x264, x265) or LGPL (FFmpeg, GnuTLS, | |
| Nettle, GMP, libunistring, alsa-lib, LAME), so redistributing the wheels | |
| carries a source-distribution obligation. pyav-ffmpeg pins every dependency | |
| by URL and SHA-256 in scripts/pkg.py, which is what this reads. | |
| """ | |
| import argparse | |
| import hashlib | |
| import json | |
| import re | |
| import subprocess | |
| import sys | |
| import tarfile | |
| from pathlib import Path | |
| LICENCE_RE = re.compile(r"^(COPYING|COPYRIGHT|LICEN[CS]E|NOTICE)", re.IGNORECASE) | |
| def load_packages(pkg_py: str): | |
| namespace: dict = {} | |
| exec(compile(pkg_py, "pkg.py", "exec"), namespace) | |
| # Linux riscv64 enables gnutls, alsa and libvpl; CUDA/AMF/nasm are x86-only. | |
| packages = ( | |
| namespace["gnutls_group"] | |
| + namespace["codec_group"] | |
| + [ | |
| namespace["alsa_package"], | |
| namespace["libvpl_package"], | |
| namespace["ffmpeg_package"], | |
| ] | |
| ) | |
| return sorted(packages, key=lambda p: p.name) | |
| def download(package, dest_dir: Path) -> Path: | |
| name = package.source_filename or package.source_url.rsplit("/", 1)[-1] | |
| # A few upstreams name their tarball after the tag alone ("v2.16.0.tar.gz"). | |
| if package.name.replace("-", "").lower() not in name.replace("-", "").lower(): | |
| name = f"{package.name}-{name}" | |
| path = dest_dir / name | |
| subprocess.run( | |
| ["curl", "--location", "--fail", "--silent", "--show-error", | |
| "--output", str(path), package.source_url], | |
| check=True, | |
| ) | |
| digest = hashlib.sha256(path.read_bytes()).hexdigest() | |
| if digest != package.sha256: | |
| raise SystemExit( | |
| f"{package.name}: sha256 mismatch for {package.source_url}\n" | |
| f" expected {package.sha256}\n got {digest}" | |
| ) | |
| print(f"{package.name}: {name} ({path.stat().st_size} bytes, sha256 ok)") | |
| return path | |
| def extract_licences(package, tarball: Path, dest_dir: Path) -> None: | |
| chunks = [] | |
| with tarfile.open(tarball) as tar: | |
| for member in tar.getmembers(): | |
| parts = Path(member.name).parts | |
| # Top level of the archive, plus one nested directory (x265 keeps | |
| # its sources under source/, gnutls its licences under doc/). | |
| if not member.isfile() or len(parts) > 3: | |
| continue | |
| if not LICENCE_RE.match(parts[-1]): | |
| continue | |
| handle = tar.extractfile(member) | |
| if handle is None: | |
| continue | |
| text = handle.read().decode("utf-8", "replace") | |
| chunks.append(f"===== {'/'.join(parts[1:])} =====\n\n{text}") | |
| if not chunks: | |
| raise SystemExit(f"{package.name}: no licence file found in {tarball.name}") | |
| header = ( | |
| f"Licence texts for {package.name}, bundled in this wheel as a prebuilt\n" | |
| f"shared library. Source: {package.source_url}\n\n" | |
| ) | |
| (dest_dir / f"LICENSE.{package.name}").write_text(header + "\n\n".join(chunks)) | |
| print(f"{package.name}: {len(chunks)} licence file(s)") | |
| def main() -> None: | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument("--pyav-dir", type=Path, required=True) | |
| parser.add_argument("--sources-dir", type=Path, required=True) | |
| parser.add_argument("--licenses-dir", type=Path, required=True) | |
| args = parser.parse_args() | |
| config = json.loads((args.pyav_dir / "scripts" / "ffmpeg-latest.json").read_text()) | |
| tag = config["url"].split("/download/")[1].split("/")[0] | |
| print(f"pyav-ffmpeg release: {tag}") | |
| args.sources_dir.mkdir(parents=True, exist_ok=True) | |
| args.licenses_dir.mkdir(parents=True, exist_ok=True) | |
| # pyav-ffmpeg carries the build recipe and the patches it applies to FFmpeg, | |
| # GMP, LAME and libvpx, so it is part of the corresponding source. | |
| recipe = args.sources_dir / f"pyav-ffmpeg-{tag}.tar.gz" | |
| subprocess.run( | |
| ["curl", "--location", "--fail", "--silent", "--show-error", "--output", str(recipe), | |
| f"https://github.com/PyAV-Org/pyav-ffmpeg/archive/refs/tags/{tag}.tar.gz"], | |
| check=True, | |
| ) | |
| with tarfile.open(recipe) as tar: | |
| member = next(m for m in tar.getmembers() if m.name.endswith("/scripts/pkg.py")) | |
| pkg_py = tar.extractfile(member).read().decode() | |
| for package in load_packages(pkg_py): | |
| tarball = download(package, args.sources_dir) | |
| extract_licences(package, tarball, args.licenses_dir) | |
| if __name__ == "__main__": | |
| sys.exit(main()) | |
| PY | |
| python3 "$RUNNER_TEMP/collect-vendor-sources.py" \ | |
| --pyav-dir pyav --sources-dir sources --licenses-dir licenses | |
| tar -cf gpl-sources.tar -C sources . | |
| - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: av-${{ env.AV_VERSION }}-gpl-sources | |
| path: gpl-sources.tar | |
| if-no-files-found: error | |
| - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: av-${{ env.AV_VERSION }}-vendor-licenses | |
| path: licenses/ | |
| if-no-files-found: error | |
| build_wheels: | |
| name: Build av ${{ matrix.version }} manylinux_riscv64 | |
| needs: [setup, vendor_sources] | |
| if: needs.setup.outputs.versions != '[]' | |
| runs-on: ubuntu-24.04-riscv | |
| timeout-minutes: 300 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| version: ${{ fromJSON(needs.setup.outputs.versions) }} | |
| env: | |
| AV_VERSION: ${{ matrix.version }} | |
| steps: | |
| - name: Checkout PyAV v${{ env.AV_VERSION }} | |
| uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| repository: PyAV-Org/PyAV | |
| ref: v${{ env.AV_VERSION }} | |
| persist-credentials: false | |
| - name: Fetch the vendored libraries' licence texts | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: av-${{ env.AV_VERSION }}-vendor-licenses | |
| path: vendor-licenses | |
| # setuptools' default license-files glob picks these up from the project root. | |
| - name: Stage the licence texts for packaging | |
| run: cp vendor-licenses/LICENSE.* . | |
| - name: Build wheels | |
| uses: pypa/cibuildwheel@1828c10ab37f080699c7b81cea34097c684a7074 # v4.2.0 | |
| with: | |
| output-dir: wheelhouse/ | |
| env: | |
| CIBW_ARCHS: riscv64 | |
| CIBW_BUILD: "cp311* cp314t*" | |
| CIBW_MANYLINUX_RISCV64_IMAGE: ${{ env.MANYLINUX_RISCV64_IMAGE }} | |
| CIBW_BEFORE_BUILD: python scripts/fetch-vendor.py --config-file scripts/ffmpeg-latest.json /tmp/vendor | |
| CIBW_ENVIRONMENT_LINUX: >- | |
| LD_LIBRARY_PATH=/tmp/vendor/lib:$LD_LIBRARY_PATH | |
| PKG_CONFIG_PATH=/tmp/vendor/lib/pkgconfig | |
| PIP_EXTRA_INDEX_URL=https://pypi.riseproject.dev/simple/ | |
| CIBW_TEST_REQUIRES: pytest numpy | |
| CIBW_TEST_COMMAND: mv {project}/av {project}/av.disabled && python -m pytest {package}/tests && mv {project}/av.disabled {project}/av | |
| - name: Check the wheels ship the compiled extensions and vendored licences | |
| run: | | |
| python3 - wheelhouse/*.whl <<'EOF' | |
| import pathlib, sys, zipfile | |
| expected = {p.name for p in pathlib.Path("vendor-licenses").iterdir()} | |
| for whl in sys.argv[1:]: | |
| names = zipfile.ZipFile(whl).namelist() | |
| sos = [n for n in names if n.startswith("av/") and n.endswith(".so")] | |
| libs = [n for n in names if n.startswith("av.libs/")] | |
| shipped = {n.rsplit("/", 1)[1] for n in names if ".dist-info/licenses/" in n} - {""} | |
| assert sos, f"no av/*.so in {whl}" | |
| assert any("libavcodec" in n for n in libs), f"no vendored FFmpeg in {whl}" | |
| assert expected <= shipped, f"{whl} is missing {sorted(expected - shipped)}" | |
| print(f"{whl}: {len(sos)} extension modules, {len(libs)} bundled libraries, " | |
| f"{len(shipped)} licence files") | |
| EOF | |
| - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: av-${{ env.AV_VERSION }}-manylinux_riscv64 | |
| path: wheelhouse/*.whl | |
| if-no-files-found: error | |
| publish: | |
| name: Publish av ${{ matrix.version }} | |
| needs: [setup, vendor_sources, build_wheels] | |
| if: needs.setup.outputs.versions != '[]' | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| version: ${{ fromJSON(needs.setup.outputs.versions) }} | |
| permissions: | |
| contents: write | |
| pull-requests: write | |
| uses: $/.github/workflows/_publish-wheel.yml | |
| secrets: | |
| app-private-key: ${{ secrets.RISEPROJECT_APP_PRIVATE_KEY }} | |
| with: | |
| artifact-pattern: av-${{ matrix.version }}-manylinux_riscv64 | |
| gpl-sources-artifact: av-${{ matrix.version }}-gpl-sources | |
| gpl-sources-description: FFmpeg, x264, x265 |