docs: Update projects (#2133) #14
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # SPDX-FileCopyrightText: 2026 The RISE Project | |
| # SPDX-License-Identifier: MIT | |
| --- | |
| # Based on upstream's own manylinux wheel recipe, narrowed to riscv64: | |
| # https://github.com/apache/pulsar-client-python/blob/v3.13.0/.github/workflows/ci-build-release-wheels.yaml | |
| # https://github.com/apache/pulsar-client-python/blob/v3.13.0/pkg/build-wheel-inside-docker.sh | |
| # Upstream compiles the Pulsar C++ client from the apache-pulsar-client-cpp release | |
| # tarball its dependencies.yaml pins, with vcpkg supplying that library's own | |
| # dependencies; vcpkg has no tested riscv64 triplet, so the same tarball is built | |
| # against the manylinux image's boost/openssl/protobuf/curl/snappy/zstd instead. | |
| name: Build pulsar-client wheels (riscv64) | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| version: | |
| description: 'Version glob to (re)build; empty builds every version of docs/packages/pulsar-client.yaml not released yet' | |
| required: false | |
| default: '' | |
| pull_request: | |
| branches: [main] | |
| paths: | |
| - '.github/workflows/build-pulsar-client.yml' | |
| - 'docs/packages/pulsar-client.yaml' | |
| push: | |
| branches: [main] | |
| paths: | |
| - '.github/workflows/build-pulsar-client.yml' | |
| - 'docs/packages/pulsar-client.yaml' | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} | |
| cancel-in-progress: true | |
| permissions: | |
| contents: read # to fetch code (actions/checkout) | |
| env: | |
| MANYLINUX_RISCV64_IMAGE: quay.io/pypa/manylinux_2_39_riscv64 | |
| jobs: | |
| setup: | |
| uses: $/.github/workflows/_setup.yml | |
| with: | |
| package: pulsar-client | |
| version: ${{ inputs.version }} | |
| build_wheels: | |
| needs: [setup] | |
| if: needs.setup.outputs.versions != '[]' | |
| name: Build pulsar-client ${{ matrix.version }} ${{ matrix.python }}-manylinux_riscv64 | |
| runs-on: ubuntu-24.04-riscv | |
| timeout-minutes: 360 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| version: ${{ fromJSON(needs.setup.outputs.versions) }} | |
| # No cp314t: upstream publishes no free-threaded wheels. | |
| python: ["cp312", "cp313", "cp314"] | |
| env: | |
| PULSAR_CLIENT_VERSION: ${{ matrix.version }} | |
| steps: | |
| - name: Checkout pulsar-client-python v${{ env.PULSAR_CLIENT_VERSION }} | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| repository: apache/pulsar-client-python | |
| ref: v${{ env.PULSAR_CLIENT_VERSION }} | |
| persist-credentials: false | |
| - name: Stage the licence-collection script | |
| run: | | |
| cat > collect-licenses.sh <<'COLLECT_EOF' | |
| #!/bin/bash | |
| # SPDX-FileCopyrightText: 2026 The RISE Project | |
| # SPDX-License-Identifier: MIT | |
| # | |
| # Stage, at the project root, the licence of every shared library auditwheel | |
| # vendors alongside libpulsar. setuptools' default LICENSE* glob copies them | |
| # into the wheel. | |
| set -euo pipefail | |
| project="${1:?usage: collect-licenses.sh <project-dir>}" | |
| # ldd is transitive, so the one linked library covers its whole closure. | |
| # glibc, the gcc runtime and zlib are on auditwheel's manylinux allowlist and | |
| # are never vendored into the wheel. | |
| mapfile -t libs < <(ldd /usr/local/lib/libpulsar.so | tr ' ' '\n' | grep '^/' | sort -u) | |
| # `rpm -qf` reports unowned files on stdout, so keep only bare package names. | |
| mapfile -t pkgs < <( | |
| rpm -qf --qf '%{NAME}\n' "${libs[@]}" 2>/dev/null | | |
| grep -E '^[A-Za-z0-9._+-]+$' | sort -u | | |
| grep -vE '^(glibc|libgcc|libstdc\+\+|gcc|zlib-ng-compat)$' | |
| ) | |
| for pkg in "${pkgs[@]}"; do | |
| mapfile -t files < <(rpm -q --licensefiles "$pkg" 2>/dev/null || true) | |
| # Some subpackages leave the licence to a sibling of the same source RPM. | |
| if [ -z "${files[0]:-}" ]; then | |
| srpm=$(rpm -q --qf '%{SOURCERPM}\n' "$pkg") | |
| mapfile -t files < <( | |
| rpm -qa --qf '%{SOURCERPM} %{NAME}\n' | | |
| awk -v s="$srpm" '$1 == s { print $2 }' | | |
| xargs -r rpm -q --licensefiles 2>/dev/null | sort -u | |
| ) | |
| fi | |
| # Others mark it %doc rather than %license, and the image installs no docs. | |
| if [ -z "${files[0]:-}" ]; then | |
| dnf -y reinstall --setopt=tsflags= "$pkg" >/dev/null | |
| mapfile -t files < <(rpm -qd "$pkg" | grep -iE '/(LICEN[CS]E|COPYING|NOTICE)') | |
| fi | |
| for f in "${files[@]}"; do | |
| [ -f "$f" ] || continue | |
| cp "$f" "$project/LICENSE.${pkg}.$(basename "$f")" | |
| done | |
| compgen -G "$project/LICENSE.$pkg.*" >/dev/null || | |
| { echo "no licence file found for $pkg" >&2; exit 1; } | |
| done | |
| ls -1 "$project"/LICENSE.* | sed "s|$project/||" | |
| COLLECT_EOF | |
| - name: Build wheels | |
| uses: pypa/cibuildwheel@1828c10ab37f080699c7b81cea34097c684a7074 # v4.2.0 | |
| with: | |
| output-dir: wheelhouse/ | |
| only: ${{ matrix.python }}-manylinux_riscv64 | |
| env: | |
| CIBW_MANYLINUX_RISCV64_IMAGE: ${{ env.MANYLINUX_RISCV64_IMAGE }} | |
| CIBW_ENVIRONMENT: PIP_EXTRA_INDEX_URL=https://pypi.riseproject.dev/simple/ | |
| # Rocky's zlib-ng ships a ZLIB CMake config pointing at a libz.a it does not | |
| # install, so LegacyFindPackages.cmake's own find_library fallback is used. | |
| CIBW_BEFORE_ALL_LINUX: | | |
| set -ex | |
| dnf -y install --enablerepo=crb boost-devel openssl-devel libcurl-devel \ | |
| zlib-devel snappy-devel libzstd-devel protobuf-devel protobuf-compiler | |
| cpp_version="$(awk '/^pulsar-cpp:/ {print $2}' {project}/dependencies.yaml)" | |
| pybind11_version="$(awk '/^pybind11:/ {print $2}' {project}/dependencies.yaml)" | |
| curl -fsSL "https://archive.apache.org/dist/pulsar/pulsar-client-cpp-$cpp_version/apache-pulsar-client-cpp-$cpp_version.tar.gz" | tar xz -C /tmp | |
| cmake -S "/tmp/apache-pulsar-client-cpp-$cpp_version" -B /tmp/build-cpp \ | |
| -D CMAKE_BUILD_TYPE=Release \ | |
| -D BUILD_TESTS=OFF \ | |
| -D BUILD_PERF_TOOLS=OFF \ | |
| -D BUILD_DYNAMIC_LIB=ON \ | |
| -D BUILD_STATIC_LIB=OFF \ | |
| -D CMAKE_DISABLE_FIND_PACKAGE_zlib=ON | |
| cmake --build /tmp/build-cpp -j "$(nproc)" --target install | |
| echo /usr/local/lib > /etc/ld.so.conf.d/pulsar.conf | |
| ldconfig | |
| curl -fsSL "https://github.com/pybind/pybind11/archive/refs/tags/v$pybind11_version.tar.gz" | tar xz -C /tmp | |
| rm -rf {project}/pybind11 | |
| mv "/tmp/pybind11-$pybind11_version" {project}/pybind11 | |
| bash {project}/collect-licenses.sh {project} | |
| # setup.py ships a build_ext that only copies an already-compiled _pulsar.so, | |
| # so the extension itself is built here, against this job's interpreter. | |
| CIBW_BEFORE_BUILD_LINUX: | | |
| set -ex | |
| rm -rf {package}/build | |
| cmake -S {package} -B {package}/build -D CMAKE_BUILD_TYPE=Release -D Python3_EXECUTABLE="$(which python)" | |
| cmake --build {package}/build -j "$(nproc)" | |
| mv {package}/build/lib_pulsar.so {package}/ | |
| CIBW_TEST_EXTRAS: avro,protobuf | |
| CIBW_TEST_REQUIRES: pytest requests | |
| CIBW_TEST_SOURCES: tests/schema_test.py | |
| # Every test file upstream runs needs a live Pulsar cluster, and the | |
| # apachepulsar/pulsar image its test service starts has no riscv64 build, so | |
| # this runs the suite's one broker-free class plus upstream's own wheel check | |
| # (pkg/test-wheel.sh) widened to exercise libpulsar's client lifecycle and its | |
| # connection-error path. | |
| CIBW_TEST_COMMAND: >- | |
| python -c "import pulsar, logging; c = pulsar.Client('pulsar://localhost:6650', logger=logging.getLogger('t')); c.close()" && | |
| python -c "import pulsar, pytest; c = pulsar.Client('pulsar://127.0.0.1:1', connection_timeout_ms=2000, operation_timeout_seconds=2); pytest.raises(pulsar.PulsarException, c.create_producer, 'topic')" && | |
| pytest -v tests/schema_test.py::ProtobufNativeSchemaTest | |
| - name: Verify the wheel ships the extension, libpulsar and the vendored licences | |
| run: | | |
| python3 - wheelhouse/*.whl <<'EOF' | |
| import sys, zipfile | |
| names = zipfile.ZipFile(sys.argv[1]).namelist() | |
| assert any(n.startswith("_pulsar.") and n.endswith(".so") for n in names), names | |
| assert any("libpulsar" in n and n.endswith(".so") for n in names), names | |
| lic = {n.split("/")[-1] for n in names if ".dist-info/licenses/" in n} | |
| have = {f.split(".", 2)[1] for f in lic if f.startswith("LICENSE.")} | |
| assert {"openssl-libs", "protobuf", "snappy", "libzstd"} <= have, have | |
| print("\n".join(sorted(lic))) | |
| EOF | |
| - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: pulsar-client-${{ env.PULSAR_CLIENT_VERSION }}-${{ matrix.python }}-manylinux_riscv64 | |
| path: wheelhouse/*.whl | |
| if-no-files-found: error | |
| gpl_sources: | |
| needs: [setup] | |
| if: needs.setup.outputs.versions != '[]' | |
| name: Collect GPL sources for pulsar-client ${{ matrix.version }} | |
| runs-on: ubuntu-24.04-riscv | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| version: ${{ fromJSON(needs.setup.outputs.versions) }} | |
| env: | |
| PULSAR_CLIENT_VERSION: ${{ matrix.version }} | |
| steps: | |
| - name: Checkout python-wheels | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| persist-credentials: false | |
| # Every copyleft library auditwheel vendors out of the build image, alongside | |
| # the gcc runtime. | |
| - uses: ./actions/collect-gpl-sources | |
| with: | |
| image: ${{ env.MANYLINUX_RISCV64_IMAGE }} | |
| packages: gcc keyutils-libs libcap libidn2 libssh libunistring libxcrypt libzstd pcre2 systemd-libs | |
| output: gpl-sources.tar | |
| - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: pulsar-client-${{ env.PULSAR_CLIENT_VERSION }}-gpl-sources | |
| path: gpl-sources.tar | |
| if-no-files-found: error | |
| publish: | |
| name: Publish pulsar-client ${{ matrix.version }} | |
| needs: [setup, build_wheels, gpl_sources] | |
| if: needs.setup.outputs.versions != '[]' | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| version: ${{ fromJSON(needs.setup.outputs.versions) }} | |
| permissions: | |
| contents: write | |
| pull-requests: write | |
| uses: $/.github/workflows/_publish-wheel.yml | |
| secrets: | |
| app-private-key: ${{ secrets.RISEPROJECT_APP_PRIVATE_KEY }} | |
| with: | |
| artifact-pattern: pulsar-client-${{ matrix.version }}-*-manylinux_riscv64 | |
| gpl-sources-artifact: pulsar-client-${{ matrix.version }}-gpl-sources | |
| gpl-sources-description: gcc and the copyleft libraries bundled in the wheel |