From 62c2cd9b9f81e3fe0ac19f49604a488b81573986 Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Sun, 20 Sep 2026 21:01:34 +0000 Subject: [PATCH] pulsar-client: Add version 3.13.0 Builds the Apache Pulsar C++ client's pybind11 bindings for riscv64. Upstream's manylinux job compiles the apache-pulsar-client-cpp release tarball pinned in dependencies.yaml with vcpkg supplying that library's dependencies. vcpkg has no tested riscv64 triplet, so the same tarball is built against the manylinux image's boost/openssl/protobuf/curl/snappy/zstd, which is also the path distro packagers take (LegacyFindPackages.cmake). Rocky's zlib-ng ships a ZLIB CMake config referencing a libz.a it does not install, so find_package(zlib) is disabled in favour of the project's own find_library fallback. --- .github/workflows/build-pulsar-client.yml | 249 ++++++++++++++++++++++ docs/packages/pulsar-client.yaml | 5 + 2 files changed, 254 insertions(+) create mode 100644 .github/workflows/build-pulsar-client.yml create mode 100644 docs/packages/pulsar-client.yaml diff --git a/.github/workflows/build-pulsar-client.yml b/.github/workflows/build-pulsar-client.yml new file mode 100644 index 00000000000..0579a1c2f28 --- /dev/null +++ b/.github/workflows/build-pulsar-client.yml @@ -0,0 +1,249 @@ +# SPDX-FileCopyrightText: 2026 The RISE Project +# SPDX-License-Identifier: MIT +--- +# Based on upstream's own manylinux wheel recipe, narrowed to riscv64: +# https://github.com/apache/pulsar-client-python/blob/v3.13.0/.github/workflows/ci-build-release-wheels.yaml +# https://github.com/apache/pulsar-client-python/blob/v3.13.0/pkg/build-wheel-inside-docker.sh +# Upstream compiles the Pulsar C++ client from the apache-pulsar-client-cpp release +# tarball its dependencies.yaml pins, with vcpkg supplying that library's own +# dependencies; vcpkg has no tested riscv64 triplet, so the same tarball is built +# against the manylinux image's boost/openssl/protobuf/curl/snappy/zstd instead. +name: Build pulsar-client wheels (riscv64) + +on: + workflow_dispatch: + inputs: + version: + description: 'Version glob to (re)build; empty builds every version of docs/packages/pulsar-client.yaml not released yet' + required: false + default: '' + pull_request: + branches: [main] + paths: + - '.github/workflows/build-pulsar-client.yml' + - 'docs/packages/pulsar-client.yaml' + push: + branches: [main] + paths: + - '.github/workflows/build-pulsar-client.yml' + - 'docs/packages/pulsar-client.yaml' + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read # to fetch code (actions/checkout) + +env: + MANYLINUX_RISCV64_IMAGE: quay.io/pypa/manylinux_2_39_riscv64 + +jobs: + setup: + uses: $/.github/workflows/_setup.yml + with: + package: pulsar-client + version: ${{ inputs.version }} + + build_wheels: + needs: [setup] + if: needs.setup.outputs.versions != '[]' + name: Build pulsar-client ${{ matrix.version }} ${{ matrix.python }}-manylinux_riscv64 + runs-on: ubuntu-24.04-riscv + timeout-minutes: 360 + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.setup.outputs.versions) }} + # No cp314t: upstream publishes no free-threaded wheels. + python: ["cp312", "cp313", "cp314"] + + env: + PULSAR_CLIENT_VERSION: ${{ matrix.version }} + + steps: + - name: Checkout pulsar-client-python v${{ env.PULSAR_CLIENT_VERSION }} + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: apache/pulsar-client-python + ref: v${{ env.PULSAR_CLIENT_VERSION }} + persist-credentials: false + + - name: Stage the licence-collection script + run: | + cat > collect-licenses.sh <<'COLLECT_EOF' + #!/bin/bash + # SPDX-FileCopyrightText: 2026 The RISE Project + # SPDX-License-Identifier: MIT + # + # Stage, at the project root, the licence of every shared library auditwheel + # vendors alongside libpulsar. setuptools' default LICENSE* glob copies them + # into the wheel. + set -euo pipefail + + project="${1:?usage: collect-licenses.sh }" + + # ldd is transitive, so the one linked library covers its whole closure. + # glibc, the gcc runtime and zlib are on auditwheel's manylinux allowlist and + # are never vendored into the wheel. + mapfile -t libs < <(ldd /usr/local/lib/libpulsar.so | tr ' ' '\n' | grep '^/' | sort -u) + + # `rpm -qf` reports unowned files on stdout, so keep only bare package names. + mapfile -t pkgs < <( + rpm -qf --qf '%{NAME}\n' "${libs[@]}" 2>/dev/null | + grep -E '^[A-Za-z0-9._+-]+$' | sort -u | + grep -vE '^(glibc|libgcc|libstdc\+\+|gcc|zlib-ng-compat)$' + ) + + for pkg in "${pkgs[@]}"; do + mapfile -t files < <(rpm -q --licensefiles "$pkg" 2>/dev/null || true) + + # Some subpackages leave the licence to a sibling of the same source RPM. + if [ -z "${files[0]:-}" ]; then + srpm=$(rpm -q --qf '%{SOURCERPM}\n' "$pkg") + mapfile -t files < <( + rpm -qa --qf '%{SOURCERPM} %{NAME}\n' | + awk -v s="$srpm" '$1 == s { print $2 }' | + xargs -r rpm -q --licensefiles 2>/dev/null | sort -u + ) + fi + + # Others mark it %doc rather than %license, and the image installs no docs. + if [ -z "${files[0]:-}" ]; then + dnf -y reinstall --setopt=tsflags= "$pkg" >/dev/null + mapfile -t files < <(rpm -qd "$pkg" | grep -iE '/(LICEN[CS]E|COPYING|NOTICE)') + fi + + for f in "${files[@]}"; do + [ -f "$f" ] || continue + cp "$f" "$project/LICENSE.${pkg}.$(basename "$f")" + done + compgen -G "$project/LICENSE.$pkg.*" >/dev/null || + { echo "no licence file found for $pkg" >&2; exit 1; } + done + + ls -1 "$project"/LICENSE.* | sed "s|$project/||" + COLLECT_EOF + + - name: Build wheels + uses: pypa/cibuildwheel@1828c10ab37f080699c7b81cea34097c684a7074 # v4.2.0 + with: + output-dir: wheelhouse/ + only: ${{ matrix.python }}-manylinux_riscv64 + env: + CIBW_MANYLINUX_RISCV64_IMAGE: ${{ env.MANYLINUX_RISCV64_IMAGE }} + CIBW_ENVIRONMENT: PIP_EXTRA_INDEX_URL=https://pypi.riseproject.dev/simple/ + # Rocky's zlib-ng ships a ZLIB CMake config pointing at a libz.a it does not + # install, so LegacyFindPackages.cmake's own find_library fallback is used. + CIBW_BEFORE_ALL_LINUX: | + set -ex + dnf -y install --enablerepo=crb boost-devel openssl-devel libcurl-devel \ + zlib-devel snappy-devel libzstd-devel protobuf-devel protobuf-compiler + cpp_version="$(awk '/^pulsar-cpp:/ {print $2}' {project}/dependencies.yaml)" + pybind11_version="$(awk '/^pybind11:/ {print $2}' {project}/dependencies.yaml)" + curl -fsSL "https://archive.apache.org/dist/pulsar/pulsar-client-cpp-$cpp_version/apache-pulsar-client-cpp-$cpp_version.tar.gz" | tar xz -C /tmp + cmake -S "/tmp/apache-pulsar-client-cpp-$cpp_version" -B /tmp/build-cpp \ + -D CMAKE_BUILD_TYPE=Release \ + -D BUILD_TESTS=OFF \ + -D BUILD_PERF_TOOLS=OFF \ + -D BUILD_DYNAMIC_LIB=ON \ + -D BUILD_STATIC_LIB=OFF \ + -D CMAKE_DISABLE_FIND_PACKAGE_zlib=ON + cmake --build /tmp/build-cpp -j "$(nproc)" --target install + echo /usr/local/lib > /etc/ld.so.conf.d/pulsar.conf + ldconfig + curl -fsSL "https://github.com/pybind/pybind11/archive/refs/tags/v$pybind11_version.tar.gz" | tar xz -C /tmp + rm -rf {project}/pybind11 + mv "/tmp/pybind11-$pybind11_version" {project}/pybind11 + bash {project}/collect-licenses.sh {project} + # setup.py ships a build_ext that only copies an already-compiled _pulsar.so, + # so the extension itself is built here, against this job's interpreter. + CIBW_BEFORE_BUILD_LINUX: | + set -ex + rm -rf {package}/build + cmake -S {package} -B {package}/build -D CMAKE_BUILD_TYPE=Release -D Python3_EXECUTABLE="$(which python)" + cmake --build {package}/build -j "$(nproc)" + mv {package}/build/lib_pulsar.so {package}/ + CIBW_TEST_EXTRAS: avro,protobuf + CIBW_TEST_REQUIRES: pytest requests + CIBW_TEST_SOURCES: tests/schema_test.py + # Every test file upstream runs needs a live Pulsar cluster, and the + # apachepulsar/pulsar image its test service starts has no riscv64 build, so + # this runs the suite's one broker-free class plus upstream's own wheel check + # (pkg/test-wheel.sh) widened to exercise libpulsar's client lifecycle and its + # connection-error path. + CIBW_TEST_COMMAND: >- + python -c "import pulsar, logging; c = pulsar.Client('pulsar://localhost:6650', logger=logging.getLogger('t')); c.close()" && + python -c "import pulsar, pytest; c = pulsar.Client('pulsar://127.0.0.1:1', connection_timeout_ms=2000, operation_timeout_seconds=2); pytest.raises(pulsar.PulsarException, c.create_producer, 'topic')" && + pytest -v tests/schema_test.py::ProtobufNativeSchemaTest + + - name: Verify the wheel ships the extension, libpulsar and the vendored licences + run: | + python3 - wheelhouse/*.whl <<'EOF' + import sys, zipfile + names = zipfile.ZipFile(sys.argv[1]).namelist() + assert any(n.startswith("_pulsar.") and n.endswith(".so") for n in names), names + assert any("libpulsar" in n and n.endswith(".so") for n in names), names + lic = {n.split("/")[-1] for n in names if ".dist-info/licenses/" in n} + have = {f.split(".", 2)[1] for f in lic if f.startswith("LICENSE.")} + assert {"openssl-libs", "protobuf", "snappy", "libzstd"} <= have, have + print("\n".join(sorted(lic))) + EOF + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: pulsar-client-${{ env.PULSAR_CLIENT_VERSION }}-${{ matrix.python }}-manylinux_riscv64 + path: wheelhouse/*.whl + if-no-files-found: error + + gpl_sources: + needs: [setup] + if: needs.setup.outputs.versions != '[]' + name: Collect GPL sources for pulsar-client ${{ matrix.version }} + runs-on: ubuntu-24.04-riscv + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.setup.outputs.versions) }} + + env: + PULSAR_CLIENT_VERSION: ${{ matrix.version }} + + steps: + - name: Checkout python-wheels + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + # Every copyleft library auditwheel vendors out of the build image, alongside + # the gcc runtime. + - uses: ./actions/collect-gpl-sources + with: + image: ${{ env.MANYLINUX_RISCV64_IMAGE }} + packages: gcc keyutils-libs libcap libidn2 libssh libunistring libxcrypt libzstd pcre2 systemd-libs + output: gpl-sources.tar + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: pulsar-client-${{ env.PULSAR_CLIENT_VERSION }}-gpl-sources + path: gpl-sources.tar + if-no-files-found: error + + publish: + name: Publish pulsar-client ${{ matrix.version }} + needs: [setup, build_wheels, gpl_sources] + if: needs.setup.outputs.versions != '[]' + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.setup.outputs.versions) }} + permissions: + contents: write + pull-requests: write + uses: $/.github/workflows/_publish-wheel.yml + secrets: + app-private-key: ${{ secrets.RISEPROJECT_APP_PRIVATE_KEY }} + with: + artifact-pattern: pulsar-client-${{ matrix.version }}-*-manylinux_riscv64 + gpl-sources-artifact: pulsar-client-${{ matrix.version }}-gpl-sources + gpl-sources-description: gcc and the copyleft libraries bundled in the wheel diff --git a/docs/packages/pulsar-client.yaml b/docs/packages/pulsar-client.yaml new file mode 100644 index 00000000000..d9480db2163 --- /dev/null +++ b/docs/packages/pulsar-client.yaml @@ -0,0 +1,5 @@ +package-name: pulsar-client +source-code: https://github.com/apache/pulsar-client-python +license: Apache-2.0 +versions: +- version: 3.13.0