Repository navigation
runai-model-streamer-gcs: Add version 0.16.1 (#2406) #2
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # SPDX-FileCopyrightText: 2026 The RISE Project | |
| # SPDX-License-Identifier: MIT | |
| # | |
| # This workflow is based on: https://github.com/run-ai/runai-model-streamer/blob/master/Makefile | |
| # and https://github.com/run-ai/runai-model-streamer/blob/master/tests/Makefile | |
| --- | |
| name: Build runai-model-streamer-gcs wheels (riscv64) | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| version: | |
| description: 'Version glob to (re)build; empty builds every version of docs/packages/runai-model-streamer-gcs.yaml not released yet' | |
| required: false | |
| default: '' | |
| pull_request: | |
| branches: [main] | |
| paths: | |
| - '.github/workflows/build-runai-model-streamer-gcs.yml' | |
| - 'docs/packages/runai-model-streamer-gcs.yaml' | |
| - 'patches/runai-model-streamer-gcs/**' | |
| push: | |
| branches: [main] | |
| paths: | |
| - '.github/workflows/build-runai-model-streamer-gcs.yml' | |
| - 'docs/packages/runai-model-streamer-gcs.yaml' | |
| - 'patches/runai-model-streamer-gcs/**' | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} | |
| cancel-in-progress: true | |
| permissions: | |
| contents: read | |
| env: | |
| MANYLINUX_RISCV64_IMAGE: quay.io/pypa/manylinux_2_39_riscv64 | |
| BAZEL_VERSION: '7.6.1' | |
| RULES_PYTHON_VERSION: '0.33.2' | |
| RULES_JAVA_VERSION: '7.6.5' | |
| # Pinned by upstream's .devcontainer/Dockerfile, which downloads the amd64/arm64 release binary. | |
| FAKE_GCS_SERVER_VERSION: 'v1.52.2' | |
| jobs: | |
| setup: | |
| uses: $/.github/workflows/_setup.yml | |
| with: | |
| package: runai-model-streamer-gcs | |
| version: ${{ inputs.version }} | |
| bazel: | |
| needs: [setup] | |
| if: needs.setup.outputs.versions != '[]' | |
| name: Bootstrap bazel (riscv64) | |
| runs-on: ubuntu-24.04-riscv | |
| timeout-minutes: 120 | |
| steps: | |
| - name: Restore bazel binary | |
| id: cache | |
| uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 | |
| with: | |
| path: bazel-bin | |
| key: bazel-${{ env.BAZEL_VERSION }}-manylinux_riscv64 | |
| - name: Bootstrap bazel ${{ env.BAZEL_VERSION }} | |
| if: steps.cache.outputs.cache-hit != 'true' | |
| run: | | |
| mkdir -p bazel-bin | |
| docker run --rm -i --network=host \ | |
| -v "${GITHUB_WORKSPACE}:/work" \ | |
| -w /work \ | |
| -e BAZEL_VERSION="${BAZEL_VERSION}" \ | |
| -e RULES_PYTHON_VERSION="${RULES_PYTHON_VERSION}" \ | |
| -e RULES_JAVA_VERSION="${RULES_JAVA_VERSION}" \ | |
| "${MANYLINUX_RISCV64_IMAGE}" \ | |
| bash <<'SCRIPT' | |
| set -eux | |
| dnf install -y --setopt=install_weak_deps=False java-21-openjdk-devel zip unzip | |
| JAVA_HOME="$(dirname "$(dirname "$(readlink -f "$(command -v javac)")")")" | |
| export JAVA_HOME | |
| # rules_python 0.33.2 has no riscv64 PLATFORMS entry (bazelbuild/bazel#23018). | |
| mkdir -p /tmp/rules_python | |
| curl -fsSLo /tmp/rules_python.tar.gz "https://github.com/bazel-contrib/rules_python/releases/download/${RULES_PYTHON_VERSION}/rules_python-${RULES_PYTHON_VERSION}.tar.gz" | |
| tar -xzf /tmp/rules_python.tar.gz -C /tmp/rules_python --strip-components=1 | |
| sed -i 's|fail("No platform declared for host OS {} on arch {}".format(os_name, arch))|return "x86_64-unknown-linux-gnu"|' \ | |
| /tmp/rules_python/python/private/toolchains_repo.bzl | |
| # rules_java 7.x maps riscv64 to a stray-colon include path for jni_md.h. | |
| mkdir -p /tmp/rules_java | |
| curl -fsSLo /tmp/rules_java.tar.gz "https://github.com/bazelbuild/rules_java/releases/download/${RULES_JAVA_VERSION}/rules_java-${RULES_JAVA_VERSION}.tar.gz" | |
| tar -xzf /tmp/rules_java.tar.gz -C /tmp/rules_java | |
| sed -i 's|\[":include/linux"\]|["include/linux"]|g' /tmp/rules_java/toolchains/BUILD | |
| mkdir -p /tmp/bazel-src | |
| cd /tmp/bazel-src | |
| curl -fsSLo dist.zip "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-dist.zip" | |
| unzip -q dist.zip | |
| EXTRA_BAZEL_ARGS="--tool_java_runtime_version=local_jdk \ | |
| --override_module=rules_python=/tmp/rules_python \ | |
| --override_module=rules_java=/tmp/rules_java" \ | |
| bash ./compile.sh | |
| install -m 0755 output/bazel /work/bazel-bin/bazel | |
| SCRIPT | |
| - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: bazel-${{ env.BAZEL_VERSION }}-riscv64 | |
| path: bazel-bin/bazel | |
| if-no-files-found: error | |
| fake_gcs_server: | |
| needs: [setup] | |
| if: needs.setup.outputs.versions != '[]' | |
| name: Build fake-gcs-server (riscv64) | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| steps: | |
| - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 | |
| with: | |
| go-version: 'stable' | |
| cache: false | |
| - name: Build fake-gcs-server ${{ env.FAKE_GCS_SERVER_VERSION }} | |
| env: | |
| CGO_ENABLED: '0' | |
| GOOS: linux | |
| GOARCH: riscv64 | |
| run: | | |
| go install "github.com/fsouza/fake-gcs-server@${FAKE_GCS_SERVER_VERSION}" | |
| mkdir -p fake-gcs-server | |
| cp "$(go env GOPATH)/bin/linux_riscv64/fake-gcs-server" fake-gcs-server/fake-gcs-server | |
| - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: fake-gcs-server-riscv64 | |
| path: fake-gcs-server/fake-gcs-server | |
| if-no-files-found: error | |
| build_wheel: | |
| name: Build runai-model-streamer-gcs ${{ matrix.version }} py3-none-manylinux_riscv64 | |
| needs: [setup, bazel] | |
| if: needs.setup.outputs.versions != '[]' | |
| runs-on: ubuntu-24.04-riscv | |
| timeout-minutes: 720 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| version: ${{ fromJSON(needs.setup.outputs.versions) }} | |
| env: | |
| RUNAI_MODEL_STREAMER_VERSION: ${{ matrix.version }} | |
| steps: | |
| - name: Checkout runai-model-streamer ${{ env.RUNAI_MODEL_STREAMER_VERSION }} | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| repository: run-ai/runai-model-streamer | |
| ref: ${{ env.RUNAI_MODEL_STREAMER_VERSION }} | |
| path: runai-model-streamer | |
| persist-credentials: false | |
| - name: Checkout python-wheels | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| path: python-wheels | |
| persist-credentials: false | |
| - name: Patch runai-model-streamer source | |
| working-directory: runai-model-streamer | |
| run: git apply -v ../python-wheels/patches/runai-model-streamer-gcs/${{ env.RUNAI_MODEL_STREAMER_VERSION }}/*.patch | |
| - name: Download bazel | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: bazel-${{ env.BAZEL_VERSION }}-riscv64 | |
| path: bazel-bin | |
| - name: Build the GCS streamer library and package the wheel | |
| run: | | |
| mkdir -p dist | |
| docker run --rm -i --network=host \ | |
| -v "${GITHUB_WORKSPACE}:/work" \ | |
| -w /work/runai-model-streamer \ | |
| -e PACKAGE_VERSION="${RUNAI_MODEL_STREAMER_VERSION}" \ | |
| "${MANYLINUX_RISCV64_IMAGE}" \ | |
| bash <<'SCRIPT' | |
| set -eux | |
| # cpp/toolchain links -l:libstdc++.a, which on this image is its own CRB package. | |
| dnf install -y --setopt=install_weak_deps=False java-21-openjdk-devel libstdc++-static | |
| JAVA_HOME="$(dirname "$(dirname "$(readlink -f "$(command -v javac)")")")" | |
| export JAVA_HOME | |
| install -m 0755 /work/bazel-bin/bazel /usr/local/bin/bazel | |
| git config --global --add safe.directory '*' | |
| # cpp/toolchain/rules.bzl expects the Debian cross-toolchain names; this image's gcc is native. | |
| for tool in gcc g++ ar ld cpp gcov nm objdump strip; do | |
| native="$(command -v "$tool" || true)" | |
| [ -n "$native" ] && ln -sf "$native" "/usr/bin/riscv64-linux-gnu-$tool" | |
| done | |
| cd cpp | |
| # A multi-hour build's default curses progress output is large enough that | |
| # GitHub drops the job log, taking the failure with it. | |
| bazel build gcs:libstreamergcs.so --config=riscv64 --curses=no --show_progress_rate_limit=60 | |
| # Stage the licence of every external repository the library is built from. | |
| licenses=/work/runai-model-streamer/py/runai_model_streamer_gcs | |
| external="$(bazel info output_base)/external" | |
| bazel cquery 'deps(gcs:libstreamergcs.so)' --config=riscv64 --output=label > /tmp/deps.txt | |
| sed -n 's|^@@\{0,1\}\([^/]*\)//.*|\1|p' /tmp/deps.txt | sort -u \ | |
| | grep -Ev '^(bazel_tools|bazel_skylib|platforms|toolchain|local_config_.*|local_jdk|remotejdk.*|remote_.*|rules_.*|build_bazel_.*|io_bazel_.*)$' \ | |
| | while read -r repo; do | |
| for f in "$external/$repo"/LICENSE* "$external/$repo"/COPYING* "$external/$repo"/NOTICE*; do | |
| [ -f "$f" ] || continue | |
| case "$(basename "$f")" in | |
| NOTICE*) dest="LICENSE.$repo.NOTICE" ;; | |
| *) dest="LICENSE.$repo"; [ -e "$licenses/$dest" ] && dest="LICENSE.$repo.$(basename "$f")" ;; | |
| esac | |
| cp "$f" "$licenses/$dest" | |
| done | |
| done | |
| ls "$licenses" | |
| cd ../py/runai_model_streamer_gcs | |
| /opt/python/cp312-cp312/bin/pip install -q setuptools wheel | |
| /opt/python/cp312-cp312/bin/python setup.py bdist_wheel --plat-name manylinux_2_39_riscv64 | |
| cp dist/*.whl /work/dist/ | |
| SCRIPT | |
| - name: Verify the wheel ships a riscv64 shared object and its licences | |
| run: | | |
| python3 - dist/*.whl <<'EOF' | |
| import sys, zipfile | |
| with zipfile.ZipFile(sys.argv[1]) as zf: | |
| names = zf.namelist() | |
| so = next(n for n in names if n.endswith("libstreamer/lib/libstreamergcs.so")) | |
| header = zf.read(so)[:20] | |
| assert header[:4] == b"\x7fELF", header | |
| assert header[18] == 0xF3, header # EM_RISCV | |
| licenses = {n.rsplit("/", 1)[-1] for n in names if ".dist-info/licenses/" in n} | |
| print(sorted(licenses)) | |
| for dep in ("google_cloud_cpp", "com_google_absl", "com_google_protobuf", "com_github_grpc_grpc", | |
| "boringssl", "com_github_curl_curl", "zlib", "com_github_cares_cares", | |
| "com_googlesource_code_re2", "com_github_google_crc32c", "com_github_nlohmann_json"): | |
| assert f"LICENSE.{dep}" in licenses, dep | |
| EOF | |
| - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: runai-model-streamer-gcs-${{ matrix.version }}-py3-none-manylinux_riscv64 | |
| path: dist/*.whl | |
| if-no-files-found: error | |
| test_wheel: | |
| name: Test runai-model-streamer-gcs ${{ matrix.version }} on Python ${{ matrix.python-version }} | |
| needs: [setup, build_wheel, fake_gcs_server] | |
| if: needs.setup.outputs.versions != '[]' | |
| runs-on: ubuntu-24.04-riscv | |
| timeout-minutes: 30 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| version: ${{ fromJSON(needs.setup.outputs.versions) }} | |
| python-version: ['3.12', '3.13', '3.14'] | |
| env: | |
| RUNAI_MODEL_STREAMER_VERSION: ${{ matrix.version }} | |
| steps: | |
| - name: Checkout runai-model-streamer ${{ env.RUNAI_MODEL_STREAMER_VERSION }} | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| repository: run-ai/runai-model-streamer | |
| ref: ${{ env.RUNAI_MODEL_STREAMER_VERSION }} | |
| path: runai-model-streamer | |
| persist-credentials: false | |
| - name: Download wheel | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: runai-model-streamer-gcs-${{ env.RUNAI_MODEL_STREAMER_VERSION }}-py3-none-manylinux_riscv64 | |
| path: wheelhouse | |
| - name: Download fake-gcs-server | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: fake-gcs-server-riscv64 | |
| path: fake-gcs-server | |
| - name: Install Python | |
| uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| activate-environment: true | |
| enable-cache: false | |
| - name: Install the wheel and upstream's test dependencies | |
| env: | |
| UV_EXTRA_INDEX_URL: https://pypi.riseproject.dev/simple/ | |
| UV_INDEX_STRATEGY: unsafe-best-match | |
| # requirements.dev's numpy==1.24.4 has no riscv64 wheel and no longer builds on Python >=3.12. | |
| run: | | |
| echo "numpy>=1.26,<3" > "$RUNNER_TEMP/numpy-override.txt" | |
| uv pip install --override "$RUNNER_TEMP/numpy-override.txt" \ | |
| -r runai-model-streamer/py/runai_model_streamer/requirements.dev \ | |
| pytest \ | |
| "runai-model-streamer[gcs]==${RUNAI_MODEL_STREAMER_VERSION}" \ | |
| wheelhouse/*.whl | |
| - name: Run the GCS package's unit tests against the installed wheel | |
| run: | | |
| site="$(python -c 'import runai_model_streamer_gcs, os; print(os.path.dirname(os.path.dirname(runai_model_streamer_gcs.__file__)))')" | |
| cd "$site" | |
| python -m unittest discover -b runai_model_streamer_gcs | |
| - name: Run upstream's GCS tests against fake-gcs-server | |
| working-directory: runai-model-streamer/tests | |
| env: | |
| GOOGLE_CLOUD_PROJECT: fake-project | |
| GCS_BUCKET: test-bucket | |
| API_ENDPOINT_OVERRIDE: http://localhost:8000 | |
| CLOUD_STORAGE_EMULATOR_ENDPOINT: http://localhost:8000 | |
| RUNAI_STREAMER_GCS_USE_ANONYMOUS_CREDENTIALS: 'True' | |
| run: | | |
| chmod +x "$GITHUB_WORKSPACE/fake-gcs-server/fake-gcs-server" | |
| root="$RUNNER_TEMP/fake-gcs-server" | |
| mkdir -p "$root/data/test-bucket" | |
| "$GITHUB_WORKSPACE/fake-gcs-server/fake-gcs-server" -scheme http -port 8000 -data "$root/data" -filesystem-root "$root" & | |
| trap 'kill $!' EXIT | |
| pytest -v gcs/ | |
| gpl_sources: | |
| needs: [setup] | |
| if: needs.setup.outputs.versions != '[]' | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| version: ${{ fromJSON(needs.setup.outputs.versions) }} | |
| name: Collect GPL sources for runai-model-streamer-gcs ${{ matrix.version }} | |
| runs-on: ubuntu-24.04-riscv | |
| env: | |
| RUNAI_MODEL_STREAMER_VERSION: ${{ matrix.version }} | |
| steps: | |
| - name: Checkout python-wheels | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| persist-credentials: false | |
| - uses: ./actions/collect-gpl-sources | |
| with: | |
| image: ${{ env.MANYLINUX_RISCV64_IMAGE }} | |
| packages: gcc | |
| output: gpl-sources.tar | |
| - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: runai-model-streamer-gcs-${{ env.RUNAI_MODEL_STREAMER_VERSION }}-gpl-sources | |
| path: gpl-sources.tar | |
| if-no-files-found: error | |
| publish: | |
| name: Publish runai-model-streamer-gcs ${{ matrix.version }} | |
| needs: [setup, build_wheel, test_wheel, gpl_sources] | |
| if: needs.setup.outputs.versions != '[]' | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| version: ${{ fromJSON(needs.setup.outputs.versions) }} | |
| permissions: | |
| contents: write | |
| pull-requests: write | |
| uses: $/.github/workflows/_publish-wheel.yml | |
| secrets: | |
| app-private-key: ${{ secrets.RISEPROJECT_APP_PRIVATE_KEY }} | |
| with: | |
| artifact-pattern: runai-model-streamer-gcs-${{ matrix.version }}-py3-none-manylinux_riscv64 | |
| gpl-sources-artifact: runai-model-streamer-gcs-${{ matrix.version }}-gpl-sources | |
| gpl-sources-description: gcc |