From 68f9e14c3a19d268db3f93bf1e2d1e933477e2a1 Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Mon, 28 Sep 2026 07:13:54 +0000 Subject: [PATCH 1/4] ydf: add riscv64 build (Bazel, bzlmod, pybind11) Ports google/yggdrasil-decision-forests' pip-installable `ydf` package. Upstream tags no releases; pin the "[YDF] Release PYDF 0.16.1" commit. Bootstraps bazel from source (no riscv64 bazel binary exists) and drives upstream's own build_test_linux.sh/package_linux.sh recipe: bzlmod build of the pybind11 extension (abseil, protobuf, grpc, boringssl, highway, google_cloud_cpp among the statically-linked deps), collect_pip_files.py, `python -m build`, then auditwheel repair. TensorFlow is only an optional runtime extra (`ydf-tf`), not needed here; the Highway-based SIMD kernel already dispatches to a portable scalar target with no forced AVX2. --- .github/workflows/build-ydf.yml | 331 ++++++++++++++++++++++++++++++++ docs/packages/ydf.yaml | 5 + 2 files changed, 336 insertions(+) create mode 100644 .github/workflows/build-ydf.yml create mode 100644 docs/packages/ydf.yaml diff --git a/.github/workflows/build-ydf.yml b/.github/workflows/build-ydf.yml new file mode 100644 index 00000000000..171bc56190b --- /dev/null +++ b/.github/workflows/build-ydf.yml @@ -0,0 +1,331 @@ +# SPDX-FileCopyrightText: 2026 The RISE Project +# SPDX-License-Identifier: MIT +--- +# This workflow is based on: +# https://github.com/google/yggdrasil-decision-forests/blob/main/.github/workflows/python_build_test.yml +# and yggdrasil_decision_forests/port/python/tools/{build_test_linux.sh,package_linux.sh,collect_pip_files.py} +name: Build ydf wheels (riscv64) + +on: + workflow_dispatch: + inputs: + version: + description: 'Version glob to (re)build; empty builds every version of docs/packages/ydf.yaml not released yet' + required: false + default: '' + pull_request: + branches: [main] + paths: + - '.github/workflows/build-ydf.yml' + - 'docs/packages/ydf.yaml' + push: + branches: [main] + paths: + - '.github/workflows/build-ydf.yml' + - 'docs/packages/ydf.yaml' + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read + +env: + # Upstream cuts PYDF releases without git tags; pin the release commit + # ("[YDF] Release PYDF 0.16.1"). Update alongside YDF_VERSION. + YDF_REF: cd115b404c35aa3e15879fc64addb50da0fa8aa7 + MANYLINUX_RISCV64_IMAGE: quay.io/pypa/manylinux_2_39_riscv64 + BAZEL_VERSION: '7.5.0' + RULES_PYTHON_VERSION: '0.33.2' + RULES_JAVA_VERSION: '7.6.5' + PROJECT_RULES_PYTHON_VERSION: '1.6.0' + +jobs: + setup: + uses: $/.github/workflows/_setup.yml + with: + package: ydf + version: ${{ inputs.version }} + + bazel: + needs: [setup] + if: needs.setup.outputs.versions != '[]' + name: Bootstrap bazel (riscv64) + runs-on: ubuntu-24.04-riscv + timeout-minutes: 120 + + steps: + - name: Restore bazel binary + id: cache + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: bazel-bin + key: bazel-${{ env.BAZEL_VERSION }}-manylinux_riscv64 + + - name: Bootstrap bazel ${{ env.BAZEL_VERSION }} + if: steps.cache.outputs.cache-hit != 'true' + run: | + mkdir -p bazel-bin + docker run --rm -i --network=host \ + -v "${GITHUB_WORKSPACE}:/work" \ + -w /work \ + -e BAZEL_VERSION="${BAZEL_VERSION}" \ + -e RULES_PYTHON_VERSION="${RULES_PYTHON_VERSION}" \ + -e RULES_JAVA_VERSION="${RULES_JAVA_VERSION}" \ + "${MANYLINUX_RISCV64_IMAGE}" \ + bash <<'SCRIPT' + set -eux + + dnf install -y --disablerepo=extras --setopt=install_weak_deps=False java-21-openjdk-devel zip unzip + JAVA_HOME="$(dirname "$(dirname "$(readlink -f "$(command -v javac)")")")" + export JAVA_HOME + + # rules_python 0.33.2's PLATFORMS has no riscv64 entry, aborting the bootstrap + # (bazelbuild/bazel#23018). Any linux entry is a safe stand-in: the toolchain it + # names is never selected on a riscv64 host. Fixed in bazel 8.2.0. + mkdir -p /tmp/rules_python + curl -fsSLo /tmp/rules_python.tar.gz "https://github.com/bazel-contrib/rules_python/releases/download/${RULES_PYTHON_VERSION}/rules_python-${RULES_PYTHON_VERSION}.tar.gz" + tar -xzf /tmp/rules_python.tar.gz -C /tmp/rules_python --strip-components=1 + sed -i 's|fail("No platform declared for host OS {} on arch {}".format(os_name, arch))|return "x86_64-unknown-linux-gnu"|' \ + /tmp/rules_python/python/private/toolchains_repo.bzl + + # rules_java 7.x maps riscv64 to a stray-colon include path for jni_md.h. + mkdir -p /tmp/rules_java + curl -fsSLo /tmp/rules_java.tar.gz "https://github.com/bazelbuild/rules_java/releases/download/${RULES_JAVA_VERSION}/rules_java-${RULES_JAVA_VERSION}.tar.gz" + tar -xzf /tmp/rules_java.tar.gz -C /tmp/rules_java + sed -i 's|\[":include/linux"\]|["include/linux"]|g' /tmp/rules_java/toolchains/BUILD + + mkdir -p /tmp/bazel-src + cd /tmp/bazel-src + curl -fsSLo dist.zip "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-dist.zip" + unzip -q dist.zip + + EXTRA_BAZEL_ARGS="--tool_java_runtime_version=local_jdk \ + --override_module=rules_python=/tmp/rules_python \ + --override_module=rules_java=/tmp/rules_java" \ + bash ./compile.sh + install -m 0755 output/bazel /work/bazel-bin/bazel + SCRIPT + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: bazel-${{ env.BAZEL_VERSION }}-riscv64 + path: bazel-bin/bazel + if-no-files-found: error + + build_wheels: + name: Build ydf ${{ matrix.version }} ${{ matrix.tag }}-manylinux_riscv64 + needs: [setup, bazel] + if: needs.setup.outputs.versions != '[]' + runs-on: ubuntu-24.04-riscv + timeout-minutes: 720 + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.setup.outputs.versions) }} + include: + - {tag: cp312, python: '3.12'} + - {tag: cp313, python: '3.13'} + - {tag: cp314, python: '3.14'} + + env: + YDF_VERSION: ${{ matrix.version }} + + steps: + - name: Checkout yggdrasil-decision-forests ${{ env.YDF_VERSION }} + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: google/yggdrasil-decision-forests + ref: ${{ env.YDF_REF }} + path: yggdrasil-decision-forests + fetch-depth: 1 + persist-credentials: false + + - name: Download bazel + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: bazel-${{ env.BAZEL_VERSION }}-riscv64 + path: bazel-bin + + - name: Restore bazel disk cache + id: bazel-cache + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: .bazel-cache + key: ydf-bazel-cache-${{ matrix.tag }}-manylinux_riscv64-${{ github.sha }} + restore-keys: | + ydf-bazel-cache-${{ matrix.tag }}-manylinux_riscv64- + + - name: Build and package the wheel + env: + PYTHON_VERSION: ${{ matrix.python }} + run: | + mkdir -p wheelhouse + set -o pipefail + docker run --rm -i --network=host \ + -v "${GITHUB_WORKSPACE}:/work" \ + -w /work \ + -e PYTHON_VERSION \ + -e PROJECT_RULES_PYTHON_VERSION \ + "${MANYLINUX_RISCV64_IMAGE}" \ + bash <<'SCRIPT' 2>&1 | tee build.log + set -eux + + dnf install -y --disablerepo=extras --setopt=install_weak_deps=False java-21-openjdk-devel zip unzip rsync + JAVA_HOME="$(dirname "$(dirname "$(readlink -f "$(command -v javac)")")")" + export JAVA_HOME + install -m 0755 /work/bazel-bin/bazel /usr/local/bin/bazel + + PYTAG="cp${PYTHON_VERSION/./}" + PYTHON_BIN="/opt/python/${PYTAG}-${PYTAG}/bin/python3" + export PATH="/opt/python/${PYTAG}-${PYTAG}/bin:${PATH}" + ln -sf "${PYTHON_BIN}" /usr/local/bin/python3 + "${PYTHON_BIN}" -m pip install -q -U pip setuptools build auditwheel + + # protobuf/googletest declare pip hubs for CPython versions + # python-build-standalone does not publish for riscv64; rules_python + # aborts resolving them even though the build never uses those hubs. + mkdir -p /tmp/rules_python + curl -fsSLo /tmp/rules_python.tar.gz \ + "https://github.com/bazel-contrib/rules_python/releases/download/${PROJECT_RULES_PYTHON_VERSION}/rules_python-${PROJECT_RULES_PYTHON_VERSION}.tar.gz" + tar -xzf /tmp/rules_python.tar.gz -C /tmp/rules_python --strip-components=1 + python3 - <<'PATCH' + path = "/tmp/rules_python/python/private/pypi/extension.bzl" + source = open(path).read() + old = " if python_name not in available_interpreters:\n fail((" + new = ( + " if python_name not in available_interpreters:\n" + " python_name = sorted(available_interpreters)[0]\n" + " if False:\n" + " fail((" + ) + assert old in source + open(path, "w").write(source.replace(old, new, 1)) + PATCH + + cd /work/yggdrasil-decision-forests/yggdrasil_decision_forests/port/python + + { + echo "build --override_module=rules_python=/tmp/rules_python" + echo "common --curses=no --show_progress_rate_limit=60" + } >> .bazelrc + + "${PYTHON_BIN}" -m pip install -q --only-binary=:all: \ + --extra-index-url https://pypi.riseproject.dev/simple/ \ + -r requirements.txt + + mkdir -p /work/.bazel-cache/disk /work/.bazel-cache/repository + bazel build --config=linux_cpp17 --features=-fully_static_link \ + --disk_cache=/work/.bazel-cache/disk \ + --repository_cache=/work/.bazel-cache/repository \ + --action_env PYTHON_BIN_PATH="${PYTHON_BIN}" -- //ydf/...:all + + "${PYTHON_BIN}" tools/collect_pip_files.py + + OUTPUT_BASE="$(bazel info output_base)" + mkdir -p tmp_package/licenses + for dir in "${OUTPUT_BASE}"/external/*/; do + name="$(basename "${dir}")" + case "${name}" in + rules_*|bazel_skylib*|platforms*|bazel_tools*|googletest*|emsdk*|local_python3*|local_toolchains*|pybind11_bazel*) + continue + ;; + esac + file="$(find "${dir}" -maxdepth 1 -type f \( -iname 'LICENSE*' -o -iname 'COPYING*' \) | head -1)" + [ -n "${file}" ] || continue + clean_name="${name%%~*}" + cp "${file}" "tmp_package/licenses/LICENSE.${clean_name}" + done + + cd tmp_package + cp licenses/LICENSE.* . + rm -rf licenses + "${PYTHON_BIN}" -m build + cd .. + + PACKAGE="$("${PYTHON_BIN}" -c 'import sys; print(f"{sys.version_info.major}{sys.version_info.minor}")')" + cp tmp_package/dist/ydf-*-cp${PACKAGE}-cp${PACKAGE}*-linux_*.whl . + auditwheel repair --plat manylinux_2_39_riscv64 -w /work/wheelhouse ydf-*.whl + SCRIPT + + - name: Save bazel disk cache + if: always() && steps.bazel-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: .bazel-cache + key: ydf-bazel-cache-${{ matrix.tag }}-manylinux_riscv64-${{ github.sha }} + + - name: Upload build log + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ydf-${{ matrix.version }}-${{ matrix.tag }}-build-log + path: build.log + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ydf-${{ matrix.version }}-${{ matrix.tag }}-manylinux_riscv64 + path: wheelhouse/*.whl + if-no-files-found: error + + - name: Test wheel + env: + PYTHON_VERSION: ${{ matrix.python }} + run: | + docker run --rm -i --network=host \ + -v "${GITHUB_WORKSPACE}:/work" \ + -e PYTHON_VERSION \ + "${MANYLINUX_RISCV64_IMAGE}" \ + bash <<'SCRIPT' + set -eux + + PYTAG="cp${PYTHON_VERSION/./}" + PYTHON_BIN="/opt/python/${PYTAG}-${PYTAG}/bin/python3" + "${PYTHON_BIN}" -m pip install -q /work/wheelhouse/*.whl pandas + + cd /tmp + "${PYTHON_BIN}" /work/yggdrasil-decision-forests/yggdrasil_decision_forests/port/python/examples/minimal.py + + "${PYTHON_BIN}" - <<'PY' + import importlib.metadata + + import ydf + print(ydf.__version__) + + dist = importlib.metadata.distribution("ydf") + so = next(f for f in dist.files if str(f).endswith(".so")) + with open(dist.locate_file(so), "rb") as f: + header = f.read(20) + assert header[:4] == b"\x7fELF", header + assert header[18] == 0xF3, header # EM_RISCV + + licenses = { + str(f).rsplit("/", 1)[-1] + for f in dist.files + if ".dist-info/licenses/" in str(f) + } + print(sorted(licenses)) + assert "LICENSE" in licenses + assert "LICENSE.abseil-cpp" in licenses + assert "LICENSE.protobuf" in licenses + assert "LICENSE.pybind11" in licenses + PY + SCRIPT + + publish: + name: Publish ydf ${{ matrix.version }} + needs: [setup, build_wheels] + if: needs.setup.outputs.versions != '[]' + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.setup.outputs.versions) }} + permissions: + contents: write + pull-requests: write + uses: $/.github/workflows/_publish-wheel.yml + secrets: + app-private-key: ${{ secrets.RISEPROJECT_APP_PRIVATE_KEY }} + with: + artifact-pattern: ydf-${{ matrix.version }}-*-manylinux_riscv64 diff --git a/docs/packages/ydf.yaml b/docs/packages/ydf.yaml new file mode 100644 index 00000000000..0dd160f5470 --- /dev/null +++ b/docs/packages/ydf.yaml @@ -0,0 +1,5 @@ +package-name: ydf +source-code: https://github.com/google/yggdrasil-decision-forests +license: Apache-2.0 +versions: +- version: 0.16.1 From 8b91317aa93903fbac2f06a1f8fc99d69639aa8b Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Mon, 28 Sep 2026 10:47:51 +0000 Subject: [PATCH 2/4] ydf: build with clang, not gcc, for the Bazel/highway riscv64 leg com_google_highway's BUILD.bazel unconditionally adds the Clang-only -menable-experimental-extensions copt on riscv64 (gotcha 567); gcc 14.3.1 in the manylinux_2_39_riscv64 image rejects it. Bazel's local toolchain autodetection defaults to gcc unless CC/CXX are exported before the build, so install clang+lld and export them ahead of the bazel invocation. --- .github/workflows/build-ydf.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-ydf.yml b/.github/workflows/build-ydf.yml index 171bc56190b..10d7c35706d 100644 --- a/.github/workflows/build-ydf.yml +++ b/.github/workflows/build-ydf.yml @@ -172,9 +172,16 @@ jobs: bash <<'SCRIPT' 2>&1 | tee build.log set -eux - dnf install -y --disablerepo=extras --setopt=install_weak_deps=False java-21-openjdk-devel zip unzip rsync + dnf install -y --disablerepo=extras --setopt=install_weak_deps=False java-21-openjdk-devel zip unzip rsync clang lld JAVA_HOME="$(dirname "$(dirname "$(readlink -f "$(command -v javac)")")")" export JAVA_HOME + + # com_google_highway's riscv64 copts unconditionally add the Clang-only + # -menable-experimental-extensions flag (CLAUDE.md gotcha 567); gcc rejects it. + CC=clang + CXX=clang++ + export CC CXX + install -m 0755 /work/bazel-bin/bazel /usr/local/bin/bazel PYTAG="cp${PYTHON_VERSION/./}" From 2745a974230c0f360dd1c15fd6b94a1b864906b5 Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Mon, 28 Sep 2026 21:44:10 +0000 Subject: [PATCH 3/4] ydf: drop cp314 from the riscv64 matrix, upstream doesn't ship it cp314 built cleanly after ~6.5h (gotcha 567's clang fix held) and then segfaulted in pybind11_protobuf's proto_caster_load_impl during upstream's own examples/minimal.py smoke test. ydf 0.16.1 ships no cp314 wheel on any platform, its own setup.py classifiers stop at 3.13, and upstream's own CI matrix is a single '3.12' entry marked "Currently unused" -- there is no upstream cp314 support to chase this against (gotcha 614). --- .github/workflows/build-ydf.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-ydf.yml b/.github/workflows/build-ydf.yml index 10d7c35706d..1e0f9c4a062 100644 --- a/.github/workflows/build-ydf.yml +++ b/.github/workflows/build-ydf.yml @@ -124,10 +124,12 @@ jobs: fail-fast: false matrix: version: ${{ fromJSON(needs.setup.outputs.versions) }} + # ydf 0.16.1 ships no cp314 wheel on any platform and its own setup.py classifiers + # and CI matrix stop at 3.13 (CLAUDE.md gotcha 614); cp314 segfaults in + # pybind11_protobuf, not this port. include: - {tag: cp312, python: '3.12'} - {tag: cp313, python: '3.13'} - - {tag: cp314, python: '3.14'} env: YDF_VERSION: ${{ matrix.version }} From c692a854f7ac5669f7f15016857b25cee99bbd11 Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Tue, 29 Sep 2026 04:50:49 +0000 Subject: [PATCH 4/4] ydf: fix license smoke-test to check for pybind11_protobuf, not pybind11 The bundled license for the pybind11 integration is collected from the bazel external repo named pybind11_protobuf (pybind11_bazel is a separate, header-only shim that is intentionally skipped), so the wheel never ships a LICENSE.pybind11 file. The test smoke script was still asserting the old name, causing a bare AssertionError after an otherwise fully successful build, train, eval and inference run. --- .github/workflows/build-ydf.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-ydf.yml b/.github/workflows/build-ydf.yml index 1e0f9c4a062..17e7f834cff 100644 --- a/.github/workflows/build-ydf.yml +++ b/.github/workflows/build-ydf.yml @@ -318,7 +318,7 @@ jobs: assert "LICENSE" in licenses assert "LICENSE.abseil-cpp" in licenses assert "LICENSE.protobuf" in licenses - assert "LICENSE.pybind11" in licenses + assert "LICENSE.pybind11_protobuf" in licenses PY SCRIPT