build-pytorch-tokenizers.yml

# SPDX-FileCopyrightText: 2026 The RISE Project
# SPDX-License-Identifier: MIT
---
# This workflow is based on:
# https://github.com/meta-pytorch/tokenizers/blob/v1.4.1/.github/workflows/build-wheels-linux.yml
name: Build pytorch-tokenizers wheels (riscv64)

on:
  workflow_dispatch:
    inputs:
      version:
        description: 'Version glob to (re)build; empty builds every version of docs/packages/pytorch-tokenizers.yaml not released yet'
        required: false
        default: ''
  pull_request:
    branches: [main]
    paths:
      - '.github/workflows/build-pytorch-tokenizers.yml'
      - 'docs/packages/pytorch-tokenizers.yaml'
  push:
    branches: [main]
    paths:
      - '.github/workflows/build-pytorch-tokenizers.yml'
      - 'docs/packages/pytorch-tokenizers.yaml'

concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

permissions:
  contents: read

env:
  MANYLINUX_RISCV64_IMAGE: quay.io/pypa/manylinux_2_39_riscv64
  # The tag CMakeLists.txt's FetchContent_Declare(pybind11) pins.
  PYBIND11_REF: v2.13.6

jobs:
  setup:
    uses: $/.github/workflows/_setup.yml
    with:
      package: pytorch-tokenizers
      version: ${{ inputs.version }}

  build_wheels:
    needs: [setup]
    if: needs.setup.outputs.versions != '[]'
    strategy:
      fail-fast: false
      matrix:
        version: ${{ fromJSON(needs.setup.outputs.versions) }}
        python: ["cp312", "cp313", "cp314"]
    name: Build pytorch-tokenizers ${{ matrix.version }} ${{ matrix.python }}-manylinux_riscv64
    runs-on: ubuntu-24.04-riscv
    timeout-minutes: 360

    env:
      PYTORCH_TOKENIZERS_VERSION: ${{ matrix.version }}

    steps:
      - name: Checkout tokenizers v${{ env.PYTORCH_TOKENIZERS_VERSION }}
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
        with:
          repository: meta-pytorch/tokenizers
          ref: v${{ env.PYTORCH_TOKENIZERS_VERSION }}
          submodules: recursive
          persist-credentials: false

      # Every tree below is compiled into the extension, and setuptools' default licence
      # glob only reaches the project root.
      - name: Stage the vendored licences
        run: |
          set -euo pipefail
          cp third-party/abseil-cpp/LICENSE LICENSE.abseil-cpp
          cp third-party/re2/LICENSE LICENSE.re2
          cp third-party/sentencepiece/LICENSE LICENSE.sentencepiece
          cp third-party/sentencepiece/third_party/darts_clone/LICENSE LICENSE.darts-clone
          cp third-party/sentencepiece/third_party/esaxx/LICENSE LICENSE.esaxx
          cp third-party/sentencepiece/third_party/protobuf-lite/LICENSE LICENSE.protobuf-lite
          cp third-party/pcre2/LICENCE.md LICENSE.pcre2
          cp third-party/json/LICENSE.MIT LICENSE.nlohmann-json
          sed -n '/^MIT License$/,/^SOFTWARE\.$/p' third-party/llama.cpp-unicode/src/unicode.cpp > LICENSE.llama.cpp-unicode
          curl -fsSL --retry 5 "https://raw.githubusercontent.com/pybind/pybind11/${PYBIND11_REF}/LICENSE" -o LICENSE.pybind11

      - uses: pypa/cibuildwheel@1828c10ab37f080699c7b81cea34097c684a7074  # v4.2.0
        with:
          only: ${{ matrix.python }}-manylinux_riscv64
        env:
          CIBW_MANYLINUX_RISCV64_IMAGE: ${{ env.MANYLINUX_RISCV64_IMAGE }}
          CIBW_ENVIRONMENT: >-
            CMAKE_BUILD_PARALLEL_LEVEL=8
            PIP_EXTRA_INDEX_URL=https://pypi.riseproject.dev/simple/
          # Upstream's smoke test (.ci/scripts/wheel/test_wheel.py) copies test/ into a
          # temp directory and runs pytest there, which is what test-sources reproduces.
          CIBW_TEST_SOURCES: test
          CIBW_TEST_REQUIRES: pytest transformers
          CIBW_TEST_COMMAND: python -m pytest -vv test

      - name: Check the extension and the vendored licences made it into the wheel
        run: |
          python3 - wheelhouse/*.whl <<'EOF'
          import sys, zipfile
          expected = {"LICENSE", "LICENSE.abseil-cpp", "LICENSE.darts-clone", "LICENSE.esaxx",
                      "LICENSE.llama.cpp-unicode", "LICENSE.nlohmann-json", "LICENSE.pcre2",
                      "LICENSE.protobuf-lite", "LICENSE.pybind11", "LICENSE.re2",
                      "LICENSE.sentencepiece"}
          for whl in sys.argv[1:]:
              names = zipfile.ZipFile(whl).namelist()
              assert [n for n in names if n.startswith("pytorch_tokenizers/pytorch_tokenizers_cpp.")
                      and n.endswith(".so")], whl
              licenses = {n.split(".dist-info/licenses/", 1)[1] for n in names
                          if ".dist-info/licenses/" in n and not n.endswith("/")}
              assert licenses == expected, f"{whl}: {sorted(licenses)}"
              print(whl, "ok")
          EOF

      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
        with:
          name: pytorch-tokenizers-${{ env.PYTORCH_TOKENIZERS_VERSION }}-${{ matrix.python }}-manylinux_riscv64
          path: wheelhouse/*.whl
          if-no-files-found: error

  publish:
    name: Publish pytorch-tokenizers ${{ matrix.version }}
    needs: [setup, build_wheels]
    if: needs.setup.outputs.versions != '[]'
    strategy:
      fail-fast: false
      matrix:
        version: ${{ fromJSON(needs.setup.outputs.versions) }}
    permissions:
      contents: write
      pull-requests: write
    uses: $/.github/workflows/_publish-wheel.yml
    secrets:
      app-private-key: ${{ secrets.RISEPROJECT_APP_PRIVATE_KEY }}
    with:
      artifact-pattern: pytorch-tokenizers-${{ matrix.version }}-*-manylinux_riscv64