Merge remote-tracking branch 'origin/main' into port/stpyv8

Author: luhenry

.github/workflows/build-cchardet.yml

@@ -51,11 +51,24 @@ jobs:
       CCHARDET_VERSION: ${{ matrix.version }}
 
     steps:
+      # Upstream tagged releases bare up to 2.1.7 and v-prefixed from 2.2.0 on.
+      - name: Resolve the cChardet tag
+        id: tag
+        run: |
+          for ref in "v$CCHARDET_VERSION" "$CCHARDET_VERSION"; do
+            if git ls-remote --exit-code --tags https://github.com/PyYoshi/cChardet "refs/tags/$ref" >/dev/null; then
+              echo "name=$ref" >> "$GITHUB_OUTPUT"
+              exit 0
+            fi
+          done
+          echo "::error::PyYoshi/cChardet has no tag for $CCHARDET_VERSION"
+          exit 1
+
       - name: Checkout cChardet ${{ env.CCHARDET_VERSION }}
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
         with:
           repository: PyYoshi/cChardet
-          ref: ${{ env.CCHARDET_VERSION }}
+          ref: ${{ steps.tag.outputs.name }}
           submodules: recursive
           persist-credentials: false
 
@@ -106,86 +119,68 @@ jobs:
           name: cchardet-${{ env.CCHARDET_VERSION }}-sdist
           path: dist/
 
-      - name: Extract sdist and stage the wheel smoke tests
-        # Upstream's own suite (src/tests/test.py) uses nose, dead on modern
-        # Python; this reproduces its checks with plain pytest assertions
-        # against the same bundled testdata/samples.
+      - name: Extract sdist and stage the wheel smoke test
+        # Upstream's own cibuildwheel config (test-command/test-sources in
+        # pyproject.toml) runs tools/wheel_smoke.py, but MANIFEST.in doesn't
+        # ship tools/ in the sdist, so this reproduces that same script here.
         run: |
           mkdir pkg
           tar xzf dist/*.tar.gz --strip-components=1 -C pkg
           cat > pkg/riscv64_test_cchardet.py <<'EOF'
-          import glob
+          import json
           import os
+          import subprocess
+          import sys
+          import sysconfig
+          import tempfile
+          from pathlib import Path
 
           import cchardet
 
-          TESTDATA_DIR = "pkg/src/tests/testdata"
-          SAMPLES_DIR = "pkg/src/tests/samples"
-
-          SKIP_LIST = [
-              TESTDATA_DIR + "/ja/utf-16le.txt",
-              TESTDATA_DIR + "/ja/utf-16be.txt",
-              TESTDATA_DIR + "/es/iso-8859-15.txt",
-              TESTDATA_DIR + "/da/iso-8859-1.txt",
-              TESTDATA_DIR + "/he/iso-8859-8.txt",
-          ]
-          SKIP_LIST_DECODE = SKIP_LIST + [
-              TESTDATA_DIR + "/vi/viscii.txt",
-              TESTDATA_DIR + "/zh/euc-tw.txt",
-          ]
 
+          def main() -> None:
+              # Upstream only added this assertion in tools/wheel_smoke.py starting at
+              # v2.3.0; v2.2.0's _cchardet extension doesn't declare free-threading
+              # support, so CPython 3.14t legitimately re-enables the GIL on import.
+              version = tuple(int(p) for p in os.environ["CCHARDET_VERSION"].split("."))
+              if sysconfig.get_config_var("Py_GIL_DISABLED") and version >= (2, 3, 0):
+                  assert not getattr(sys, "_is_gil_enabled")()
 
-          def test_ascii():
-              assert cchardet.detect(b"abcdefghijklmnopqrstuvwxyz")["encoding"].lower() == "ascii"
+              package_dir = Path(cchardet.__file__).parent
+              assert (package_dir / "py.typed").is_file()
+              assert (package_dir / "_cchardet.pyi").is_file()
 
+              expected = cchardet.detect(b"plain ASCII")
+              assert expected["encoding"] == "ASCII"
+              assert cchardet.detect(bytearray(b"plain ASCII")) == expected
+              assert cchardet.detect(memoryview(b"plain ASCII"), max_bytes=32) == expected
 
-          def test_detect():
-              testfiles = glob.glob(TESTDATA_DIR + "/*/*.txt")
-              assert testfiles
-              for testfile in testfiles:
-                  if testfile in SKIP_LIST:
-                      continue
-                  expected = os.path.splitext(os.path.basename(testfile))[0]
-                  with open(testfile, "rb") as f:
-                      result = cchardet.detect(f.read())
-                  assert result["encoding"] is not None, testfile
-                  assert result["encoding"].lower() == expected.lower(), (testfile, result)
+              french = ("Français, où êtes-vous ? Voilà l'été.").encode(
+                  "windows-1252"
+              )
+              candidates = cchardet.detect_all(french)
+              assert candidates
+              assert candidates[0] == cchardet.detect(french)
+              assert candidates[0]["language"] == "fr"
 
-
-          def test_detector():
               detector = cchardet.UniversalDetector()
-              with open(SAMPLES_DIR + "/wikipediaJa_One_Thousand_and_One_Nights_SJIS.txt", "rb") as f:
-                  for line in f:
-                      detector.feed(line)
-                      if detector.done:
-                          break
+              detector.feed(french[:10])
+              detector.feed(french[10:])
               detector.close()
-              assert detector.result["encoding"].lower() == "shift_jis"
-
-
-          def test_github_issue_20():
-              msg = b"\x8f"
-              cchardet.detect(msg)
-              detector = cchardet.UniversalDetector()
-              detector.feed(msg)
-              detector.close()
-
-
-          def test_decode():
-              for testfile in glob.glob(TESTDATA_DIR + "/*/*.txt"):
-                  if testfile in SKIP_LIST_DECODE:
-                      continue
-                  with open(testfile, "rb") as f:
-                      msg = f.read()
-                  msg.decode(cchardet.detect(msg)["encoding"])
-
+              assert detector.result == candidates[0]
 
-          def test_utf8_with_bom():
-              assert cchardet.detect(b"\xef\xbb\xbf")["encoding"].lower() == "utf-8-sig"
+              with tempfile.TemporaryDirectory() as directory:
+                  sample = Path(directory) / "sample.txt"
+                  sample.write_bytes(french)
+                  output = subprocess.check_output(
+                      [sys.executable, "-m", "cchardet", "--json", str(sample)],
+                      text=True,
+                  )
+                  assert json.loads(output)["language"] == "fr"
 
 
-          def test_null_bytes():
-              assert cchardet.detect(b"ABC\x00\x80\x81")["encoding"] is None
+          if __name__ == "__main__":
+              main()
           EOF
 
       - name: Build and test wheels
@@ -196,10 +191,12 @@ jobs:
           only: ${{ matrix.python }}-manylinux_riscv64
         env:
           CIBW_MANYLINUX_RISCV64_IMAGE: ${{ env.MANYLINUX_RISCV64_IMAGE }}
-          CIBW_TEST_REQUIRES: pytest
           # CIBW_TEST_SOURCES is relative to the cwd, not package-dir, hence "pkg/".
-          CIBW_TEST_SOURCES: pkg/riscv64_test_cchardet.py pkg/src/tests/testdata pkg/src/tests/samples
-          CIBW_TEST_COMMAND: python -m pytest pkg/riscv64_test_cchardet.py -v
+          CIBW_TEST_SOURCES: pkg/riscv64_test_cchardet.py
+          CIBW_TEST_COMMAND: python pkg/riscv64_test_cchardet.py
+          # A job-level `env:` never reaches the Linux container (gotcha 129), so the
+          # test script's os.environ["CCHARDET_VERSION"] read needs it via CIBW_ENVIRONMENT.
+          CIBW_ENVIRONMENT: CCHARDET_VERSION=${{ env.CCHARDET_VERSION }}
 
       - name: Verify the wheel ships the compiled extension
         run: |

.github/workflows/build-coremltools.yml

@@ -0,0 +1,198 @@
+# SPDX-FileCopyrightText: 2026 The RISE Project
+# SPDX-License-Identifier: MIT
+#
+# This workflow is based on: https://github.com/apple/coremltools/blob/main/scripts/build.sh
+name: Build coremltools wheels (riscv64)
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'Version glob to (re)build; empty builds every version of docs/packages/coremltools.yaml not released yet'
+        required: false
+        default: ''
+  pull_request:
+    branches: [main]
+    paths:
+      - '.github/workflows/build-coremltools.yml'
+      - 'docs/packages/coremltools.yaml'
+      - 'patches/coremltools/**'
+  push:
+    branches: [main]
+    paths:
+      - '.github/workflows/build-coremltools.yml'
+      - 'docs/packages/coremltools.yaml'
+      - 'patches/coremltools/**'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+env:
+  MANYLINUX_RISCV64_IMAGE: quay.io/pypa/manylinux_2_39_riscv64
+  PIP_EXTRA_INDEX_URL: https://pypi.riseproject.dev/simple/
+
+jobs:
+  setup:
+    uses: $/.github/workflows/_setup.yml
+    with:
+      package: coremltools
+      version: ${{ inputs.version }}
+
+  build_wheels:
+    needs: [setup]
+    if: needs.setup.outputs.versions != '[]'
+    name: Build coremltools ${{ matrix.version }} ${{ matrix.python }}-manylinux_riscv64
+    runs-on: ubuntu-24.04-riscv
+    timeout-minutes: 720
+    strategy:
+      fail-fast: false
+      matrix:
+        version: ${{ fromJSON(needs.setup.outputs.versions) }}
+        # Upstream stops at 3.13, and the vendored pybind11 2.13.1 predates 3.14.
+        python: [cp312-cp312, cp313-cp313]
+
+    env:
+      COREMLTOOLS_VERSION: ${{ matrix.version }}
+
+    steps:
+      - name: Checkout apple/coremltools ${{ env.COREMLTOOLS_VERSION }}
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+        with:
+          repository: apple/coremltools
+          ref: ${{ env.COREMLTOOLS_VERSION }}
+          path: coremltools
+          persist-credentials: false
+
+      - name: Checkout python-wheels
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+        with:
+          path: python-wheels
+          persist-credentials: false
+
+      - name: Patch coremltools source
+        working-directory: coremltools
+        run: git apply ../python-wheels/patches/coremltools/${{ env.COREMLTOOLS_VERSION }}/00*.patch
+
+      - name: Build the C++ libraries and the wheel
+        run: |
+          mkdir -p output "$HOME/.cache/coremltools-ccache"
+          docker run -i --rm \
+            -v "$PWD/coremltools:/coremltools" \
+            -v "$PWD/output:/output" \
+            -v "$HOME/.cache/coremltools-ccache:/ccache" \
+            -e CCACHE_DIR=/ccache \
+            -e COREMLTOOLS_VERSION \
+            "$MANYLINUX_RISCV64_IMAGE" bash -s <<'COREMLTOOLS_BUILD_EOF'
+          #!/usr/bin/env bash
+          set -euxo pipefail
+
+          COREMLTOOLS_VERSION="${COREMLTOOLS_VERSION:?must be set, e.g. 9.0}"
+          PYBIN=/opt/python/${{ matrix.python }}/bin
+          # CMakeLists.txt builds the vendored kmeans1d with a bare `python3`, so the
+          # interpreter this wheel is for has to be the one that wins on PATH.
+          export PATH="$PYBIN:$PATH"
+          # The vendored protobuf 3.19 declares cmake_minimum_required(VERSION 3.0.0).
+          export CMAKE_POLICY_VERSION_MINIMUM=3.5
+
+          dnf install -y --setopt=install_weak_deps=False libuuid-devel
+
+          python3 -m pip install -U setuptools wheel auditwheel
+
+          # Upstream's NOTICE.txt covers only kmeans1d; the rest of deps/ is compiled
+          # into the shipped .so files, and auditwheel grafts in libuuid.
+          cp /coremltools/deps/protobuf/LICENSE /coremltools/LICENSE.protobuf
+          cp /coremltools/deps/pybind11/LICENSE /coremltools/LICENSE.pybind11
+          cp /coremltools/deps/nlohmann/LICENSE.MIT /coremltools/LICENSE.nlohmann-json
+          cp /coremltools/deps/FP16/LICENSE /coremltools/LICENSE.FP16
+          uuid_pkg=$(rpm -qf --qf '%{NAME}\n' "$(ldconfig -p | awk '/libuuid\.so\.1 /{print $NF; exit}')")
+          dnf reinstall -y --setopt=tsflags= "$uuid_pkg" || true
+          cat /usr/share/licenses/"$uuid_pkg"/* > /coremltools/LICENSE.libuuid 2>/dev/null \
+            || rpm -q --qf '%{NAME} %{VERSION}: %{LICENSE}\n' "$uuid_pkg" > /coremltools/LICENSE.libuuid
+
+          CCACHE_ARGS=""
+          command -v ccache >/dev/null 2>&1 && \
+            CCACHE_ARGS="-DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache"
+
+          # FindPythonLibs searches standard prefixes and would find the image's own
+          # /usr/local interpreter instead of /opt/python/<tag>, so pin it the way
+          # scripts/env_activate.sh pins it for upstream's conda env.
+          py_inc=$(python3 -c 'import sysconfig; print(sysconfig.get_path("include"))')
+          py_lib=$(python3 -c 'import os, sysconfig; print(os.path.join(sysconfig.get_config_var("LIBDIR"), sysconfig.get_config_var("LIBRARY")))')
+
+          cmake -S /coremltools -B /tmp/build \
+            -DCMAKE_BUILD_TYPE=Release \
+            -DPYTHON_EXECUTABLE:FILEPATH="$PYBIN/python3" \
+            -DPYTHON_INCLUDE_DIR="$py_inc" \
+            -DPYTHON_LIBRARY="$py_lib" \
+            -DOVERWRITE_PB_SOURCE=0 \
+            ${CCACHE_ARGS}
+          cmake --build /tmp/build -j "$(nproc)"
+          cmake --build /tmp/build --target dist
+
+          auditwheel repair --strip /tmp/build/dist/coremltools-*.whl -w /output
+          ls -la /output
+          COREMLTOOLS_BUILD_EOF
+
+      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
+        with:
+          name: coremltools-${{ env.COREMLTOOLS_VERSION }}-${{ matrix.python }}-manylinux_riscv64
+          path: output/*.whl
+          if-no-files-found: error
+
+      - name: Test coremltools wheel
+        run: |
+          docker run -i --rm \
+            -v "$PWD/output:/output" \
+            -e PIP_EXTRA_INDEX_URL \
+            -e PIP_ONLY_BINARY=numpy,pillow \
+            "$MANYLINUX_RISCV64_IMAGE" bash -s <<'COREMLTOOLS_TEST_EOF'
+          #!/usr/bin/env bash
+          set -euxo pipefail
+
+          PYBIN=/opt/python/${{ matrix.python }}/bin
+
+          "$PYBIN/pip" install -U pytest pytest-timeout parameterized pillow
+          "$PYBIN/pip" install /output/coremltools-*.whl
+
+          "$PYBIN/python3" -c "
+          from coremltools import libmilstoragepython, libmodelpackage
+          assert libmilstoragepython.__file__.endswith('.so'), libmilstoragepython.__file__
+          assert libmodelpackage.__file__.endswith('.so'), libmodelpackage.__file__
+          "
+
+          # Upstream's scripts/test.sh runs the suite out of the installed wheel with
+          # --pyargs, from a directory that is not the checkout. torch, tensorflow,
+          # scikit-learn and xgboost have no riscv64 wheels, so this is the subset of
+          # upstream's coremltools.test / mil jobs that collects without them; the two
+          # -k exclusions are tests that import torch inside the test body.
+          mkdir -p /tmp/testrun && cd /tmp/testrun
+          "$PYBIN/python3" -m pytest -v -ra --timeout=600 \
+            -k "not (TestApiVisibilities and test_optimize) and not test_model_save_no_extension" \
+            --pyargs \
+            coremltools.test.api \
+            coremltools.test.blob \
+            coremltools.test.modelpackage.test_modelpackage \
+            coremltools.converters.mil.backend \
+            coremltools.converters.mil.mil.tests.test_types
+          COREMLTOOLS_TEST_EOF
+
+  publish:
+    name: Publish coremltools ${{ matrix.version }}
+    needs: [setup, build_wheels]
+    if: needs.setup.outputs.versions != '[]'
+    strategy:
+      fail-fast: false
+      matrix:
+        version: ${{ fromJSON(needs.setup.outputs.versions) }}
+    permissions:
+      contents: write
+      pull-requests: write
+    uses: $/.github/workflows/_publish-wheel.yml
+    secrets:
+      app-private-key: ${{ secrets.RISEPROJECT_APP_PRIVATE_KEY }}
+    with:
+      artifact-pattern: coremltools-${{ matrix.version }}-*-manylinux_riscv64