From 786b099e5e49b0bf9caad8d6162d932b535054ab Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Thu, 20 Aug 2026 14:13:46 +0100 Subject: [PATCH 01/28] Generate SBOM for wheel - Generate an SBOM for PyThaiNLP package and embed it inside the wheel, following PEP 770 guidelines - Upload the wheel and the SBOM to GitHub Releases too Signed-off-by: Arthit Suriyawongkul --- .github/workflows/pypi-publish.yml | 127 +++++++++++++++++++++++++++-- 1 file changed, 118 insertions(+), 9 deletions(-) diff --git a/.github/workflows/pypi-publish.yml b/.github/workflows/pypi-publish.yml index 14ef4d7ee..664a47c5f 100644 --- a/.github/workflows/pypi-publish.yml +++ b/.github/workflows/pypi-publish.yml @@ -30,6 +30,8 @@ concurrency: }}-${{ github.head_ref || github.ref_name }} cancel-in-progress: true +permissions: read-all + jobs: echo_github_env: name: Echo env variables @@ -52,7 +54,7 @@ jobs: build: ${{ steps.check_build_trigger.outputs.build }} steps: - name: Checkout source code - uses: actions/checkout@v7.0.1 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.event.pull_request.head.sha }} - id: check_build_trigger @@ -68,19 +70,24 @@ jobs: strategy: matrix: python-version: ["3.12"] + outputs: + sbom-path: ${{ steps.pitloom.outputs.sbom-path }} steps: - name: Checkout - uses: actions/checkout@v7.0.1 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6.2.0 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ matrix.python-version }} + cache: pip + cache-dependency-path: pyproject.toml - name: Install dependencies run: | - pip install --upgrade build pip twine + pip install --upgrade pip + pip install build spdx3-validate twine - name: Build source distribution and wheels run: python -m build @@ -88,8 +95,62 @@ jobs: - name: Check distributions run: twine check dist/* + - name: Generate SBOM and embed into the wheel + id: pitloom + uses: bact/pitloom@4842922f65fb7ed1c5e51507c0fb253c62d97960 # v0.16.2 + with: + embed-wheel: "dist/*.whl" + project-path: "." + upload-artifact: "true" + artifact-name: "sbom" + + - name: Check SBOM is at the PEP 770 location + id: locate-sbom + env: + SBOM_PATH: ${{ steps.pitloom.outputs.sbom-path }} + run: | + set -euo pipefail + shopt -s nullglob + wheels=(dist/*.whl) + if [ "${#wheels[@]}" -ne 1 ]; then + echo "::error::expected exactly one wheel in dist/, found" \ + "${#wheels[@]}: ${wheels[*]}" + exit 1 + fi + whl="${wheels[0]}" + sbom_basename=$(basename "${SBOM_PATH}") + + matched="" + while IFS= read -r entry; do + case "${entry}" in + *.dist-info/sboms/"${sbom_basename}") + matched="${entry}" + break + ;; + esac + done < <(unzip -Z1 "${whl}") + + if [ -z "${matched}" ]; then + echo "::error::${sbom_basename} not found under" \ + "*.dist-info/sboms/ in ${whl}" + exit 1 + fi + echo "Found: ${matched}" + echo "wheel=${whl}" >> "${GITHUB_OUTPUT}" + echo "entry=${matched}" >> "${GITHUB_OUTPUT}" + + - name: Validate SBOM extracted from the wheel + env: + WHEEL: ${{ steps.locate-sbom.outputs.wheel }} + ENTRY: ${{ steps.locate-sbom.outputs.entry }} + run: | + set -euo pipefail + extracted="$(mktemp -d)/temp-sbom.json" + unzip -p "${WHEEL}" "${ENTRY}" > "${extracted}" + spdx3-validate --json "${extracted}" + - name: Store distributions - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: path: dist @@ -97,17 +158,65 @@ jobs: name: Publish to PyPI runs-on: ubuntu-latest needs: [build] - if: github.event_name == 'release' && github.event.action == 'published' + # Only runs if "build" (including all SBOM checks) succeeded, so a + # missing, misplaced, or invalid SBOM stops publishing entirely. + if: needs.build.result == 'success' && github.event_name == 'release' && github.event.action == 'published' steps: - name: Retrieve distributions - uses: actions/download-artifact@v8 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: artifact path: dist - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 - if: github.event_name == 'release' && github.event.action == 'published' + uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 with: skip-existing: true user: __token__ password: ${{ secrets.PYPI_API_TOKEN }} + + attach_release_sbom: + name: Attach SBOM to GitHub Release + needs: [build] + if: needs.build.result == 'success' && needs.build.outputs.sbom-path != '' && github.event_name == 'release' && github.event.action == 'published' + runs-on: ubuntu-latest + permissions: + contents: write # gh release upload, for the standalone SBOM asset + steps: + - name: Download SBOM artifact + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: sbom + path: sbom + - name: Attach SBOM to GitHub Release + env: + GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ github.event.release.tag_name }} + SBOM_BASENAME: ${{ needs.build.outputs.sbom-path }} + REPO: ${{ github.repository }} + run: | + set -euo pipefail + gh release upload "${RELEASE_TAG}" "sbom/${SBOM_BASENAME}" \ + --clobber --repo "${REPO}" + + attach_release_dist: + name: Attach distribution to GitHub Release + needs: [build, publish_pypi] + if: needs.build.result == 'success' && needs.publish_pypi.result == 'success' + runs-on: ubuntu-latest + permissions: + contents: write # gh release upload, for the sdist/wheel assets + steps: + - name: Download distribution artifacts + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: artifact + path: dist + - name: Attach sdist and wheel to GitHub Release + env: + GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ github.event.release.tag_name }} + REPO: ${{ github.repository }} + run: | + set -euo pipefail + gh release upload "${RELEASE_TAG}" dist/*.whl dist/*.tar.gz \ + --clobber --repo "${REPO}" From 795b080782861dc6b81bbfad6d9916231c75c9e3 Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Thu, 20 Aug 2026 14:39:17 +0100 Subject: [PATCH 02/28] Fix SonarCloud issues Signed-off-by: Arthit Suriyawongkul --- .github/workflows/pypi-publish.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pypi-publish.yml b/.github/workflows/pypi-publish.yml index 664a47c5f..852cc7042 100644 --- a/.github/workflows/pypi-publish.yml +++ b/.github/workflows/pypi-publish.yml @@ -30,7 +30,8 @@ concurrency: }}-${{ github.head_ref || github.ref_name }} cancel-in-progress: true -permissions: read-all +permissions: + contents: read jobs: echo_github_env: @@ -69,7 +70,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.12"] + python-version: ["3.13"] outputs: sbom-path: ${{ steps.pitloom.outputs.sbom-path }} @@ -86,8 +87,8 @@ jobs: - name: Install dependencies run: | - pip install --upgrade pip - pip install build spdx3-validate twine + pip install --only-binary :all: "pip==26.2.1" + pip install --only-binary :all: "build==1.5.0" "spdx3-validate==0.0.7" "twine==7.0.0" - name: Build source distribution and wheels run: python -m build From 31abae7ec6bd9b2bb2ec77a8ec6302b32fdfb01f Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Thu, 20 Aug 2026 15:02:10 +0100 Subject: [PATCH 03/28] Fix SonarCloud issues Signed-off-by: Arthit Suriyawongkul --- .github/workflows/pypi-publish.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pypi-publish.yml b/.github/workflows/pypi-publish.yml index 852cc7042..433009fd1 100644 --- a/.github/workflows/pypi-publish.yml +++ b/.github/workflows/pypi-publish.yml @@ -30,13 +30,12 @@ concurrency: }}-${{ github.head_ref || github.ref_name }} cancel-in-progress: true -permissions: - contents: read - jobs: echo_github_env: name: Echo env variables runs-on: ubuntu-latest + permissions: + contents: read steps: - run: | echo "github.event.action : ${{ github.event.action }}" @@ -51,6 +50,8 @@ jobs: runs-on: ubuntu-latest # Not for forks if: github.repository == 'pythainlp/pythainlp' + permissions: + contents: read outputs: build: ${{ steps.check_build_trigger.outputs.build }} steps: @@ -71,6 +72,8 @@ jobs: strategy: matrix: python-version: ["3.13"] + permissions: + contents: read outputs: sbom-path: ${{ steps.pitloom.outputs.sbom-path }} @@ -162,6 +165,8 @@ jobs: # Only runs if "build" (including all SBOM checks) succeeded, so a # missing, misplaced, or invalid SBOM stops publishing entirely. if: needs.build.result == 'success' && github.event_name == 'release' && github.event.action == 'published' + permissions: + contents: read steps: - name: Retrieve distributions uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 From 9dd6188ebfd9ab6080876863d5f1387e2b7fe864 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 20 Aug 2026 14:53:10 +0000 Subject: [PATCH 04/28] Bump budoux from 0.7.0 to 0.9.0 Bumps [budoux](https://github.com/google/budoux) from 0.7.0 to 0.9.0. - [Release notes](https://github.com/google/budoux/releases) - [Commits](https://github.com/google/budoux/compare/v0.7.0...v0.9.0) --- updated-dependencies: - dependency-name: budoux dependency-version: 0.9.0 dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 09907e343..a3bdb8866 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -246,7 +246,7 @@ full = [ "attacut==1.0.6", "attaparse==1.0.0", "bpemb>=0.3.6,<0.4", - "budoux==0.7.0", + "budoux==0.9.0", "epitran==1.26.0", "esupar>=1.3.9,<2", 'fairseq>=0.10.0,<0.13;python_version<"3.11"', From 211d3b7dc433916b7a986913d59d1815bfb3e98f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 20 Aug 2026 14:53:25 +0000 Subject: [PATCH 05/28] Bump transformers from 5.14.1 to 5.15.0 Bumps [transformers](https://github.com/huggingface/transformers) from 5.14.1 to 5.15.0. - [Release notes](https://github.com/huggingface/transformers/releases) - [Commits](https://github.com/huggingface/transformers/compare/v5.14.1...v5.15.0) --- updated-dependencies: - dependency-name: transformers dependency-version: 5.15.0 dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 09907e343..3d8d07970 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -278,7 +278,7 @@ full = [ "thai-nner==0.3", "tltk>=1.10,<2", "torch>=1.13.1,<3", - "transformers==5.14.1", + "transformers==5.15.0", "ufal.chu-liu-edmonds==1.0.3", "word2word>=1.0.0,<2", "wtpsplit==1.3.0", From 9fc388251ba290ed2b1466f626024140ee926bf1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 20 Aug 2026 14:54:28 +0000 Subject: [PATCH 06/28] Bump github/codeql-action from 4.37.6 to 4.37.7 Bumps [github/codeql-action](https://github.com/github/codeql-action) from 4.37.6 to 4.37.7. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/v4.37.6...v4.37.7) --- updated-dependencies: - dependency-name: github/codeql-action dependency-version: 4.37.7 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 4774bb471..4d0165ea4 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -66,7 +66,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v4.37.6 + uses: github/codeql-action/init@v4.37.7 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -77,7 +77,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@v4.37.6 + uses: github/codeql-action/autobuild@v4.37.7 # ℹ️ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl @@ -91,4 +91,4 @@ jobs: # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v4.37.6 + uses: github/codeql-action/analyze@v4.37.7 From 4fdfa90618428cb233a112f0bf8801ad9d1051eb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 20 Aug 2026 14:54:42 +0000 Subject: [PATCH 07/28] Bump EndBug/add-and-commit from 10.0.0 to 11.0.0 Bumps [EndBug/add-and-commit](https://github.com/endbug/add-and-commit) from 10.0.0 to 11.0.0. - [Release notes](https://github.com/endbug/add-and-commit/releases) - [Commits](https://github.com/endbug/add-and-commit/compare/290ea2c423ad77ca9c62ae0f5b224379612c0321...645ecc0dd0a57f4d86d26c0aa5fc42c0a856fbca) --- updated-dependencies: - dependency-name: EndBug/add-and-commit dependency-version: 11.0.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/codemeta2cff.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codemeta2cff.yml b/.github/workflows/codemeta2cff.yml index 9acfc23c6..f5d43eff9 100644 --- a/.github/workflows/codemeta2cff.yml +++ b/.github/workflows/codemeta2cff.yml @@ -40,7 +40,7 @@ jobs: uses: dieghernan/cff-validator@d8f85828214016ce6976e740b6e0504c0fdc4dbb #v5.1.1 - name: Commit and push updated CITATION.cff - uses: EndBug/add-and-commit@290ea2c423ad77ca9c62ae0f5b224379612c0321 #v10.0.0 + uses: EndBug/add-and-commit@645ecc0dd0a57f4d86d26c0aa5fc42c0a856fbca #v11.0.0 with: message: "Update CITATION.cff from codemeta.json" add: "CITATION.cff" From 75691abf435b2ad67ce82ed48d39e025597ccff5 Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Thu, 20 Aug 2026 16:07:02 +0100 Subject: [PATCH 08/28] Add SBOM creator info Signed-off-by: Arthit Suriyawongkul --- pyproject.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 09907e343..70b3784fb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -364,6 +364,11 @@ values = ["dev", "beta", "prod"] [tool.coverage.run] source = ["pythainlp"] +[[tool.pitloom.creator]] +name = "Wannaphong Phatthiyaphaibun" +email = "wannaphong@pythainlp.org" +type = "person" + [tool.ruff] indent-width = 4 line-length = 79 From 2f2c5cb62fb883dee698055b4f31626d40fdedee Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 20 Aug 2026 18:59:29 +0000 Subject: [PATCH 09/28] Bump epitran from 1.26.0 to 1.35.2 Bumps [epitran](https://github.com/dmort27/epitran) from 1.26.0 to 1.35.2. - [Release notes](https://github.com/dmort27/epitran/releases) - [Changelog](https://github.com/dmort27/epitran/blob/master/CHANGELOG.md) - [Commits](https://github.com/dmort27/epitran/compare/v1.26.0...v1.35.2) --- updated-dependencies: - dependency-name: epitran dependency-version: 1.35.2 dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index e840415c1..ad304f519 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -247,7 +247,7 @@ full = [ "attaparse==1.0.0", "bpemb>=0.3.6,<0.4", "budoux==0.9.0", - "epitran==1.26.0", + "epitran==1.35.2", "esupar>=1.3.9,<2", 'fairseq>=0.10.0,<0.13;python_version<"3.11"', 'fairseq-fixed==0.12.3.1,<0.13;python_version>="3.11"', From 4f812ccc30e97d515e3790fc8bff0a69d0f4ce76 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 27 Aug 2026 14:53:06 +0000 Subject: [PATCH 10/28] Bump transformers from 5.15.0 to 5.15.1 Bumps [transformers](https://github.com/huggingface/transformers) from 5.15.0 to 5.15.1. - [Release notes](https://github.com/huggingface/transformers/releases) - [Commits](https://github.com/huggingface/transformers/compare/v5.15.0...v5.15.1) --- updated-dependencies: - dependency-name: transformers dependency-version: 5.15.1 dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index ad304f519..1b537256e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -278,7 +278,7 @@ full = [ "thai-nner==0.3", "tltk>=1.10,<2", "torch>=1.13.1,<3", - "transformers==5.15.0", + "transformers==5.15.1", "ufal.chu-liu-edmonds==1.0.3", "word2word>=1.0.0,<2", "wtpsplit==1.3.0", From 127db377b992f9bc8c20eb58822b873084e28533 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 27 Aug 2026 14:54:04 +0000 Subject: [PATCH 11/28] Bump EndBug/add-and-commit from 11.0.0 to 11.1.1 Bumps [EndBug/add-and-commit](https://github.com/endbug/add-and-commit) from 11.0.0 to 11.1.1. - [Release notes](https://github.com/endbug/add-and-commit/releases) - [Commits](https://github.com/endbug/add-and-commit/compare/645ecc0dd0a57f4d86d26c0aa5fc42c0a856fbca...cc9c08ba6c8df3b93a8f2db63e89b98368ae2ae8) --- updated-dependencies: - dependency-name: EndBug/add-and-commit dependency-version: 11.1.1 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/codemeta2cff.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codemeta2cff.yml b/.github/workflows/codemeta2cff.yml index f5d43eff9..2cdbb6449 100644 --- a/.github/workflows/codemeta2cff.yml +++ b/.github/workflows/codemeta2cff.yml @@ -40,7 +40,7 @@ jobs: uses: dieghernan/cff-validator@d8f85828214016ce6976e740b6e0504c0fdc4dbb #v5.1.1 - name: Commit and push updated CITATION.cff - uses: EndBug/add-and-commit@645ecc0dd0a57f4d86d26c0aa5fc42c0a856fbca #v11.0.0 + uses: EndBug/add-and-commit@cc9c08ba6c8df3b93a8f2db63e89b98368ae2ae8 #v11.1.1 with: message: "Update CITATION.cff from codemeta.json" add: "CITATION.cff" From 1f40d78f0d0b918f4a6053568516cd4660073c51 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 3 Sep 2026 14:52:56 +0000 Subject: [PATCH 12/28] Bump transformers from 5.15.1 to 5.16.1 Bumps [transformers](https://github.com/huggingface/transformers) from 5.15.1 to 5.16.1. - [Release notes](https://github.com/huggingface/transformers/releases) - [Commits](https://github.com/huggingface/transformers/compare/v5.15.1...v5.16.1) --- updated-dependencies: - dependency-name: transformers dependency-version: 5.16.1 dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 1b537256e..10a300ece 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -278,7 +278,7 @@ full = [ "thai-nner==0.3", "tltk>=1.10,<2", "torch>=1.13.1,<3", - "transformers==5.15.1", + "transformers==5.16.1", "ufal.chu-liu-edmonds==1.0.3", "word2word>=1.0.0,<2", "wtpsplit==1.3.0", From b0f35810c554c7273b4c5c5f038af1ee6d2dd77b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 3 Sep 2026 14:54:12 +0000 Subject: [PATCH 13/28] Bump github/codeql-action from 4.37.7 to 4.37.9 Bumps [github/codeql-action](https://github.com/github/codeql-action) from 4.37.7 to 4.37.9. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/v4.37.7...v4.37.9) --- updated-dependencies: - dependency-name: github/codeql-action dependency-version: 4.37.9 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 4d0165ea4..bcd3c6e61 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -66,7 +66,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v4.37.7 + uses: github/codeql-action/init@v4.37.9 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -77,7 +77,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@v4.37.7 + uses: github/codeql-action/autobuild@v4.37.9 # ℹ️ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl @@ -91,4 +91,4 @@ jobs: # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v4.37.7 + uses: github/codeql-action/analyze@v4.37.9 From 19732d0bc7c5113211ead5373ee35d6fca687231 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 10 Sep 2026 14:53:07 +0000 Subject: [PATCH 14/28] Bump budoux from 0.9.0 to 0.9.1 Bumps [budoux](https://github.com/google/budoux) from 0.9.0 to 0.9.1. - [Release notes](https://github.com/google/budoux/releases) - [Commits](https://github.com/google/budoux/compare/v0.9.0...v0.9.1) --- updated-dependencies: - dependency-name: budoux dependency-version: 0.9.1 dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 10a300ece..f1ec892f9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -246,7 +246,7 @@ full = [ "attacut==1.0.6", "attaparse==1.0.0", "bpemb>=0.3.6,<0.4", - "budoux==0.9.0", + "budoux==0.9.1", "epitran==1.35.2", "esupar>=1.3.9,<2", 'fairseq>=0.10.0,<0.13;python_version<"3.11"', From f18cc111386eb4bf84837a069ccb53fbb0fb58cc Mon Sep 17 00:00:00 2001 From: Kamthorn Krairaksa Date: Sun, 13 Sep 2026 19:07:37 +0700 Subject: [PATCH 15/28] fix(transliterate): stop runaway repetition in thai2rom romanize MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit romanize(..., engine="thai2rom" or "thai2rom_onnx") could return a ~100-character string of repeated characters instead of a valid romanization, e.g. romanize("กรุงเทพฯ", engine="thai2rom") produced "krungtheppaaaa...aaaa" (100 chars). Same for "ฯลฯ" and long Pali/Sanskrit-derived compounds such as "ราษฎรบำรุง" and "สตรีเศรษฐบุตรบำเพ็ญ". Both engines decode greedily (argmax/topk(1)) with no repetition penalty or n-gram blocking. When the attention mechanism gets trapped on certain syllable transitions or out-of-vocabulary sequences, it keeps re-attending to the same input position and never emits , so decoding runs to the hard _maxlength=100 cap and returns the truncated, meaningless repeat. Add pythainlp/transliterate/_repetition.py with find_trailing_repeat_period(), which detects a short cycle (1-12 target characters) repeating 3+ times in a row at the end of the tokens generated so far. Both Seq2Seq.forward() (thai2rom.py) and Seq2Seq_ONNX.run() (thai2rom_onnx.py) now check for this after each inference step and, if found, stop decoding and truncate the output to keep only the cycle's first occurrence rather than running to max_len. Adds regression tests using the reproduction cases above to tests/noauto_torch/testn_transliterate_torch.py and tests/noauto_onnx/testn_transliterate_onnx.py, plus unit tests for find_trailing_repeat_period() in tests/core/test_transliterate.py. Closes https://github.com/PyThaiNLP/pythainlp/issues/1403 --- CHANGELOG.md | 9 +++ pythainlp/transliterate/_repetition.py | 57 +++++++++++++++++++ pythainlp/transliterate/thai2rom.py | 26 ++++++++- pythainlp/transliterate/thai2rom_onnx.py | 21 +++++++ tests/core/test_transliterate.py | 34 +++++++++++ tests/noauto_onnx/testn_transliterate_onnx.py | 15 +++++ .../noauto_torch/testn_transliterate_torch.py | 15 +++++ 7 files changed, 175 insertions(+), 2 deletions(-) create mode 100644 pythainlp/transliterate/_repetition.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 75b064033..2894c4395 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,15 @@ and this project adheres to ## [Unreleased] +### Fixed + +- `pythainlp.transliterate.romanize` with the `thai2rom` and `thai2rom_onnx` + engines: greedy decoding could get stuck in a cycle and run to the hard + 100-character length cap, returning strings like + `krungtheppaaaa...aaaa` for inputs such as `กรุงเทพฯ`, `ฯลฯ`, or long + Pali/Sanskrit-derived compounds. Decoding now stops as soon as a short + output cycle repeats 3 times in a row. (issue #1403) + ## Changed - Improve guardrails in `check_sara()` and `nighit()` diff --git a/pythainlp/transliterate/_repetition.py b/pythainlp/transliterate/_repetition.py new file mode 100644 index 000000000..ac5d070f8 --- /dev/null +++ b/pythainlp/transliterate/_repetition.py @@ -0,0 +1,57 @@ +# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project +# SPDX-FileType: SOURCE +# SPDX-License-Identifier: Apache-2.0 +"""Cycle detection for greedy seq2seq decoding. + +Greedy decoding (picking the highest-probability token at each step, +with no repetition penalty or n-gram blocking) can get the decoder +stuck in a loop: the attention mechanism repeatedly attends to the +same input position and keeps emitting the same short token sequence +until the hard length limit is hit. This module detects such a cycle +as it forms so a caller can stop decoding early instead of returning +an unbounded run of repeated characters. + +See: https://github.com/PyThaiNLP/pythainlp/issues/1403 +""" + +from __future__ import annotations + +from typing import List, Optional + +__all__: List[str] = ["find_trailing_repeat_period"] + + +def find_trailing_repeat_period( + tokens: List[int], + min_period: int = 1, + max_period: int = 12, + min_repeats: int = 3, +) -> Optional[int]: + """Detect a short cycle repeating at the end of a token sequence. + + Checks period lengths from ``min_period`` to ``max_period`` + (smallest first) and returns the first period whose last + ``min_repeats`` copies at the end of ``tokens`` are identical. + + :param tokens: sequence of decoded token ids, in generation order + :param min_period: shortest cycle length to check, in tokens + :param max_period: longest cycle length to check, in tokens + :param min_repeats: number of consecutive copies of the cycle + required at the end of ``tokens`` to count as a repetition loop + :return: the period of the detected cycle, or None if no trailing + cycle of at least ``min_repeats`` copies is found + :rtype: Optional[int] + """ + n = len(tokens) + for period in range(min_period, max_period + 1): + window = period * min_repeats + if n < window: + continue + segment = tokens[-window:] + first = segment[:period] + if all( + segment[i * period : (i + 1) * period] == first + for i in range(1, min_repeats) + ): + return period + return None diff --git a/pythainlp/transliterate/thai2rom.py b/pythainlp/transliterate/thai2rom.py index 194503416..39a3d3f66 100644 --- a/pythainlp/transliterate/thai2rom.py +++ b/pythainlp/transliterate/thai2rom.py @@ -13,6 +13,7 @@ from torch import nn from pythainlp.corpus import get_corpus_path +from pythainlp.transliterate._repetition import find_trailing_repeat_period if TYPE_CHECKING: from typing import Dict @@ -23,6 +24,11 @@ _MODEL_NAME: str = "thai2rom-pytorch-attn" +# Minimum consecutive copies of a repeating cycle that marks the greedy +# decoder as stuck in a loop. See: find_trailing_repeat_period() and +# https://github.com/PyThaiNLP/pythainlp/issues/1403 +_REPEAT_MIN_CYCLES: int = 3 + class ThaiTransliterator: __model_filename: str @@ -410,6 +416,7 @@ def forward( max_source_len = encoder_outputs.size(1) mask = self.create_mask(source_seq[:, 0:max_source_len]) + generated_tokens: list[int] = [] for di in range(max_len): decoder_output, decoder_hidden, _ = self.decoder( decoder_input, decoder_hidden, encoder_outputs, mask @@ -426,8 +433,23 @@ def forward( else: decoder_input = topi.detach() - if inference and decoder_input == end_token: - return outputs[:di] + if inference: + if decoder_input == end_token: + return outputs[:di] + + # Greedy decoding has no repetition penalty, so a trapped + # attention pattern can loop forever instead of emitting + # . Stop as soon as a short cycle repeats, keeping + # one copy of it, rather than running to max_len. + generated_tokens.append(int(decoder_input.item())) + period = find_trailing_repeat_period( + generated_tokens, min_repeats=_REPEAT_MIN_CYCLES + ) + if period is not None: + cutoff = len(generated_tokens) - period * ( + _REPEAT_MIN_CYCLES - 1 + ) + return outputs[:cutoff] return outputs diff --git a/pythainlp/transliterate/thai2rom_onnx.py b/pythainlp/transliterate/thai2rom_onnx.py index e11e614ce..9b2471137 100644 --- a/pythainlp/transliterate/thai2rom_onnx.py +++ b/pythainlp/transliterate/thai2rom_onnx.py @@ -11,6 +11,7 @@ from onnxruntime import InferenceSession from pythainlp.corpus import get_corpus_path +from pythainlp.transliterate._repetition import find_trailing_repeat_period if TYPE_CHECKING: from typing import Dict, List @@ -22,6 +23,11 @@ _MODEL_DECODER_NAME: str = "thai2rom_decoder_onnx" _MODEL_CONFIG_NAME: str = "thai2rom_config_onnx" +# Minimum consecutive copies of a repeating cycle that marks the greedy +# decoder as stuck in a loop. See: find_trailing_repeat_period() and +# https://github.com/PyThaiNLP/pythainlp/issues/1403 +_REPEAT_MIN_CYCLES: int = 3 + class ThaiTransliterator_ONNX: def __init__(self) -> None: @@ -231,6 +237,7 @@ def run( max_source_len = encoder_outputs.shape[1] mask = self.create_mask(source_seq[:, 0:max_source_len]) + generated_tokens: List[int] = [] for di in range(max_len): decoder_output_raw, decoder_hidden = self.decoder.run( input_feed={ @@ -254,6 +261,20 @@ def run( if decoder_input.item() == end_token: return outputs[:di] + # Greedy decoding has no repetition penalty, so a trapped + # attention pattern can loop forever instead of emitting + # . Stop as soon as a short cycle repeats, keeping one + # copy of it, rather than running to max_len. + generated_tokens.append(int(decoder_input.item())) + period = find_trailing_repeat_period( + generated_tokens, min_repeats=_REPEAT_MIN_CYCLES + ) + if period is not None: + cutoff = len(generated_tokens) - period * ( + _REPEAT_MIN_CYCLES - 1 + ) + return outputs[:cutoff] + return outputs diff --git a/tests/core/test_transliterate.py b/tests/core/test_transliterate.py index 757d87a4d..474f6e984 100644 --- a/tests/core/test_transliterate.py +++ b/tests/core/test_transliterate.py @@ -5,6 +5,7 @@ import unittest from pythainlp.transliterate import pronunciate_pali, romanize, transliterate +from pythainlp.transliterate._repetition import find_trailing_repeat_period BASIC_TESTS = { None: "", @@ -190,3 +191,36 @@ def test_pronunciate_pali(self): self.assertEqual( pronunciate_pali("พฺราหฺมณ"), "พราหมะณะ" ) + + +class RepetitionCycleTestCase(unittest.TestCase): + """Tests for the greedy-decoding cycle detector. + + See: https://github.com/PyThaiNLP/pythainlp/issues/1403 + """ + + def test_no_cycle(self): + self.assertIsNone(find_trailing_repeat_period([])) + self.assertIsNone(find_trailing_repeat_period([1, 2, 3, 4, 5])) + self.assertIsNone(find_trailing_repeat_period([1, 1, 2, 2])) + + def test_single_char_cycle(self): + # e.g. the "aaaa..." tail seen for "กรุงเทพฯ" + self.assertEqual(find_trailing_repeat_period([9, 1, 1, 1]), 1) + self.assertEqual(find_trailing_repeat_period([1, 1]), None) + + def test_multi_char_cycle(self): + # e.g. the "botbotbot..." tail seen for "ราษฎรบำรุง" + tokens = [9, 8, 7, 1, 2, 3, 1, 2, 3, 1, 2, 3] + self.assertEqual(find_trailing_repeat_period(tokens), 3) + + def test_longer_period_requires_larger_max_period(self): + period = list(range(8)) + tokens = period * 3 + self.assertIsNone(find_trailing_repeat_period(tokens, max_period=5)) + self.assertEqual(find_trailing_repeat_period(tokens, max_period=8), 8) + + def test_min_repeats_threshold(self): + tokens = [1, 2, 1, 2] # only repeats twice + self.assertIsNone(find_trailing_repeat_period(tokens, min_repeats=3)) + self.assertEqual(find_trailing_repeat_period(tokens, min_repeats=2), 2) diff --git a/tests/noauto_onnx/testn_transliterate_onnx.py b/tests/noauto_onnx/testn_transliterate_onnx.py index 0576e3f16..10e9cdcab 100644 --- a/tests/noauto_onnx/testn_transliterate_onnx.py +++ b/tests/noauto_onnx/testn_transliterate_onnx.py @@ -39,3 +39,18 @@ def test_thai2rom_onnx_mixed_text(self): result = romanize("ภาษาไทย") self.assertIsInstance(result, str) self.assertGreater(len(result), 0) + + def test_thai2rom_onnx_no_repetition_runaway(self): + # Regression test for https://github.com/PyThaiNLP/pythainlp/issues/1403 + # Greedy decoding used to get stuck in a repetition loop and run + # to the hard _maxlength=100 cap for these inputs. + from pythainlp.transliterate.thai2rom_onnx import romanize + + for word in ( + "กรุงเทพฯ", + "ฯลฯ", + "ราษฎรบำรุง", + "สตรีเศรษฐบุตรบำเพ็ญ", + ): + result = romanize(word) + self.assertLess(len(result), 100) diff --git a/tests/noauto_torch/testn_transliterate_torch.py b/tests/noauto_torch/testn_transliterate_torch.py index 5d7268e35..0f642db5a 100644 --- a/tests/noauto_torch/testn_transliterate_torch.py +++ b/tests/noauto_torch/testn_transliterate_torch.py @@ -36,6 +36,21 @@ def test_thai2rom_empty_string(self): result = romanize("") self.assertIsInstance(result, str) + def test_thai2rom_no_repetition_runaway(self): + # Regression test for https://github.com/PyThaiNLP/pythainlp/issues/1403 + # Greedy decoding used to get stuck in a repetition loop and run + # to the hard _maxlength=100 cap for these inputs. + from pythainlp.transliterate.thai2rom import romanize + + for word in ( + "กรุงเทพฯ", + "ฯลฯ", + "ราษฎรบำรุง", + "สตรีเศรษฐบุตรบำเพ็ญ", + ): + result = romanize(word) + self.assertLess(len(result), 100) + def test_thaig2p_returns_string(self): from pythainlp.transliterate.thaig2p import transliterate From 53a1be098403787386cacb2c14b60a2408f8fd0f Mon Sep 17 00:00:00 2001 From: Kamthorn Krairaksa Date: Sun, 13 Sep 2026 19:18:11 +0700 Subject: [PATCH 16/28] fix(transliterate): stop runaway repetition in thaig2p g2p decoding MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit While fixing #1403 (thai2rom's romanize() looping to a 100-character repeat under greedy decoding), pythainlp/transliterate/thaig2p.py was found to share the exact same vulnerable Seq2Seq.forward() decode loop: argmax/topk(1) greedy decoding, no repetition penalty, and no guard beyond checking for the token. Reproduced the same failure mode on thaig2p.transliterate(): transliterate("สตรีเศรษฐบุตรบำเพ็ญ") # -> 's a ˨˩ . t r iː ˧ . s u ˧ . s u ˧ . s u ˧ ...' (100 chars) transliterate("เอ็มเอฟซีบัญชีเพื่อการชำระค่ารับซื้อคืน") # -> 'ʔ e m ˧ . b r i ˨˩ . b aː ˧ . b aː ˧ . b aː ˧ ...' (100 chars) Reuses find_trailing_repeat_period() from pythainlp/transliterate/_repetition.py (added on the thai2rom fix branch) in thaig2p.py's Seq2Seq.forward(), the same way it was wired into thai2rom.py and thai2rom_onnx.py: after each inference step, stop decoding and truncate to the cycle's first occurrence as soon as a short cycle repeats 3+ times, instead of running to max_len. Adds a regression test with the reproduction cases above to tests/noauto_torch/testn_transliterate_torch.py. Note: this branch is stacked on fix/thai2rom-repetition-loop (#1500) to reuse _repetition.py; this diff will shrink to just the thaig2p.py change once that PR merges. Refs https://github.com/PyThaiNLP/pythainlp/issues/1403 Refs https://github.com/PyThaiNLP/pythainlp/pull/1500 --- CHANGELOG.md | 6 +++++ pythainlp/transliterate/thaig2p.py | 26 +++++++++++++++++-- .../noauto_torch/testn_transliterate_torch.py | 18 +++++++++++++ 3 files changed, 48 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2894c4395..ad18275d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,12 @@ and this project adheres to `krungtheppaaaa...aaaa` for inputs such as `กรุงเทพฯ`, `ฯลฯ`, or long Pali/Sanskrit-derived compounds. Decoding now stops as soon as a short output cycle repeats 3 times in a row. (issue #1403) +- `pythainlp.transliterate.thaig2p.transliterate`: found while fixing + issue #1403, the same greedy-decoding `Seq2Seq` loop as `thai2rom` + could get stuck in a repetition cycle and run to the hard 100-character + cap for the same class of inputs (e.g. `สตรีเศรษฐบุตรบำเพ็ญ`, and long + unsegmented multi-word phrases). Fixed with the same cycle-detection + guard used for `thai2rom`. ## Changed diff --git a/pythainlp/transliterate/thaig2p.py b/pythainlp/transliterate/thaig2p.py index a2df19022..b9f72afad 100644 --- a/pythainlp/transliterate/thaig2p.py +++ b/pythainlp/transliterate/thaig2p.py @@ -15,6 +15,7 @@ from torch import nn from pythainlp.corpus import get_corpus_path +from pythainlp.transliterate._repetition import find_trailing_repeat_period if TYPE_CHECKING: from numpy.typing import NDArray @@ -25,6 +26,11 @@ _MODEL_NAME: str = "thai-g2p" +# Minimum consecutive copies of a repeating cycle that marks the greedy +# decoder as stuck in a loop. See: find_trailing_repeat_period() and +# https://github.com/PyThaiNLP/pythainlp/issues/1403 +_REPEAT_MIN_CYCLES: int = 3 + class ThaiG2P: """ @@ -429,6 +435,7 @@ def forward( max_source_len = encoder_outputs.size(1) mask = self.create_mask(source_seq[:, 0:max_source_len]) + generated_tokens: list[int] = [] for di in range(max_len): decoder_output, decoder_hidden, _ = self.decoder( decoder_input, decoder_hidden, encoder_outputs, mask @@ -446,8 +453,23 @@ def forward( else topi.detach() ) - if inference and decoder_input == end_token: - return outputs[:di] + if inference: + if decoder_input == end_token: + return outputs[:di] + + # Greedy decoding has no repetition penalty, so a trapped + # attention pattern can loop forever instead of emitting + # . Stop as soon as a short cycle repeats, keeping + # one copy of it, rather than running to max_len. + generated_tokens.append(int(decoder_input.item())) + period = find_trailing_repeat_period( + generated_tokens, min_repeats=_REPEAT_MIN_CYCLES + ) + if period is not None: + cutoff = len(generated_tokens) - period * ( + _REPEAT_MIN_CYCLES - 1 + ) + return outputs[:cutoff] return outputs diff --git a/tests/noauto_torch/testn_transliterate_torch.py b/tests/noauto_torch/testn_transliterate_torch.py index 0f642db5a..e49197e0a 100644 --- a/tests/noauto_torch/testn_transliterate_torch.py +++ b/tests/noauto_torch/testn_transliterate_torch.py @@ -68,6 +68,24 @@ def test_thaig2p_model_loaded(self): self.assertIn("", g2p._target_char_to_ix) self.assertIn("", g2p._target_char_to_ix) + def test_thaig2p_no_repetition_runaway(self): + # Regression test for https://github.com/PyThaiNLP/pythainlp/issues/1403 + # thaig2p.py shares the same greedy-decoding Seq2Seq loop as + # thai2rom.py and could get stuck in a repetition loop, running + # to the hard _maxlength=100 cap for these inputs. + from pythainlp.transliterate.thaig2p import transliterate + + for word in ( + "กรุงเทพฯ", + "ฯลฯ", + "ราษฎรบำรุง", + "สตรีเศรษฐบุตรบำเพ็ญ", + "เอ็มเอฟซีบัญชีเพื่อการชำระค่ารับซื้อคืน", + "บัญชีเพื่อการชำระค่าขายคืนหน่วยลงทุน", + ): + result = transliterate(word) + self.assertLess(len(result), 100) + def test_thaig2p_v2_returns_string(self): from pythainlp.transliterate.thaig2p_v2 import transliterate From e4c480231e165af57b23cef2c5dd254d2f5b86d7 Mon Sep 17 00:00:00 2001 From: Palmkonde Date: Wed, 16 Sep 2026 14:18:23 +0700 Subject: [PATCH 17/28] refactor(text_to_num): repeatly call _check_is_thai_num over and over --- pythainlp/util/wordtonum.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pythainlp/util/wordtonum.py b/pythainlp/util/wordtonum.py index 82bc00570..e39724e36 100644 --- a/pythainlp/util/wordtonum.py +++ b/pythainlp/util/wordtonum.py @@ -188,21 +188,22 @@ def text_to_num(text: str) -> list[str]: last_index = -1 list_word_new = [] for i, word in enumerate(_temp): + isthainum = _check_is_thainum(word, _temp, i, thainum)[0] if ( - _check_is_thainum(word)[0] + isthainum and last_index + 1 == i and i + 1 == len(_temp) ): thainum.append(word) list_word_new.append(str(words_to_num(thainum))) - elif _check_is_thainum(word)[0] and last_index + 1 == i: + elif isthainum and last_index + 1 == i: thainum.append(word) last_index = i - elif _check_is_thainum(word)[0]: + elif isthainum: thainum.append(word) last_index = i elif ( - not _check_is_thainum(word)[0] + not isthainum and last_index + 1 == i and last_index != -1 ): From 1413565775228dfacd0c9a89437c6213b3f5f225 Mon Sep 17 00:00:00 2001 From: Palmkonde Date: Wed, 16 Sep 2026 14:19:12 +0700 Subject: [PATCH 18/28] =?UTF-8?q?fix(wordtonum):=20add=20"=E0=B8=A8?= =?UTF-8?q?=E0=B8=B9=E0=B8=99=E0=B8=A2=E0=B9=8C"=20handle=20as=20a=20speci?= =?UTF-8?q?al=20case?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pythainlp/util/wordtonum.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/pythainlp/util/wordtonum.py b/pythainlp/util/wordtonum.py index e39724e36..e8704f08e 100644 --- a/pythainlp/util/wordtonum.py +++ b/pythainlp/util/wordtonum.py @@ -57,7 +57,23 @@ def _tokenizer() -> Tokenizer: return Tokenizer(custom_dict=_valid_tokens) -def _check_is_thainum(word: str) -> tuple[bool, Optional[str]]: +def _check_is_thainum( + word: str, + tokens: Optional[list[str]] = None, + i: Optional[int] = None, + thainum: Optional[list[str]] = None, +) -> tuple[bool, Optional[str]]: + if ( + word == "ศูนย์" + and tokens is not None + and i is not None + and thainum is not None + ): + if "จุด" in thainum or ( + i + 1 < len(tokens) and tokens[i + 1] == "จุด" + ): + return (True, "num") + return (False, None) for j in _digits: if j in word: return (True, "num") From 7e244ca77f9335ee05dde8b6160fdc3a2f91ea4f Mon Sep 17 00:00:00 2001 From: Palmkonde Date: Wed, 16 Sep 2026 14:55:50 +0700 Subject: [PATCH 19/28] fix(wordtonum): add more condition for flush to list --- pythainlp/util/wordtonum.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/pythainlp/util/wordtonum.py b/pythainlp/util/wordtonum.py index e8704f08e..0e3f9ad8f 100644 --- a/pythainlp/util/wordtonum.py +++ b/pythainlp/util/wordtonum.py @@ -183,6 +183,16 @@ def words_to_num(words: list[str]) -> float: return num +def _flush(thainum: list[str], result: list[str]) -> None: + has_digit = any( + w == "ศูนย์" or _check_is_thainum(w)[1] == "num" for w in thainum + ) + if has_digit: + result.append(str(words_to_num(thainum))) + else: + result.extend(thainum) + + def text_to_num(text: str) -> list[str]: """Thai text to list of Thai words with floating point numbers @@ -211,7 +221,7 @@ def text_to_num(text: str) -> list[str]: and i + 1 == len(_temp) ): thainum.append(word) - list_word_new.append(str(words_to_num(thainum))) + _flush(thainum, list_word_new) elif isthainum and last_index + 1 == i: thainum.append(word) last_index = i @@ -223,7 +233,7 @@ def text_to_num(text: str) -> list[str]: and last_index + 1 == i and last_index != -1 ): - list_word_new.append(str(words_to_num(thainum))) + _flush(thainum, list_word_new) thainum = [] list_word_new.append(word) else: From 8fe262282fc715edfa91e895ef96b852a0c6eb48 Mon Sep 17 00:00:00 2001 From: Palmkonde Date: Wed, 16 Sep 2026 15:03:10 +0700 Subject: [PATCH 20/28] refactor(wordtonum): refactor due to black --- pythainlp/util/wordtonum.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/pythainlp/util/wordtonum.py b/pythainlp/util/wordtonum.py index 0e3f9ad8f..9aa25abeb 100644 --- a/pythainlp/util/wordtonum.py +++ b/pythainlp/util/wordtonum.py @@ -215,11 +215,7 @@ def text_to_num(text: str) -> list[str]: list_word_new = [] for i, word in enumerate(_temp): isthainum = _check_is_thainum(word, _temp, i, thainum)[0] - if ( - isthainum - and last_index + 1 == i - and i + 1 == len(_temp) - ): + if isthainum and last_index + 1 == i and i + 1 == len(_temp): thainum.append(word) _flush(thainum, list_word_new) elif isthainum and last_index + 1 == i: @@ -228,11 +224,7 @@ def text_to_num(text: str) -> list[str]: elif isthainum: thainum.append(word) last_index = i - elif ( - not isthainum - and last_index + 1 == i - and last_index != -1 - ): + elif not isthainum and last_index + 1 == i and last_index != -1: _flush(thainum, list_word_new) thainum = [] list_word_new.append(word) From 61faccc7c71ece7ba18566a01461e955e1d4a388 Mon Sep 17 00:00:00 2001 From: Palmkonde Date: Wed, 16 Sep 2026 15:04:11 +0700 Subject: [PATCH 21/28] add(test_util): add testcase for 0 digit handle in text_to_num --- tests/core/test_util.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/tests/core/test_util.py b/tests/core/test_util.py index 514eb6753..d09c48c62 100644 --- a/tests/core/test_util.py +++ b/tests/core/test_util.py @@ -287,6 +287,39 @@ def test_number(self): with self.assertRaises(TypeError): text_to_thai_digit(None) # type: ignore[arg-type] + def test_text_to_num_zero(self): + # "ศูนย์" (zero) is excluded from the digit table as a special + # case, which used to corrupt or crash on any floating-point + self.assertEqual( + text_to_num("หนึ่งร้อยยี่สิบสี่จุดศูนย์สี่"), ["124.04"] + ) + self.assertEqual( + text_to_num("หนึ่งร้อยยี่สิบเอ็ดจุดศูนย์สี่ห้า"), ["121.045"] + ) + self.assertEqual(text_to_num("ศูนย์จุดศูนย์เก้า"), ["0.09"]) + self.assertEqual(text_to_num("ห้าจุดศูนย์ศูนย์เก้า"), ["5.009"]) + self.assertEqual(text_to_num("สามจุดสี่ศูนย์เก้าศูนย์"), ["3.409"]) + + # "ศูนย์" as part of an ordinary word (e.g. "center") must stay + self.assertEqual( + text_to_num("ศูนย์ประชุมอยู่ที่กรุงเทพ"), + ["ศูนย์", "ประชุม", "อยู่", "ที่", "กรุงเทพ"], + ) + self.assertEqual( + text_to_num("ค่าเช่าศูนย์ประชุมคือหนึ่งร้อยบาท"), + ["ค่าเช่า", "ศูนย์", "ประชุม", "คือ", "100", "บาท"], + ) + + # "จุด" as an ordinary word (e.g. "point/spot") must not crash + self.assertEqual( + text_to_num("จุดศูนย์กลางของเมืองอยู่ที่นี่"), + ["จุด", "ศูนย์กลาง", "ของ", "เมือง", "อยู่", "ที่นี่"], + ) + self.assertEqual( + text_to_num("จุดศูนย์รวมของทุกคนคือที่นี่"), + ["จุด", "ศูนย์รวม", "ของ", "ทุกคน", "คือ", "ที่นี่"], + ) + # ### pythainlp.util.keyboard def test_keyboard(self): From 2a4351051d7d9b277ad9c8f89af807e37c2caa1d Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Wed, 16 Sep 2026 15:46:02 +0100 Subject: [PATCH 22/28] Update Pitloom==0.18.0; build==1.6.1 Signed-off-by: Arthit Suriyawongkul --- .github/workflows/pypi-publish.yml | 4 ++-- pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pypi-publish.yml b/.github/workflows/pypi-publish.yml index 433009fd1..1d98156d3 100644 --- a/.github/workflows/pypi-publish.yml +++ b/.github/workflows/pypi-publish.yml @@ -91,7 +91,7 @@ jobs: - name: Install dependencies run: | pip install --only-binary :all: "pip==26.2.1" - pip install --only-binary :all: "build==1.5.0" "spdx3-validate==0.0.7" "twine==7.0.0" + pip install --only-binary :all: "build==1.6.1" "spdx3-validate==0.0.7" "twine==7.0.0" - name: Build source distribution and wheels run: python -m build @@ -101,7 +101,7 @@ jobs: - name: Generate SBOM and embed into the wheel id: pitloom - uses: bact/pitloom@4842922f65fb7ed1c5e51507c0fb253c62d97960 # v0.16.2 + uses: bact/pitloom@f5d6cf5e4ce669e2131d34422304576b5d31f218 # v0.18.0 with: embed-wheel: "dist/*.whl" project-path: "." diff --git a/pyproject.toml b/pyproject.toml index e8f4745af..f44d92e85 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,7 +73,7 @@ dependencies = [ dev = [ "bandit>=1.9.4", "black>=25.11.0", - "build>=1.0.0", + "build>=1.6.1", "bump-my-version>=1.2.6", "coverage>=7.10.7", "flake8>=7.0.0", From 63a28a42a92aec685716a80cbad3dc82f1ba7a12 Mon Sep 17 00:00:00 2001 From: Palmkonde Date: Wed, 16 Sep 2026 22:14:47 +0700 Subject: [PATCH 23/28] refactor(wordtonum): replace tokens/index with next_word per review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit bact suggested passing just the next token instead of threading the full token list and current index through _check_is_thainum; thainum still needed to detect "ศูนย์" coming after "จุด". Co-Authored-By: Claude Sonnet 5 --- pythainlp/util/wordtonum.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/pythainlp/util/wordtonum.py b/pythainlp/util/wordtonum.py index 9aa25abeb..ea71b8161 100644 --- a/pythainlp/util/wordtonum.py +++ b/pythainlp/util/wordtonum.py @@ -59,19 +59,11 @@ def _tokenizer() -> Tokenizer: def _check_is_thainum( word: str, - tokens: Optional[list[str]] = None, - i: Optional[int] = None, + next_word: str = "", thainum: Optional[list[str]] = None, ) -> tuple[bool, Optional[str]]: - if ( - word == "ศูนย์" - and tokens is not None - and i is not None - and thainum is not None - ): - if "จุด" in thainum or ( - i + 1 < len(tokens) and tokens[i + 1] == "จุด" - ): + if word == "ศูนย์" and thainum is not None: + if "จุด" in thainum or next_word == "จุด": return (True, "num") return (False, None) for j in _digits: @@ -214,7 +206,8 @@ def text_to_num(text: str) -> list[str]: last_index = -1 list_word_new = [] for i, word in enumerate(_temp): - isthainum = _check_is_thainum(word, _temp, i, thainum)[0] + next_word = _temp[i + 1] if i + 1 < len(_temp) else "" + isthainum = _check_is_thainum(word, next_word, thainum)[0] if isthainum and last_index + 1 == i and i + 1 == len(_temp): thainum.append(word) _flush(thainum, list_word_new) From 801c526a314bbdab35b9b08fe2ad0ffe6f7c756c Mon Sep 17 00:00:00 2001 From: Kamthorn Krairaksa Date: Thu, 17 Sep 2026 08:51:36 +0700 Subject: [PATCH 24/28] fix(transliterate): use assertIsNone per review on PR #1500 bact requested this change on PR #1500: prefer assertIsNone() over assertEqual(x, None) for the unittest idiom. --- tests/core/test_transliterate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/test_transliterate.py b/tests/core/test_transliterate.py index 474f6e984..dcee3f9d7 100644 --- a/tests/core/test_transliterate.py +++ b/tests/core/test_transliterate.py @@ -207,7 +207,7 @@ def test_no_cycle(self): def test_single_char_cycle(self): # e.g. the "aaaa..." tail seen for "กรุงเทพฯ" self.assertEqual(find_trailing_repeat_period([9, 1, 1, 1]), 1) - self.assertEqual(find_trailing_repeat_period([1, 1]), None) + self.assertIsNone(find_trailing_repeat_period([1, 1])) def test_multi_char_cycle(self): # e.g. the "botbotbot..." tail seen for "ราษฎรบำรุง" From a4b2575a436c88e72591f634be378225b78c6b60 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 17 Sep 2026 14:53:20 +0000 Subject: [PATCH 25/28] chore(deps-dev): bump transformers from 5.16.1 to 5.17.0 Bumps [transformers](https://github.com/huggingface/transformers) from 5.16.1 to 5.17.0. - [Release notes](https://github.com/huggingface/transformers/releases) - [Commits](https://github.com/huggingface/transformers/compare/v5.16.1...v5.17.0) --- updated-dependencies: - dependency-name: transformers dependency-version: 5.17.0 dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 2ff061d2b..1dedf4d16 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -278,7 +278,7 @@ full = [ "thai-nner==0.3", "tltk>=1.10,<2", "torch>=1.13.1,<3", - "transformers==5.16.1", + "transformers==5.17.0", "ufal.chu-liu-edmonds==1.0.3", "word2word>=1.0.0,<2", "wtpsplit==1.3.0", From fbc4654a4e9d98101a0b72ae33978efcd1b74aa6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 17 Sep 2026 14:53:56 +0000 Subject: [PATCH 26/28] chore(deps): bump github/codeql-action from 4.37.9 to 4.38.0 Bumps [github/codeql-action](https://github.com/github/codeql-action) from 4.37.9 to 4.38.0. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/v4.37.9...v4.38.0) --- updated-dependencies: - dependency-name: github/codeql-action dependency-version: 4.38.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index bcd3c6e61..f433a157e 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -66,7 +66,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v4.37.9 + uses: github/codeql-action/init@v4.38.0 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -77,7 +77,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@v4.37.9 + uses: github/codeql-action/autobuild@v4.38.0 # ℹ️ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl @@ -91,4 +91,4 @@ jobs: # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v4.37.9 + uses: github/codeql-action/analyze@v4.38.0 From 5a985d18b4891bec08831da5b0901be01cb29ed1 Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Fri, 18 Sep 2026 17:46:38 +0100 Subject: [PATCH 27/28] Update Pitloom to 0.18.1 Hatchling 1.32.3 breaks Pitloom 0.18.0. Fix by update to Pitloom 0.18.1. Signed-off-by: Arthit Suriyawongkul --- .github/workflows/pypi-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pypi-publish.yml b/.github/workflows/pypi-publish.yml index 1d98156d3..b4698e772 100644 --- a/.github/workflows/pypi-publish.yml +++ b/.github/workflows/pypi-publish.yml @@ -101,7 +101,7 @@ jobs: - name: Generate SBOM and embed into the wheel id: pitloom - uses: bact/pitloom@f5d6cf5e4ce669e2131d34422304576b5d31f218 # v0.18.0 + uses: bact/pitloom@6a5473c42efcd3727d2a1ff94745ba93ab46722f # v0.18.1 with: embed-wheel: "dist/*.whl" project-path: "." From fe695d47a3a6bac995441a22452c075cbf2a5e9b Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Fri, 18 Sep 2026 17:56:46 +0100 Subject: [PATCH 28/28] Update pitloom-version [cd build] Signed-off-by: Arthit Suriyawongkul --- .github/workflows/pypi-publish.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pypi-publish.yml b/.github/workflows/pypi-publish.yml index b4698e772..d87b94a59 100644 --- a/.github/workflows/pypi-publish.yml +++ b/.github/workflows/pypi-publish.yml @@ -103,6 +103,8 @@ jobs: id: pitloom uses: bact/pitloom@6a5473c42efcd3727d2a1ff94745ba93ab46722f # v0.18.1 with: + pitloom-version: "0.18.1" + python-version: ${{ matrix.python-version }} embed-wheel: "dist/*.whl" project-path: "." upload-artifact: "true"