diff --git a/.github/workflows/codemeta2cff.yml b/.github/workflows/codemeta2cff.yml index 9acfc23c6..2cdbb6449 100644 --- a/.github/workflows/codemeta2cff.yml +++ b/.github/workflows/codemeta2cff.yml @@ -40,7 +40,7 @@ jobs: uses: dieghernan/cff-validator@d8f85828214016ce6976e740b6e0504c0fdc4dbb #v5.1.1 - name: Commit and push updated CITATION.cff - uses: EndBug/add-and-commit@290ea2c423ad77ca9c62ae0f5b224379612c0321 #v10.0.0 + uses: EndBug/add-and-commit@cc9c08ba6c8df3b93a8f2db63e89b98368ae2ae8 #v11.1.1 with: message: "Update CITATION.cff from codemeta.json" add: "CITATION.cff" diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 4774bb471..f433a157e 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -66,7 +66,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v4.37.6 + uses: github/codeql-action/init@v4.38.0 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -77,7 +77,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@v4.37.6 + uses: github/codeql-action/autobuild@v4.38.0 # ℹ️ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl @@ -91,4 +91,4 @@ jobs: # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v4.37.6 + uses: github/codeql-action/analyze@v4.38.0 diff --git a/.github/workflows/pypi-publish.yml b/.github/workflows/pypi-publish.yml index 14ef4d7ee..d87b94a59 100644 --- a/.github/workflows/pypi-publish.yml +++ b/.github/workflows/pypi-publish.yml @@ -34,6 +34,8 @@ jobs: echo_github_env: name: Echo env variables runs-on: ubuntu-latest + permissions: + contents: read steps: - run: | echo "github.event.action : ${{ github.event.action }}" @@ -48,11 +50,13 @@ jobs: runs-on: ubuntu-latest # Not for forks if: github.repository == 'pythainlp/pythainlp' + permissions: + contents: read outputs: build: ${{ steps.check_build_trigger.outputs.build }} steps: - name: Checkout source code - uses: actions/checkout@v7.0.1 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.event.pull_request.head.sha }} - id: check_build_trigger @@ -67,20 +71,27 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.12"] + python-version: ["3.13"] + permissions: + contents: read + outputs: + sbom-path: ${{ steps.pitloom.outputs.sbom-path }} steps: - name: Checkout - uses: actions/checkout@v7.0.1 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6.2.0 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ matrix.python-version }} + cache: pip + cache-dependency-path: pyproject.toml - name: Install dependencies run: | - pip install --upgrade build pip twine + pip install --only-binary :all: "pip==26.2.1" + pip install --only-binary :all: "build==1.6.1" "spdx3-validate==0.0.7" "twine==7.0.0" - name: Build source distribution and wheels run: python -m build @@ -88,8 +99,64 @@ jobs: - name: Check distributions run: twine check dist/* + - name: Generate SBOM and embed into the wheel + id: pitloom + uses: bact/pitloom@6a5473c42efcd3727d2a1ff94745ba93ab46722f # v0.18.1 + with: + pitloom-version: "0.18.1" + python-version: ${{ matrix.python-version }} + embed-wheel: "dist/*.whl" + project-path: "." + upload-artifact: "true" + artifact-name: "sbom" + + - name: Check SBOM is at the PEP 770 location + id: locate-sbom + env: + SBOM_PATH: ${{ steps.pitloom.outputs.sbom-path }} + run: | + set -euo pipefail + shopt -s nullglob + wheels=(dist/*.whl) + if [ "${#wheels[@]}" -ne 1 ]; then + echo "::error::expected exactly one wheel in dist/, found" \ + "${#wheels[@]}: ${wheels[*]}" + exit 1 + fi + whl="${wheels[0]}" + sbom_basename=$(basename "${SBOM_PATH}") + + matched="" + while IFS= read -r entry; do + case "${entry}" in + *.dist-info/sboms/"${sbom_basename}") + matched="${entry}" + break + ;; + esac + done < <(unzip -Z1 "${whl}") + + if [ -z "${matched}" ]; then + echo "::error::${sbom_basename} not found under" \ + "*.dist-info/sboms/ in ${whl}" + exit 1 + fi + echo "Found: ${matched}" + echo "wheel=${whl}" >> "${GITHUB_OUTPUT}" + echo "entry=${matched}" >> "${GITHUB_OUTPUT}" + + - name: Validate SBOM extracted from the wheel + env: + WHEEL: ${{ steps.locate-sbom.outputs.wheel }} + ENTRY: ${{ steps.locate-sbom.outputs.entry }} + run: | + set -euo pipefail + extracted="$(mktemp -d)/temp-sbom.json" + unzip -p "${WHEEL}" "${ENTRY}" > "${extracted}" + spdx3-validate --json "${extracted}" + - name: Store distributions - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: path: dist @@ -97,17 +164,67 @@ jobs: name: Publish to PyPI runs-on: ubuntu-latest needs: [build] - if: github.event_name == 'release' && github.event.action == 'published' + # Only runs if "build" (including all SBOM checks) succeeded, so a + # missing, misplaced, or invalid SBOM stops publishing entirely. + if: needs.build.result == 'success' && github.event_name == 'release' && github.event.action == 'published' + permissions: + contents: read steps: - name: Retrieve distributions - uses: actions/download-artifact@v8 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: artifact path: dist - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 - if: github.event_name == 'release' && github.event.action == 'published' + uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 with: skip-existing: true user: __token__ password: ${{ secrets.PYPI_API_TOKEN }} + + attach_release_sbom: + name: Attach SBOM to GitHub Release + needs: [build] + if: needs.build.result == 'success' && needs.build.outputs.sbom-path != '' && github.event_name == 'release' && github.event.action == 'published' + runs-on: ubuntu-latest + permissions: + contents: write # gh release upload, for the standalone SBOM asset + steps: + - name: Download SBOM artifact + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: sbom + path: sbom + - name: Attach SBOM to GitHub Release + env: + GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ github.event.release.tag_name }} + SBOM_BASENAME: ${{ needs.build.outputs.sbom-path }} + REPO: ${{ github.repository }} + run: | + set -euo pipefail + gh release upload "${RELEASE_TAG}" "sbom/${SBOM_BASENAME}" \ + --clobber --repo "${REPO}" + + attach_release_dist: + name: Attach distribution to GitHub Release + needs: [build, publish_pypi] + if: needs.build.result == 'success' && needs.publish_pypi.result == 'success' + runs-on: ubuntu-latest + permissions: + contents: write # gh release upload, for the sdist/wheel assets + steps: + - name: Download distribution artifacts + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: artifact + path: dist + - name: Attach sdist and wheel to GitHub Release + env: + GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ github.event.release.tag_name }} + REPO: ${{ github.repository }} + run: | + set -euo pipefail + gh release upload "${RELEASE_TAG}" dist/*.whl dist/*.tar.gz \ + --clobber --repo "${REPO}" diff --git a/CHANGELOG.md b/CHANGELOG.md index f86555e28..9f41e4765 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,20 @@ and this project adheres to - Commit history: ## [Unreleased] +### Fixed + +- `pythainlp.transliterate.romanize` with the `thai2rom` and `thai2rom_onnx` + engines: greedy decoding could get stuck in a cycle and run to the hard + 100-character length cap, returning strings like + `krungtheppaaaa...aaaa` for inputs such as `กรุงเทพฯ`, `ฯลฯ`, or long + Pali/Sanskrit-derived compounds. Decoding now stops as soon as a short + output cycle repeats 3 times in a row. (issue #1403) +- `pythainlp.transliterate.thaig2p.transliterate`: found while fixing + issue #1403, the same greedy-decoding `Seq2Seq` loop as `thai2rom` + could get stuck in a repetition cycle and run to the hard 100-character + cap for the same class of inputs (e.g. `สตรีเศรษฐบุตรบำเพ็ญ`, and long + unsegmented multi-word phrases). Fixed with the same cycle-detection + guard used for `thai2rom`. ### Added - Add word tokenization evaluation metrics (`evaluate_word_tokenization`, diff --git a/pyproject.toml b/pyproject.toml index f33aa2097..e1f3c9066 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,7 +73,7 @@ dependencies = [ dev = [ "bandit>=1.9.4", "black>=25.11.0", - "build>=1.0.0", + "build>=1.6.1", "bump-my-version>=1.2.6", "coverage>=7.10.7", "flake8>=7.0.0", @@ -246,8 +246,8 @@ noauto-network = [ full = [ "attaparse==1.0.0", "bpemb>=0.3.6,<0.4", - "budoux==0.7.0", - "epitran==1.26.0", + "budoux==0.9.1", + "epitran==1.35.2", "esupar>=1.3.9,<2", 'fairseq>=0.10.0,<0.13;python_version<"3.11"', 'fairseq-fixed==0.12.3.1,<0.13;python_version>="3.11"', @@ -277,7 +277,7 @@ full = [ "thai-nner==0.3", "tltk>=1.10,<2", "torch>=1.13.1,<3", - "transformers==5.14.1", + "transformers==5.17.0", "ufal.chu-liu-edmonds==1.0.3", "word2word>=1.0.0,<2", "wtpsplit==1.3.0", @@ -363,6 +363,11 @@ values = ["dev", "beta", "prod"] [tool.coverage.run] source = ["pythainlp"] +[[tool.pitloom.creator]] +name = "Wannaphong Phatthiyaphaibun" +email = "wannaphong@pythainlp.org" +type = "person" + [tool.ruff] indent-width = 4 line-length = 79 diff --git a/pythainlp/transliterate/_repetition.py b/pythainlp/transliterate/_repetition.py new file mode 100644 index 000000000..ac5d070f8 --- /dev/null +++ b/pythainlp/transliterate/_repetition.py @@ -0,0 +1,57 @@ +# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project +# SPDX-FileType: SOURCE +# SPDX-License-Identifier: Apache-2.0 +"""Cycle detection for greedy seq2seq decoding. + +Greedy decoding (picking the highest-probability token at each step, +with no repetition penalty or n-gram blocking) can get the decoder +stuck in a loop: the attention mechanism repeatedly attends to the +same input position and keeps emitting the same short token sequence +until the hard length limit is hit. This module detects such a cycle +as it forms so a caller can stop decoding early instead of returning +an unbounded run of repeated characters. + +See: https://github.com/PyThaiNLP/pythainlp/issues/1403 +""" + +from __future__ import annotations + +from typing import List, Optional + +__all__: List[str] = ["find_trailing_repeat_period"] + + +def find_trailing_repeat_period( + tokens: List[int], + min_period: int = 1, + max_period: int = 12, + min_repeats: int = 3, +) -> Optional[int]: + """Detect a short cycle repeating at the end of a token sequence. + + Checks period lengths from ``min_period`` to ``max_period`` + (smallest first) and returns the first period whose last + ``min_repeats`` copies at the end of ``tokens`` are identical. + + :param tokens: sequence of decoded token ids, in generation order + :param min_period: shortest cycle length to check, in tokens + :param max_period: longest cycle length to check, in tokens + :param min_repeats: number of consecutive copies of the cycle + required at the end of ``tokens`` to count as a repetition loop + :return: the period of the detected cycle, or None if no trailing + cycle of at least ``min_repeats`` copies is found + :rtype: Optional[int] + """ + n = len(tokens) + for period in range(min_period, max_period + 1): + window = period * min_repeats + if n < window: + continue + segment = tokens[-window:] + first = segment[:period] + if all( + segment[i * period : (i + 1) * period] == first + for i in range(1, min_repeats) + ): + return period + return None diff --git a/pythainlp/transliterate/thai2rom.py b/pythainlp/transliterate/thai2rom.py index 194503416..39a3d3f66 100644 --- a/pythainlp/transliterate/thai2rom.py +++ b/pythainlp/transliterate/thai2rom.py @@ -13,6 +13,7 @@ from torch import nn from pythainlp.corpus import get_corpus_path +from pythainlp.transliterate._repetition import find_trailing_repeat_period if TYPE_CHECKING: from typing import Dict @@ -23,6 +24,11 @@ _MODEL_NAME: str = "thai2rom-pytorch-attn" +# Minimum consecutive copies of a repeating cycle that marks the greedy +# decoder as stuck in a loop. See: find_trailing_repeat_period() and +# https://github.com/PyThaiNLP/pythainlp/issues/1403 +_REPEAT_MIN_CYCLES: int = 3 + class ThaiTransliterator: __model_filename: str @@ -410,6 +416,7 @@ def forward( max_source_len = encoder_outputs.size(1) mask = self.create_mask(source_seq[:, 0:max_source_len]) + generated_tokens: list[int] = [] for di in range(max_len): decoder_output, decoder_hidden, _ = self.decoder( decoder_input, decoder_hidden, encoder_outputs, mask @@ -426,8 +433,23 @@ def forward( else: decoder_input = topi.detach() - if inference and decoder_input == end_token: - return outputs[:di] + if inference: + if decoder_input == end_token: + return outputs[:di] + + # Greedy decoding has no repetition penalty, so a trapped + # attention pattern can loop forever instead of emitting + # . Stop as soon as a short cycle repeats, keeping + # one copy of it, rather than running to max_len. + generated_tokens.append(int(decoder_input.item())) + period = find_trailing_repeat_period( + generated_tokens, min_repeats=_REPEAT_MIN_CYCLES + ) + if period is not None: + cutoff = len(generated_tokens) - period * ( + _REPEAT_MIN_CYCLES - 1 + ) + return outputs[:cutoff] return outputs diff --git a/pythainlp/transliterate/thai2rom_onnx.py b/pythainlp/transliterate/thai2rom_onnx.py index e11e614ce..9b2471137 100644 --- a/pythainlp/transliterate/thai2rom_onnx.py +++ b/pythainlp/transliterate/thai2rom_onnx.py @@ -11,6 +11,7 @@ from onnxruntime import InferenceSession from pythainlp.corpus import get_corpus_path +from pythainlp.transliterate._repetition import find_trailing_repeat_period if TYPE_CHECKING: from typing import Dict, List @@ -22,6 +23,11 @@ _MODEL_DECODER_NAME: str = "thai2rom_decoder_onnx" _MODEL_CONFIG_NAME: str = "thai2rom_config_onnx" +# Minimum consecutive copies of a repeating cycle that marks the greedy +# decoder as stuck in a loop. See: find_trailing_repeat_period() and +# https://github.com/PyThaiNLP/pythainlp/issues/1403 +_REPEAT_MIN_CYCLES: int = 3 + class ThaiTransliterator_ONNX: def __init__(self) -> None: @@ -231,6 +237,7 @@ def run( max_source_len = encoder_outputs.shape[1] mask = self.create_mask(source_seq[:, 0:max_source_len]) + generated_tokens: List[int] = [] for di in range(max_len): decoder_output_raw, decoder_hidden = self.decoder.run( input_feed={ @@ -254,6 +261,20 @@ def run( if decoder_input.item() == end_token: return outputs[:di] + # Greedy decoding has no repetition penalty, so a trapped + # attention pattern can loop forever instead of emitting + # . Stop as soon as a short cycle repeats, keeping one + # copy of it, rather than running to max_len. + generated_tokens.append(int(decoder_input.item())) + period = find_trailing_repeat_period( + generated_tokens, min_repeats=_REPEAT_MIN_CYCLES + ) + if period is not None: + cutoff = len(generated_tokens) - period * ( + _REPEAT_MIN_CYCLES - 1 + ) + return outputs[:cutoff] + return outputs diff --git a/pythainlp/transliterate/thaig2p.py b/pythainlp/transliterate/thaig2p.py index a2df19022..b9f72afad 100644 --- a/pythainlp/transliterate/thaig2p.py +++ b/pythainlp/transliterate/thaig2p.py @@ -15,6 +15,7 @@ from torch import nn from pythainlp.corpus import get_corpus_path +from pythainlp.transliterate._repetition import find_trailing_repeat_period if TYPE_CHECKING: from numpy.typing import NDArray @@ -25,6 +26,11 @@ _MODEL_NAME: str = "thai-g2p" +# Minimum consecutive copies of a repeating cycle that marks the greedy +# decoder as stuck in a loop. See: find_trailing_repeat_period() and +# https://github.com/PyThaiNLP/pythainlp/issues/1403 +_REPEAT_MIN_CYCLES: int = 3 + class ThaiG2P: """ @@ -429,6 +435,7 @@ def forward( max_source_len = encoder_outputs.size(1) mask = self.create_mask(source_seq[:, 0:max_source_len]) + generated_tokens: list[int] = [] for di in range(max_len): decoder_output, decoder_hidden, _ = self.decoder( decoder_input, decoder_hidden, encoder_outputs, mask @@ -446,8 +453,23 @@ def forward( else topi.detach() ) - if inference and decoder_input == end_token: - return outputs[:di] + if inference: + if decoder_input == end_token: + return outputs[:di] + + # Greedy decoding has no repetition penalty, so a trapped + # attention pattern can loop forever instead of emitting + # . Stop as soon as a short cycle repeats, keeping + # one copy of it, rather than running to max_len. + generated_tokens.append(int(decoder_input.item())) + period = find_trailing_repeat_period( + generated_tokens, min_repeats=_REPEAT_MIN_CYCLES + ) + if period is not None: + cutoff = len(generated_tokens) - period * ( + _REPEAT_MIN_CYCLES - 1 + ) + return outputs[:cutoff] return outputs diff --git a/pythainlp/util/wordtonum.py b/pythainlp/util/wordtonum.py index 82bc00570..ea71b8161 100644 --- a/pythainlp/util/wordtonum.py +++ b/pythainlp/util/wordtonum.py @@ -57,7 +57,15 @@ def _tokenizer() -> Tokenizer: return Tokenizer(custom_dict=_valid_tokens) -def _check_is_thainum(word: str) -> tuple[bool, Optional[str]]: +def _check_is_thainum( + word: str, + next_word: str = "", + thainum: Optional[list[str]] = None, +) -> tuple[bool, Optional[str]]: + if word == "ศูนย์" and thainum is not None: + if "จุด" in thainum or next_word == "จุด": + return (True, "num") + return (False, None) for j in _digits: if j in word: return (True, "num") @@ -167,6 +175,16 @@ def words_to_num(words: list[str]) -> float: return num +def _flush(thainum: list[str], result: list[str]) -> None: + has_digit = any( + w == "ศูนย์" or _check_is_thainum(w)[1] == "num" for w in thainum + ) + if has_digit: + result.append(str(words_to_num(thainum))) + else: + result.extend(thainum) + + def text_to_num(text: str) -> list[str]: """Thai text to list of Thai words with floating point numbers @@ -188,25 +206,19 @@ def text_to_num(text: str) -> list[str]: last_index = -1 list_word_new = [] for i, word in enumerate(_temp): - if ( - _check_is_thainum(word)[0] - and last_index + 1 == i - and i + 1 == len(_temp) - ): + next_word = _temp[i + 1] if i + 1 < len(_temp) else "" + isthainum = _check_is_thainum(word, next_word, thainum)[0] + if isthainum and last_index + 1 == i and i + 1 == len(_temp): thainum.append(word) - list_word_new.append(str(words_to_num(thainum))) - elif _check_is_thainum(word)[0] and last_index + 1 == i: + _flush(thainum, list_word_new) + elif isthainum and last_index + 1 == i: thainum.append(word) last_index = i - elif _check_is_thainum(word)[0]: + elif isthainum: thainum.append(word) last_index = i - elif ( - not _check_is_thainum(word)[0] - and last_index + 1 == i - and last_index != -1 - ): - list_word_new.append(str(words_to_num(thainum))) + elif not isthainum and last_index + 1 == i and last_index != -1: + _flush(thainum, list_word_new) thainum = [] list_word_new.append(word) else: diff --git a/tests/core/test_transliterate.py b/tests/core/test_transliterate.py index 9717d903a..70755a6a8 100644 --- a/tests/core/test_transliterate.py +++ b/tests/core/test_transliterate.py @@ -6,6 +6,7 @@ from unittest.mock import patch from pythainlp.transliterate import pronunciate_pali, romanize, transliterate +from pythainlp.transliterate._repetition import find_trailing_repeat_period BASIC_TESTS = { None: "", @@ -198,3 +199,36 @@ def test_pronunciate_pali(self): self.assertEqual( pronunciate_pali("พฺราหฺมณ"), "พราหมะณะ" ) + + +class RepetitionCycleTestCase(unittest.TestCase): + """Tests for the greedy-decoding cycle detector. + + See: https://github.com/PyThaiNLP/pythainlp/issues/1403 + """ + + def test_no_cycle(self): + self.assertIsNone(find_trailing_repeat_period([])) + self.assertIsNone(find_trailing_repeat_period([1, 2, 3, 4, 5])) + self.assertIsNone(find_trailing_repeat_period([1, 1, 2, 2])) + + def test_single_char_cycle(self): + # e.g. the "aaaa..." tail seen for "กรุงเทพฯ" + self.assertEqual(find_trailing_repeat_period([9, 1, 1, 1]), 1) + self.assertIsNone(find_trailing_repeat_period([1, 1])) + + def test_multi_char_cycle(self): + # e.g. the "botbotbot..." tail seen for "ราษฎรบำรุง" + tokens = [9, 8, 7, 1, 2, 3, 1, 2, 3, 1, 2, 3] + self.assertEqual(find_trailing_repeat_period(tokens), 3) + + def test_longer_period_requires_larger_max_period(self): + period = list(range(8)) + tokens = period * 3 + self.assertIsNone(find_trailing_repeat_period(tokens, max_period=5)) + self.assertEqual(find_trailing_repeat_period(tokens, max_period=8), 8) + + def test_min_repeats_threshold(self): + tokens = [1, 2, 1, 2] # only repeats twice + self.assertIsNone(find_trailing_repeat_period(tokens, min_repeats=3)) + self.assertEqual(find_trailing_repeat_period(tokens, min_repeats=2), 2) diff --git a/tests/core/test_util.py b/tests/core/test_util.py index 514eb6753..d09c48c62 100644 --- a/tests/core/test_util.py +++ b/tests/core/test_util.py @@ -287,6 +287,39 @@ def test_number(self): with self.assertRaises(TypeError): text_to_thai_digit(None) # type: ignore[arg-type] + def test_text_to_num_zero(self): + # "ศูนย์" (zero) is excluded from the digit table as a special + # case, which used to corrupt or crash on any floating-point + self.assertEqual( + text_to_num("หนึ่งร้อยยี่สิบสี่จุดศูนย์สี่"), ["124.04"] + ) + self.assertEqual( + text_to_num("หนึ่งร้อยยี่สิบเอ็ดจุดศูนย์สี่ห้า"), ["121.045"] + ) + self.assertEqual(text_to_num("ศูนย์จุดศูนย์เก้า"), ["0.09"]) + self.assertEqual(text_to_num("ห้าจุดศูนย์ศูนย์เก้า"), ["5.009"]) + self.assertEqual(text_to_num("สามจุดสี่ศูนย์เก้าศูนย์"), ["3.409"]) + + # "ศูนย์" as part of an ordinary word (e.g. "center") must stay + self.assertEqual( + text_to_num("ศูนย์ประชุมอยู่ที่กรุงเทพ"), + ["ศูนย์", "ประชุม", "อยู่", "ที่", "กรุงเทพ"], + ) + self.assertEqual( + text_to_num("ค่าเช่าศูนย์ประชุมคือหนึ่งร้อยบาท"), + ["ค่าเช่า", "ศูนย์", "ประชุม", "คือ", "100", "บาท"], + ) + + # "จุด" as an ordinary word (e.g. "point/spot") must not crash + self.assertEqual( + text_to_num("จุดศูนย์กลางของเมืองอยู่ที่นี่"), + ["จุด", "ศูนย์กลาง", "ของ", "เมือง", "อยู่", "ที่นี่"], + ) + self.assertEqual( + text_to_num("จุดศูนย์รวมของทุกคนคือที่นี่"), + ["จุด", "ศูนย์รวม", "ของ", "ทุกคน", "คือ", "ที่นี่"], + ) + # ### pythainlp.util.keyboard def test_keyboard(self): diff --git a/tests/noauto_onnx/testn_transliterate_onnx.py b/tests/noauto_onnx/testn_transliterate_onnx.py index f4081824d..2e6b8e6b2 100644 --- a/tests/noauto_onnx/testn_transliterate_onnx.py +++ b/tests/noauto_onnx/testn_transliterate_onnx.py @@ -40,6 +40,20 @@ def test_thai2rom_onnx_mixed_text(self): self.assertIsInstance(result, str) self.assertGreater(len(result), 0) + def test_thai2rom_onnx_no_repetition_runaway(self): + # Regression test for https://github.com/PyThaiNLP/pythainlp/issues/1403 + # Greedy decoding used to get stuck in a repetition loop and run + # to the hard _maxlength=100 cap for these inputs. + from pythainlp.transliterate.thai2rom_onnx import romanize + + for word in ( + "กรุงเทพฯ", + "ฯลฯ", + "ราษฎรบำรุง", + "สตรีเศรษฐบุตรบำเพ็ญ", + ): + result = romanize(word) + self.assertLess(len(result), 100) def test_thaig2p_v4_returns_string(self): from pythainlp.transliterate.thaig2p_v4 import transliterate diff --git a/tests/noauto_torch/testn_transliterate_torch.py b/tests/noauto_torch/testn_transliterate_torch.py index 5d7268e35..e49197e0a 100644 --- a/tests/noauto_torch/testn_transliterate_torch.py +++ b/tests/noauto_torch/testn_transliterate_torch.py @@ -36,6 +36,21 @@ def test_thai2rom_empty_string(self): result = romanize("") self.assertIsInstance(result, str) + def test_thai2rom_no_repetition_runaway(self): + # Regression test for https://github.com/PyThaiNLP/pythainlp/issues/1403 + # Greedy decoding used to get stuck in a repetition loop and run + # to the hard _maxlength=100 cap for these inputs. + from pythainlp.transliterate.thai2rom import romanize + + for word in ( + "กรุงเทพฯ", + "ฯลฯ", + "ราษฎรบำรุง", + "สตรีเศรษฐบุตรบำเพ็ญ", + ): + result = romanize(word) + self.assertLess(len(result), 100) + def test_thaig2p_returns_string(self): from pythainlp.transliterate.thaig2p import transliterate @@ -53,6 +68,24 @@ def test_thaig2p_model_loaded(self): self.assertIn("", g2p._target_char_to_ix) self.assertIn("", g2p._target_char_to_ix) + def test_thaig2p_no_repetition_runaway(self): + # Regression test for https://github.com/PyThaiNLP/pythainlp/issues/1403 + # thaig2p.py shares the same greedy-decoding Seq2Seq loop as + # thai2rom.py and could get stuck in a repetition loop, running + # to the hard _maxlength=100 cap for these inputs. + from pythainlp.transliterate.thaig2p import transliterate + + for word in ( + "กรุงเทพฯ", + "ฯลฯ", + "ราษฎรบำรุง", + "สตรีเศรษฐบุตรบำเพ็ญ", + "เอ็มเอฟซีบัญชีเพื่อการชำระค่ารับซื้อคืน", + "บัญชีเพื่อการชำระค่าขายคืนหน่วยลงทุน", + ): + result = transliterate(word) + self.assertLess(len(result), 100) + def test_thaig2p_v2_returns_string(self): from pythainlp.transliterate.thaig2p_v2 import transliterate