From 317d072132747f783b8960d8c9c3a5b5639ab319 Mon Sep 17 00:00:00 2001 From: JasonDeng <15301580353@163.com> Date: Tue, 13 Oct 2026 14:34:14 +0800 Subject: [PATCH] fix(search): keep Chinese function words out of lexical query admission MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit analyze_fts_query documents that unquoted function words cannot supply the sole overlap during a relaxed recall-gate round, but _QUERY_FUNCTION_WORDS is Latin-only, so that filter never applied to a Chinese query. Analyzer v1 does not segment Chinese: a run becomes one unigram per character plus one bigram per adjacent pair. Every function character a Chinese query adds joins the query own denominator, required_matches = ceil(len(query_terms) * 0.25), and can also join the numerator when the stored entry happens to contain that character. Against the entry text reported in the issue, on ad75ccf2, "PowerContext Server 地址是什么？" has 11 terms, required 3 and matched 2, so it is rejected, while "的 PowerContext Server 地址是什么？" has 12 terms, required 3 and matched 3, so it is admitted. The leading 的 matched the 的 in the entry and crossed a threshold the query itself had raised, which is why one extra character flipped the result. _QUERY_FUNCTION_CHARACTERS mirrors the categories of _QUERY_FUNCTION_WORDS and also covers the traditional writing of the same characters, because Analyzer v1 treats every Han ideograph alike and codepoints are what reach the filter: 麼 and 么 are different characters, so a simplified-only set would leave a traditional query unguarded. Negations are absent for the same reason they are absent in the English list. Characters that also carry content stay out, because a lone character cannot distinguish 地址, 地方, 得到 and 取得 from the particle 地, and 有, 会, 能, 要 and 帮 appear in 有效, 会议, 功能, 要求 and 帮助; 著 is left out on the same principle, being also the verb in 著名 and 著作. Filtering is per occurrence and decided by provenance rather than by the shape of the token. A u_/b_ term counts as Chinese only when its source span opens on a CJK character, because flush_cjk emits those terms over CJK runs while flush_word emits verbatim terms over alphanumeric runs. A literal identifier such as u_7684 therefore keeps its documented whole-term behavior, while the same string generated from 的 is filtered. A bigram counts as a function word only when both of its characters are one. Several listed characters also carry content (为 in 行为, 向 in 方向, 并 in 并发), and a bigram is the only evidence a two-character compound has, so retiring every pair that merely touches a listed character removes the evidence rather than the function word. Measured with real SQLite Memory search, 行为, 方向 and 并发 each lost their pair and stopped being returned for the exact phrase; under this rule all three are returned again. The change is query-side only. analyze_text is untouched, so persisted Analyzer v1 projections keep every term, and no threshold, constant or fusion input changes. The short question from the report is admitted, the long form this pull request documents as rejected stays rejected, and appending a pure function word to a query leaves the threshold unchanged in both writings. The boundary the pair rule leaves is deliberate. An inserted function character beside content can still contribute one mixed bigram, so 的 raises required_matches where the characters around it carry content: "alpha 项目地址" keeps 8 terms and needs 2, while "alpha 的项目地址" keeps 9 and needs 3. Closing that would need the segmentation Analyzer v1 does not have, and the attempt is what cost the three compounds above. Tested: pytest tests/builtin/runtime/test_recall_sufficiency.py -q (109 passed) and together with tests/e2e/test_recall_sufficiency_gate.py (146 passed); pytest tests/builtin/artifacts/experience/test_search.py tests/builtin/artifacts/topic_memory/test_search.py tests/builtin/persistence/test_topic_memory.py tests/builtin/persistence/test_topic_memory_integrity.py -q (126 passed, 26 skipped); ruff format --check, ruff check and ty check on both changed files. With search.py restored to ad75ccf2, 27 of these tests fail; with it restored to the previous commit of this branch, 6 fail, including every assertion added for the compounds above. Refs #1912 --- src/powercontext/builtin/artifacts/search.py | 171 +++++++++++++++- .../runtime/test_recall_sufficiency.py | 186 ++++++++++++++++++ 2 files changed, 354 insertions(+), 3 deletions(-) diff --git a/src/powercontext/builtin/artifacts/search.py b/src/powercontext/builtin/artifacts/search.py index 2676a83251..77d7d34421 100644 --- a/src/powercontext/builtin/artifacts/search.py +++ b/src/powercontext/builtin/artifacts/search.py @@ -87,6 +87,90 @@ "those", "please", ]) +# The same rule for Chinese queries, whose function words Analyzer v1 cannot return as whole +# terms. The categories mirror the English list above: structural and aspectual particles, +# the copula, coverbs, conjunctions and adverbs, demonstrative and personal pronouns, +# interrogatives and sentence particles, and politeness markers. Negations are absent for the +# same reason they are absent above. Characters that also carry content are deliberately left +# out, because a lone character cannot tell the two uses apart: 地 and 得 are also 地址, 地方, +# 得到 and 取得, and 有, 会, 能, 要 and 帮 appear in 有效, 会议, 功能, 要求 and 帮助. +# +# Analyzer v1 treats every Han ideograph alike and the term filter runs on characters rather +# than on words, so the set has to cover the traditional writing of the same function words. +# Codepoints, not spellings, are what reach the filter: 麼 and 么 are different characters and +# a traditional query would otherwise keep raising its own admission bar. 著 is the one +# traditional form left out, on the same principle as 地 and 得 above: it is also the verb in +# 著名 and 著作, so a lone character cannot tell the particle from the content word. +_QUERY_FUNCTION_CHARACTERS = frozenset([ + # Aspectual and structural particles, the copula, and coverbs. + "的", + "了", + "着", + "过", + "過", + "之", + "是", + "在", + "从", + "從", + "向", + "对", + "對", + "给", + "給", + "被", + "把", + "让", + "讓", + "于", + "於", + "跟", + # Conjunctions and adverbs. + "为", + "為", + "和", + "与", + "與", + "或", + "及", + "并", + "並", + "也", + "都", + "就", + "还", + "還", + # Demonstrative, personal and interrogative pronouns. + "我", + "你", + "他", + "她", + "它", + "们", + "們", + "这", + "這", + "那", + "哪", + "此", + "其", + "什", + "么", + "麼", + "怎", + "谁", + "誰", + "何", + # Sentence particles and politeness markers. + "吗", + "嗎", + "呢", + "吧", + "啊", + "呀", + "请", + "請", +]) _QUOTED_QUERY_TEXT = re.compile(r"""(`[^`]*`|"[^"]*"|“[^”]*”|‘[^’]*’|(? str: return " ".join(term for term, _start, _end in analyze_text_with_spans(value)) +def _analyzed_character(code: str) -> str: + """Return the character an Analyzer v1 ``u_``/``b_`` component encodes, or an empty string.""" + + try: + return chr(int(code, 16)) + except ValueError: + return "" + + +def _is_cjk_generated_term(normalized: str, term: str, start: int) -> bool: + """Return whether Analyzer v1 generated one term from CJK source text. + + Analyzer v1 leaves Latin and underscore identifiers unchanged, so a literal query token such + as ``u_7684`` lands in the term list exactly like the token generated for the character 的. + The only thing that separates the two is provenance, which the span carries: ``flush_cjk`` + emits ``u_``/``b_`` terms over CJK runs, while ``flush_word`` emits verbatim terms over + alphanumeric runs. A term is therefore generated from Chinese text only when it has that + shape *and* its source span opens on a CJK character. + """ + + return term.startswith(("u_", "b_")) and start < len(normalized) and _is_cjk(normalized[start]) + + +def _is_query_function_term(term: str) -> bool: + """Return whether one analyzed query term is a function word for admission purposes. + + This is the historical whole-term rule, which applies to Latin terms. It deliberately does + not look at ``u_``/``b_`` terms: those are filtered by :func:`_is_cjk_function_term` only + when Analyzer v1 generated them from Chinese text, so a literal identifier that merely + looks like a Chinese token keeps its documented whole-term behavior. Anything unrecognized + stays evidence. + """ + + return term in _QUERY_FUNCTION_WORDS + + +def _is_cjk_function_term(term: str) -> bool: + """Return whether one CJK-generated term is a function word for admission purposes. + + Analyze Chinese terms only; call this exclusively for terms :func:`_is_cjk_generated_term` + accepted. Analyzer v1 does not segment Chinese: a Chinese run becomes one unigram per + character plus one bigram per adjacent pair, so a Chinese function word is never a single + term. A unigram is a function word exactly when its character is. A bigram is one only when + **both** of its characters are, because several listed characters also carry content — 为 in + 行为, 向 in 方向, 并 in 并发 — and a bigram is the only evidence those two-character + compounds have. Retiring every pair that merely touches a listed character would leave 行为 + with the single term ``u_884c``, and a query for the exact phrase would then be refused + against an entry that contains it. + + The boundary this leaves is deliberate and documented: inserting a function character can + still contribute one mixed bigram to the denominator, so 的 raises ``required_matches`` when + the characters around it carry content (``项目地址`` keeps 8 terms and needs 2, ``的项目地址`` + keeps 9 and needs 3). That is the ordinary coverage behaviour of any query that grows, which + the module documents above; removing it would require the segmentation this analyzer lacks. + """ + + if term.startswith("u_"): + return _analyzed_character(term[2:]) in _QUERY_FUNCTION_CHARACTERS + if term.startswith("b_"): + left, _, right = term[2:].partition("_") + return all( + character in _QUERY_FUNCTION_CHARACTERS + for character in (_analyzed_character(left), _analyzed_character(right)) + ) + return False + + def analyze_fts_query(value: str) -> str: """Extract lexical evidence terms without changing stored text or semantic queries. Recognize only standalone, generic English execution instructions accompanying other query content. Preserve quoted text, domain-specific directives and unknown phrasing. - Unquoted function words cannot supply the sole overlap during a relaxed recall-gate round. + Unquoted function words cannot supply the sole overlap during a relaxed recall-gate round, + in English or in Chinese, so a query cannot raise its own admission bar by adding them. + Filtering is per occurrence and decided by provenance, so a literal ``u_``/``b_`` token + keeps its whole-term behavior while the same string generated from Chinese text does not. """ content: list[str] = [] @@ -285,9 +439,20 @@ def analyze_fts_query(value: str) -> str: sentences = re.split(r"(?<=[.!?])\s+|\n+", part) content.extend(sentence for sentence in sentences if not _is_execution_sentence(sentence)) # A search consisting solely of a directive may be looking up that very policy. - terms = analyze_text(" ".join(content) if any(sentence.strip() for sentence in content) else value).split() + searched = " ".join(content) if any(sentence.strip() for sentence in content) else value + analyzed = analyze_text_with_spans(searched) + terms = [term for term, _start, _end in analyzed] if len(set(terms)) > _FTS_SHORT_QUERY_MAX_TERMS: - meaningful = [term for term in terms if term in quoted_terms or term not in _QUERY_FUNCTION_WORDS] + normalized = unicodedata.normalize("NFC", searched).casefold() + meaningful = [ + term + for term, start, _end in analyzed + if term in quoted_terms + or not ( + _is_query_function_term(term) + or (_is_cjk_generated_term(normalized, term, start) and _is_cjk_function_term(term)) + ) + ] if meaningful: terms = meaningful return " ".join(terms) diff --git a/tests/builtin/runtime/test_recall_sufficiency.py b/tests/builtin/runtime/test_recall_sufficiency.py index fea4bd689d..f758b31427 100644 --- a/tests/builtin/runtime/test_recall_sufficiency.py +++ b/tests/builtin/runtime/test_recall_sufficiency.py @@ -30,6 +30,8 @@ AdmissionCounts, AdmissionFloor, admits_fts_text, + analyze_fts_query, + analyze_text, fts_query_requirements, ) from powercontext.builtin.artifacts.topic_memory import TopicMemorySearchHit @@ -232,6 +234,190 @@ def test_lower_floor_admits_a_vector_candidate_the_default_rejects() -> None: assert admit_vector_candidates(candidates, admission=round_one) == candidates +# ── Query normalization: function words in English and Chinese ────────────────────────────── + +# The entry text and the query pair reported in #1912. The first form was rejected and the form +# with a leading 的 was admitted, because 的 is in the entry and supplied the third matching term. +_ISSUE_1912_ENTRY = "本项目的 PowerContext Server 固定在 127.0.0.1:8000；测试写入。" # noqa: RUF001 +_ISSUE_1912_QUESTION = "PowerContext Server 地址是什么？" # noqa: RUF001 + + +def test_reported_chinese_function_word_prefix_admits_consistently() -> None: + """Regression test for #1912: a leading function word may not flip admission. + + Analyzer v1 returns one unigram per Chinese character plus one bigram per adjacent pair, so + a Chinese function word is never a single term and the English whole-term filter never saw + it. Counting 的 as query evidence let a query cross the coverage floor only because the + entry also contained 的. + """ + + assert admits_fts_text(_ISSUE_1912_QUESTION, _ISSUE_1912_ENTRY) is True + assert admits_fts_text(f"的 {_ISSUE_1912_QUESTION}", _ISSUE_1912_ENTRY) is True + + +@pytest.mark.parametrize("suffix", ["的", "了", "吗", "我的", "是的", "它们的"]) +def test_chinese_function_words_do_not_raise_the_admission_bar(suffix: str) -> None: + """Adding function words cannot change either side of the comparison.""" + + assert fts_query_requirements(f"{_ISSUE_1912_QUESTION}{suffix}") == fts_query_requirements(_ISSUE_1912_QUESTION) + + +@pytest.mark.parametrize( + ("hanzi", "terms"), + [ + ("地址", ("u_5730", "u_5740", "b_5730_5740")), + ("地方", ("u_5730", "u_65b9", "b_5730_65b9")), + ("得到", ("u_5f97", "u_5230", "b_5f97_5230")), + ("有效", ("u_6709", "u_6548", "b_6709_6548")), + ("会议", ("u_4f1a", "u_8bae", "b_4f1a_8bae")), + ("帮助", ("u_5e2e", "u_52a9", "b_5e2e_52a9")), + ("要求", ("u_8981", "u_6c42", "b_8981_6c42")), + ], +) +def test_chinese_content_characters_stay_query_evidence(hanzi: str, terms: tuple[str, ...]) -> None: + """Characters that also carry content must survive the function-word filter.""" + + analyzed = analyze_fts_query(f"PowerContext Server {hanzi}").split() + assert set(terms) <= set(analyzed) + + +def test_chinese_bigram_drops_only_when_both_characters_are_function_words() -> None: + """A pair with one content character stays evidence, so 行为 and 方向 keep their compound. + + Reported by @hidb4ai with real SQLite Memory search at 207a63d3: retiring every pair that + merely touched a listed character left 行为 with the single term ``u_884c``, and the exact + phrase query was then refused against an entry that contained it. 址是 keeps its pair for the + same reason; only the all-function pairs 是什 and 什么 leave the term list. + """ + + analyzed = set(analyze_fts_query(_ISSUE_1912_QUESTION).split()) + assert {"u_5730", "u_5740", "b_5730_5740", "b_5740_662f"} <= analyzed + assert not {"b_662f_4ec0", "b_4ec0_4e48"} & analyzed + + +@pytest.mark.parametrize("compound", ["行为", "方向", "并发"]) +def test_content_compound_that_contains_a_listed_character_stays_recallable(compound: str) -> None: + """A listed character inside a content word may not cost the compound its evidence. + + Reported by @hidb4ai with real SQLite Memory search at 207a63d3: 行为, 方向 and 并发 were each + returned by the exact-phrase query before the boundary rule widened, and each stopped being + returned after it, because the unigram and the bigram of the listed character were withdrawn + together and one match remained against a requirement of two. + """ + + assert admits_fts_text(f"alpha beta gamma {compound}", compound) is True + + +def test_mixed_boundary_bigram_keeps_one_pair_and_pins_the_shift() -> None: + """Inserting 的 beside content keeps one mixed pair, and that shift is deliberate. + + Reported on this PR: `alpha 项目地址` keeps 8 terms and needs 2, while `alpha 的项目地址` + keeps 9 and needs 3, because 的项 mixes the function character with a content one. Retiring + that pair as well is what cost 行为, 方向 and 并发 their only evidence, so the shift is kept + and pinned here rather than traded for a recall failure on real compounds. + """ + + without_function_word = fts_query_requirements("alpha 项目地址") + with_function_word = fts_query_requirements("alpha 的项目地址") + assert len(without_function_word[0]) == 8 + assert without_function_word[1] == 2 + assert len(with_function_word[0]) == 9 + assert with_function_word[1] == 3 + + +def test_polite_long_query_keeps_the_documented_coverage_rejection() -> None: + """A query whose extra characters carry content still fails the coverage rule. + + Reported by Copilot on this PR: once the mixed pairs were removed the query kept 8 terms and + needed 2, so `powercontext` and `server` alone admitted it, although this pull request + documents it as rejected. With the pair rule restored it keeps 11 terms and needs 3, so the + documented rejection holds while the shorter question is still admitted. + """ + + entry = "PowerContext Server 固定 127.0.0.1" + assert admits_fts_text("请告诉我 PowerContext Server 地址是什么？", entry) is False # noqa: RUF001 + assert admits_fts_text("PowerContext Server 地址是什么？", entry) is True # noqa: RUF001 + + +@pytest.mark.parametrize( + ("simplified", "traditional"), + [ + ("么", "麼"), + ("谁", "誰"), + ("吗", "嗎"), + ("们", "們"), + ("这", "這"), + ("还", "還"), + ("请", "請"), + ("对", "對"), + ("给", "給"), + ("让", "讓"), + ("从", "從"), + ("于", "於"), + ("与", "與"), + ("为", "為"), + ("并", "並"), + ("过", "過"), + ], +) +def test_traditional_chinese_function_characters_share_the_simplified_threshold( + simplified: str, traditional: str +) -> None: + """Analyzer v1 sees codepoints, so each traditional form needs its own entry in the set. + + Reported on this PR: 地址是什麼 kept u_9ebc and b_4ec0_9ebc, so a traditional query could + still raise its own admission bar and reproduce the form-sensitive recall failure. The + threshold is what must match; a trailing traditional character can still add one mixed + bigram, exactly as its simplified counterpart does. + """ + + assert fts_query_requirements(f"PowerContext Server 地址是{traditional}") == fts_query_requirements( + f"PowerContext Server 地址是{simplified}" + ) + trailing = fts_query_requirements(f"PowerContext Server 地址{traditional}") + base = fts_query_requirements("PowerContext Server 地址") + assert trailing[1] == base[1] + assert len(trailing[0]) <= len(base[0]) + 1 + + +@pytest.mark.parametrize("literal", ["u_7684", "b_7684_9879"]) +def test_literal_analyzer_shaped_identifier_stays_query_evidence(literal: str) -> None: + """A literal analyzer-shaped token is not Chinese text and keeps whole-term behavior. + + Reported by @hidb4ai with real SQLite Memory search at 59530d16: stored text `alpha u_7684` + queried with `alpha beta gamma u_7684` returned nothing, because the filter decoded the + literal identifier as the character 的 and withdrew it from both MATCH retrieval and + admission, dropping the overlap from two terms to one against a requirement of two. + """ + + requirements = fts_query_requirements(f"alpha beta gamma {literal}") + assert literal in requirements[0] + stored = set(analyze_text(f"alpha {literal}").split()) + assert len(stored & set(requirements[0])) >= requirements[1] + + +def test_literal_identifier_survives_a_query_full_of_chinese_function_words() -> None: + """One query carries both readings: generated function tokens go, the literal one stays.""" + + analyzed = analyze_fts_query("PowerContext Server 地址是什么？ u_7684").split() # noqa: RUF001 + assert "u_7684" in analyzed + assert not {"u_662f", "u_4ec0", "u_4e48", "b_662f_4ec0", "b_4ec0_4e48"} & set(analyzed) + + +def test_quoted_chinese_function_text_stays_query_evidence() -> None: + """Quoting a Chinese function word still asks for that word explicitly.""" + + assert "u_7684" in analyze_fts_query(f'PowerContext Server {_ISSUE_1912_QUESTION} "的"').split() + + +def test_query_normalization_leaves_the_chinese_projection_unchanged() -> None: + """The filter is query-only: persisted Analyzer v1 projections keep every term.""" + + stored = set(analyze_text("地址是什么").split()) + assert {"u_662f", "u_4ec0", "u_4e48", "b_662f_4ec0", "b_4ec0_4e48"} <= stored + assert analyze_text("的").split() == ["u_7684"] + + # ── Gate verdict order ──────────────────────────────────────────────────────────────────────