diff --git a/src/powercontext/builtin/artifacts/search.py b/src/powercontext/builtin/artifacts/search.py index 2676a8325..77d7d3442 100644 --- a/src/powercontext/builtin/artifacts/search.py +++ b/src/powercontext/builtin/artifacts/search.py @@ -87,6 +87,90 @@ "those", "please", ]) +# The same rule for Chinese queries, whose function words Analyzer v1 cannot return as whole +# terms. The categories mirror the English list above: structural and aspectual particles, +# the copula, coverbs, conjunctions and adverbs, demonstrative and personal pronouns, +# interrogatives and sentence particles, and politeness markers. Negations are absent for the +# same reason they are absent above. Characters that also carry content are deliberately left +# out, because a lone character cannot tell the two uses apart: 地 and 得 are also 地址, 地方, +# 得到 and 取得, and 有, 会, 能, 要 and 帮 appear in 有效, 会议, 功能, 要求 and 帮助. +# +# Analyzer v1 treats every Han ideograph alike and the term filter runs on characters rather +# than on words, so the set has to cover the traditional writing of the same function words. +# Codepoints, not spellings, are what reach the filter: 麼 and 么 are different characters and +# a traditional query would otherwise keep raising its own admission bar. 著 is the one +# traditional form left out, on the same principle as 地 and 得 above: it is also the verb in +# 著名 and 著作, so a lone character cannot tell the particle from the content word. +_QUERY_FUNCTION_CHARACTERS = frozenset([ + # Aspectual and structural particles, the copula, and coverbs. + "的", + "了", + "着", + "过", + "過", + "之", + "是", + "在", + "从", + "從", + "向", + "对", + "對", + "给", + "給", + "被", + "把", + "让", + "讓", + "于", + "於", + "跟", + # Conjunctions and adverbs. + "为", + "為", + "和", + "与", + "與", + "或", + "及", + "并", + "並", + "也", + "都", + "就", + "还", + "還", + # Demonstrative, personal and interrogative pronouns. + "我", + "你", + "他", + "她", + "它", + "们", + "們", + "这", + "這", + "那", + "哪", + "此", + "其", + "什", + "么", + "麼", + "怎", + "谁", + "誰", + "何", + # Sentence particles and politeness markers. + "吗", + "嗎", + "呢", + "吧", + "啊", + "呀", + "请", + "請", +]) _QUOTED_QUERY_TEXT = re.compile(r"""(`[^`]*`|"[^"]*"|“[^”]*”|‘[^’]*’|(? str: return " ".join(term for term, _start, _end in analyze_text_with_spans(value)) +def _analyzed_character(code: str) -> str: + """Return the character an Analyzer v1 ``u_``/``b_`` component encodes, or an empty string.""" + + try: + return chr(int(code, 16)) + except ValueError: + return "" + + +def _is_cjk_generated_term(normalized: str, term: str, start: int) -> bool: + """Return whether Analyzer v1 generated one term from CJK source text. + + Analyzer v1 leaves Latin and underscore identifiers unchanged, so a literal query token such + as ``u_7684`` lands in the term list exactly like the token generated for the character 的. + The only thing that separates the two is provenance, which the span carries: ``flush_cjk`` + emits ``u_``/``b_`` terms over CJK runs, while ``flush_word`` emits verbatim terms over + alphanumeric runs. A term is therefore generated from Chinese text only when it has that + shape *and* its source span opens on a CJK character. + """ + + return term.startswith(("u_", "b_")) and start < len(normalized) and _is_cjk(normalized[start]) + + +def _is_query_function_term(term: str) -> bool: + """Return whether one analyzed query term is a function word for admission purposes. + + This is the historical whole-term rule, which applies to Latin terms. It deliberately does + not look at ``u_``/``b_`` terms: those are filtered by :func:`_is_cjk_function_term` only + when Analyzer v1 generated them from Chinese text, so a literal identifier that merely + looks like a Chinese token keeps its documented whole-term behavior. Anything unrecognized + stays evidence. + """ + + return term in _QUERY_FUNCTION_WORDS + + +def _is_cjk_function_term(term: str) -> bool: + """Return whether one CJK-generated term is a function word for admission purposes. + + Analyze Chinese terms only; call this exclusively for terms :func:`_is_cjk_generated_term` + accepted. Analyzer v1 does not segment Chinese: a Chinese run becomes one unigram per + character plus one bigram per adjacent pair, so a Chinese function word is never a single + term. A unigram is a function word exactly when its character is. A bigram is one only when + **both** of its characters are, because several listed characters also carry content — 为 in + 行为, 向 in 方向, 并 in 并发 — and a bigram is the only evidence those two-character + compounds have. Retiring every pair that merely touches a listed character would leave 行为 + with the single term ``u_884c``, and a query for the exact phrase would then be refused + against an entry that contains it. + + The boundary this leaves is deliberate and documented: inserting a function character can + still contribute one mixed bigram to the denominator, so 的 raises ``required_matches`` when + the characters around it carry content (``项目地址`` keeps 8 terms and needs 2, ``的项目地址`` + keeps 9 and needs 3). That is the ordinary coverage behaviour of any query that grows, which + the module documents above; removing it would require the segmentation this analyzer lacks. + """ + + if term.startswith("u_"): + return _analyzed_character(term[2:]) in _QUERY_FUNCTION_CHARACTERS + if term.startswith("b_"): + left, _, right = term[2:].partition("_") + return all( + character in _QUERY_FUNCTION_CHARACTERS + for character in (_analyzed_character(left), _analyzed_character(right)) + ) + return False + + def analyze_fts_query(value: str) -> str: """Extract lexical evidence terms without changing stored text or semantic queries. Recognize only standalone, generic English execution instructions accompanying other query content. Preserve quoted text, domain-specific directives and unknown phrasing. - Unquoted function words cannot supply the sole overlap during a relaxed recall-gate round. + Unquoted function words cannot supply the sole overlap during a relaxed recall-gate round, + in English or in Chinese, so a query cannot raise its own admission bar by adding them. + Filtering is per occurrence and decided by provenance, so a literal ``u_``/``b_`` token + keeps its whole-term behavior while the same string generated from Chinese text does not. """ content: list[str] = [] @@ -285,9 +439,20 @@ def analyze_fts_query(value: str) -> str: sentences = re.split(r"(?<=[.!?])\s+|\n+", part) content.extend(sentence for sentence in sentences if not _is_execution_sentence(sentence)) # A search consisting solely of a directive may be looking up that very policy. - terms = analyze_text(" ".join(content) if any(sentence.strip() for sentence in content) else value).split() + searched = " ".join(content) if any(sentence.strip() for sentence in content) else value + analyzed = analyze_text_with_spans(searched) + terms = [term for term, _start, _end in analyzed] if len(set(terms)) > _FTS_SHORT_QUERY_MAX_TERMS: - meaningful = [term for term in terms if term in quoted_terms or term not in _QUERY_FUNCTION_WORDS] + normalized = unicodedata.normalize("NFC", searched).casefold() + meaningful = [ + term + for term, start, _end in analyzed + if term in quoted_terms + or not ( + _is_query_function_term(term) + or (_is_cjk_generated_term(normalized, term, start) and _is_cjk_function_term(term)) + ) + ] if meaningful: terms = meaningful return " ".join(terms) diff --git a/tests/builtin/runtime/test_recall_sufficiency.py b/tests/builtin/runtime/test_recall_sufficiency.py index fea4bd689..f758b3142 100644 --- a/tests/builtin/runtime/test_recall_sufficiency.py +++ b/tests/builtin/runtime/test_recall_sufficiency.py @@ -30,6 +30,8 @@ AdmissionCounts, AdmissionFloor, admits_fts_text, + analyze_fts_query, + analyze_text, fts_query_requirements, ) from powercontext.builtin.artifacts.topic_memory import TopicMemorySearchHit @@ -232,6 +234,190 @@ def test_lower_floor_admits_a_vector_candidate_the_default_rejects() -> None: assert admit_vector_candidates(candidates, admission=round_one) == candidates +# ── Query normalization: function words in English and Chinese ────────────────────────────── + +# The entry text and the query pair reported in #1912. The first form was rejected and the form +# with a leading 的 was admitted, because 的 is in the entry and supplied the third matching term. +_ISSUE_1912_ENTRY = "本项目的 PowerContext Server 固定在 127.0.0.1:8000；测试写入。" # noqa: RUF001 +_ISSUE_1912_QUESTION = "PowerContext Server 地址是什么？" # noqa: RUF001 + + +def test_reported_chinese_function_word_prefix_admits_consistently() -> None: + """Regression test for #1912: a leading function word may not flip admission. + + Analyzer v1 returns one unigram per Chinese character plus one bigram per adjacent pair, so + a Chinese function word is never a single term and the English whole-term filter never saw + it. Counting 的 as query evidence let a query cross the coverage floor only because the + entry also contained 的. + """ + + assert admits_fts_text(_ISSUE_1912_QUESTION, _ISSUE_1912_ENTRY) is True + assert admits_fts_text(f"的 {_ISSUE_1912_QUESTION}", _ISSUE_1912_ENTRY) is True + + +@pytest.mark.parametrize("suffix", ["的", "了", "吗", "我的", "是的", "它们的"]) +def test_chinese_function_words_do_not_raise_the_admission_bar(suffix: str) -> None: + """Adding function words cannot change either side of the comparison.""" + + assert fts_query_requirements(f"{_ISSUE_1912_QUESTION}{suffix}") == fts_query_requirements(_ISSUE_1912_QUESTION) + + +@pytest.mark.parametrize( + ("hanzi", "terms"), + [ + ("地址", ("u_5730", "u_5740", "b_5730_5740")), + ("地方", ("u_5730", "u_65b9", "b_5730_65b9")), + ("得到", ("u_5f97", "u_5230", "b_5f97_5230")), + ("有效", ("u_6709", "u_6548", "b_6709_6548")), + ("会议", ("u_4f1a", "u_8bae", "b_4f1a_8bae")), + ("帮助", ("u_5e2e", "u_52a9", "b_5e2e_52a9")), + ("要求", ("u_8981", "u_6c42", "b_8981_6c42")), + ], +) +def test_chinese_content_characters_stay_query_evidence(hanzi: str, terms: tuple[str, ...]) -> None: + """Characters that also carry content must survive the function-word filter.""" + + analyzed = analyze_fts_query(f"PowerContext Server {hanzi}").split() + assert set(terms) <= set(analyzed) + + +def test_chinese_bigram_drops_only_when_both_characters_are_function_words() -> None: + """A pair with one content character stays evidence, so 行为 and 方向 keep their compound. + + Reported by @hidb4ai with real SQLite Memory search at 207a63d3: retiring every pair that + merely touched a listed character left 行为 with the single term ``u_884c``, and the exact + phrase query was then refused against an entry that contained it. 址是 keeps its pair for the + same reason; only the all-function pairs 是什 and 什么 leave the term list. + """ + + analyzed = set(analyze_fts_query(_ISSUE_1912_QUESTION).split()) + assert {"u_5730", "u_5740", "b_5730_5740", "b_5740_662f"} <= analyzed + assert not {"b_662f_4ec0", "b_4ec0_4e48"} & analyzed + + +@pytest.mark.parametrize("compound", ["行为", "方向", "并发"]) +def test_content_compound_that_contains_a_listed_character_stays_recallable(compound: str) -> None: + """A listed character inside a content word may not cost the compound its evidence. + + Reported by @hidb4ai with real SQLite Memory search at 207a63d3: 行为, 方向 and 并发 were each + returned by the exact-phrase query before the boundary rule widened, and each stopped being + returned after it, because the unigram and the bigram of the listed character were withdrawn + together and one match remained against a requirement of two. + """ + + assert admits_fts_text(f"alpha beta gamma {compound}", compound) is True + + +def test_mixed_boundary_bigram_keeps_one_pair_and_pins_the_shift() -> None: + """Inserting 的 beside content keeps one mixed pair, and that shift is deliberate. + + Reported on this PR: `alpha 项目地址` keeps 8 terms and needs 2, while `alpha 的项目地址` + keeps 9 and needs 3, because 的项 mixes the function character with a content one. Retiring + that pair as well is what cost 行为, 方向 and 并发 their only evidence, so the shift is kept + and pinned here rather than traded for a recall failure on real compounds. + """ + + without_function_word = fts_query_requirements("alpha 项目地址") + with_function_word = fts_query_requirements("alpha 的项目地址") + assert len(without_function_word[0]) == 8 + assert without_function_word[1] == 2 + assert len(with_function_word[0]) == 9 + assert with_function_word[1] == 3 + + +def test_polite_long_query_keeps_the_documented_coverage_rejection() -> None: + """A query whose extra characters carry content still fails the coverage rule. + + Reported by Copilot on this PR: once the mixed pairs were removed the query kept 8 terms and + needed 2, so `powercontext` and `server` alone admitted it, although this pull request + documents it as rejected. With the pair rule restored it keeps 11 terms and needs 3, so the + documented rejection holds while the shorter question is still admitted. + """ + + entry = "PowerContext Server 固定 127.0.0.1" + assert admits_fts_text("请告诉我 PowerContext Server 地址是什么？", entry) is False # noqa: RUF001 + assert admits_fts_text("PowerContext Server 地址是什么？", entry) is True # noqa: RUF001 + + +@pytest.mark.parametrize( + ("simplified", "traditional"), + [ + ("么", "麼"), + ("谁", "誰"), + ("吗", "嗎"), + ("们", "們"), + ("这", "這"), + ("还", "還"), + ("请", "請"), + ("对", "對"), + ("给", "給"), + ("让", "讓"), + ("从", "從"), + ("于", "於"), + ("与", "與"), + ("为", "為"), + ("并", "並"), + ("过", "過"), + ], +) +def test_traditional_chinese_function_characters_share_the_simplified_threshold( + simplified: str, traditional: str +) -> None: + """Analyzer v1 sees codepoints, so each traditional form needs its own entry in the set. + + Reported on this PR: 地址是什麼 kept u_9ebc and b_4ec0_9ebc, so a traditional query could + still raise its own admission bar and reproduce the form-sensitive recall failure. The + threshold is what must match; a trailing traditional character can still add one mixed + bigram, exactly as its simplified counterpart does. + """ + + assert fts_query_requirements(f"PowerContext Server 地址是{traditional}") == fts_query_requirements( + f"PowerContext Server 地址是{simplified}" + ) + trailing = fts_query_requirements(f"PowerContext Server 地址{traditional}") + base = fts_query_requirements("PowerContext Server 地址") + assert trailing[1] == base[1] + assert len(trailing[0]) <= len(base[0]) + 1 + + +@pytest.mark.parametrize("literal", ["u_7684", "b_7684_9879"]) +def test_literal_analyzer_shaped_identifier_stays_query_evidence(literal: str) -> None: + """A literal analyzer-shaped token is not Chinese text and keeps whole-term behavior. + + Reported by @hidb4ai with real SQLite Memory search at 59530d16: stored text `alpha u_7684` + queried with `alpha beta gamma u_7684` returned nothing, because the filter decoded the + literal identifier as the character 的 and withdrew it from both MATCH retrieval and + admission, dropping the overlap from two terms to one against a requirement of two. + """ + + requirements = fts_query_requirements(f"alpha beta gamma {literal}") + assert literal in requirements[0] + stored = set(analyze_text(f"alpha {literal}").split()) + assert len(stored & set(requirements[0])) >= requirements[1] + + +def test_literal_identifier_survives_a_query_full_of_chinese_function_words() -> None: + """One query carries both readings: generated function tokens go, the literal one stays.""" + + analyzed = analyze_fts_query("PowerContext Server 地址是什么？ u_7684").split() # noqa: RUF001 + assert "u_7684" in analyzed + assert not {"u_662f", "u_4ec0", "u_4e48", "b_662f_4ec0", "b_4ec0_4e48"} & set(analyzed) + + +def test_quoted_chinese_function_text_stays_query_evidence() -> None: + """Quoting a Chinese function word still asks for that word explicitly.""" + + assert "u_7684" in analyze_fts_query(f'PowerContext Server {_ISSUE_1912_QUESTION} "的"').split() + + +def test_query_normalization_leaves_the_chinese_projection_unchanged() -> None: + """The filter is query-only: persisted Analyzer v1 projections keep every term.""" + + stored = set(analyze_text("地址是什么").split()) + assert {"u_662f", "u_4ec0", "u_4e48", "b_662f_4ec0", "b_4ec0_4e48"} <= stored + assert analyze_text("的").split() == ["u_7684"] + + # ── Gate verdict order ──────────────────────────────────────────────────────────────────────