From 9520724c9a576229de6fd8024e19fe40627e9230 Mon Sep 17 00:00:00 2001 From: Chojan Shang Date: Tue, 13 Oct 2026 20:04:42 +0800 Subject: [PATCH 1/9] feat(search): unify word analysis across backends --- docs/en/docs/workflows/search-artifacts.md | 12 + docs/zh/docs/workflows/search-artifacts.md | 11 + pyproject.toml | 1 + .../builtin/artifacts/lexical_data/LICENSE | 21 + .../builtin/artifacts/lexical_data/README.md | 11 + .../builtin/artifacts/lexical_data/en.txt | 228 +++++ .../builtin/artifacts/lexical_data/zh.txt | 794 ++++++++++++++++++ src/powercontext/builtin/artifacts/search.py | 236 +++--- .../builtin/artifacts/topic_memory/fusion.py | 4 +- .../artifacts/topic_memory/relatedness.py | 4 +- .../builtin/persistence/experience_index.py | 18 +- .../builtin/persistence/lexical_projection.py | 59 ++ .../persistence/sqlite/experience_index.py | 50 +- .../builtin/runtime/application.py | 9 +- .../builtin/runtime/composition.py | 7 + .../runtime/topic_memory_processing.py | 4 +- .../artifacts/test_lexical_analysis.py | 93 ++ tests/e2e/test_lexical_projection_upgrade.py | 187 +++++ tests/e2e/test_unified_lexical_search.py | 134 +++ uv.lock | 12 + 20 files changed, 1740 insertions(+), 155 deletions(-) create mode 100644 src/powercontext/builtin/artifacts/lexical_data/LICENSE create mode 100644 src/powercontext/builtin/artifacts/lexical_data/README.md create mode 100644 src/powercontext/builtin/artifacts/lexical_data/en.txt create mode 100644 src/powercontext/builtin/artifacts/lexical_data/zh.txt create mode 100644 src/powercontext/builtin/persistence/lexical_projection.py create mode 100644 tests/builtin/artifacts/test_lexical_analysis.py create mode 100644 tests/e2e/test_lexical_projection_upgrade.py create mode 100644 tests/e2e/test_unified_lexical_search.py diff --git a/docs/en/docs/workflows/search-artifacts.md b/docs/en/docs/workflows/search-artifacts.md index cc3eba93a8..1a39e1f58e 100644 --- a/docs/en/docs/workflows/search-artifacts.md +++ b/docs/en/docs/workflows/search-artifacts.md @@ -83,6 +83,18 @@ The existing retrieval page does not retain every raw channel score, so `include with `artifact_search_not_supported` and field `include_scores`. Other Family-specific controls such as `admission`, `fusion`, `min_score`, and `rerank` are not accepted by this adapter. +## Shared word analysis + +All built-in searchable Families use Jieba's packaged dictionary for Chinese with `HMM=False`, while Latin words +and identifiers remain whole. SQLite, seekdb and OceanBase rank the same backend-safe terms. Unquoted queries use +bundled Snowball English and stopwords-iso Chinese stopwords; negations remain evidence. Documents retain all words, +and quotations retain stopwords without requesting phrase matching. Admission and ranking controls keep their defaults. + +Record and projection formats remain v1. Stop old Server and Worker processes before upgrading. Normal initialization +compares derived words and updates changed rows, preserving content, history, identities and vectors. Compatible +projections and FTS helpers are reused; no extra database table or maintenance command is needed. Startup comparison +reads current searchable projections, with cost proportional to their text size. + ## Experience and Skill text search Both Families search one `text` channel. Their admission fields are: diff --git a/docs/zh/docs/workflows/search-artifacts.md b/docs/zh/docs/workflows/search-artifacts.md index 4be38964dc..4c3fa50385 100644 --- a/docs/zh/docs/workflows/search-artifacts.md +++ b/docs/zh/docs/workflows/search-artifacts.md @@ -80,6 +80,17 @@ Atomic 接受 `query`、`limit`、`mode`、`filters` 和 `include_scores: false` `artifact_search_not_supported`，字段为 `include_scores`。此适配层不接受 `admission`、`fusion`、 `min_score`、`rerank` 等其他 Family 参数。 +## 统一分词 + +内置可检索 Family 通过 Jieba 随包词典对中文分词，关闭 HMM，保留完整的拉丁词和标识符。 +SQLite、seekdb 和 OceanBase 对相同的后端安全词元排序。未加引号的查询使用随包的 Snowball 英文和 +stopwords-iso 中文停止词表，但保留否定词。文档保留所有词；引号保留停止词，不表示短语匹配。 +准入和排序参数沿用现有默认值。 + +记录和投影格式保持 v1。升级前停止旧 Server 和 Worker 进程。正常初始化比较派生词元，只更新变化的行， +保留正文、历史、身份和向量。已兼容的投影和 FTS 辅助索引直接沿用，无需新增数据库表或维护命令。 +初始化会读取当前可检索投影，成本随正文规模增长。 + ## Experience 和 Skill 文本检索 这两个 Family 都使用一个 `text` 通道，支持以下准入字段： diff --git a/pyproject.toml b/pyproject.toml index 0ea7d7f720..ac43c15148 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,6 +51,7 @@ builtin = [ "apscheduler>=3.11,<4", "filelock>=3.16,<4", "jsonschema>=4.23,<5", + "jieba==0.42.1", "packaging>=24,<27", "pydantic-ai-slim[anthropic,openai]>=2.27.1,<3", "pydantic-settings>=2.7,<3", diff --git a/src/powercontext/builtin/artifacts/lexical_data/LICENSE b/src/powercontext/builtin/artifacts/lexical_data/LICENSE new file mode 100644 index 0000000000..866a3b472e --- /dev/null +++ b/src/powercontext/builtin/artifacts/lexical_data/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2016 Gene Diaz + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/src/powercontext/builtin/artifacts/lexical_data/README.md b/src/powercontext/builtin/artifacts/lexical_data/README.md new file mode 100644 index 0000000000..40cc124fce --- /dev/null +++ b/src/powercontext/builtin/artifacts/lexical_data/README.md @@ -0,0 +1,11 @@ +# Bundled query stopwords + +`en.txt` is the Snowball English list from `raw/snowball-tartarus.txt` in +[stopwords-iso/stopwords-en](https://github.com/stopwords-iso/stopwords-en/tree/ccc8898188850d8fb019d5f69c14a6635c3bd115). +`zh.txt` is `stopwords-zh.txt` from +[stopwords-iso/stopwords-zh](https://github.com/stopwords-iso/stopwords-zh/tree/0f62ec9b9061b9142ed87fa6b39d386945c7cf47). +Both are redistributed under the accompanying MIT license. The files retain their source content. + +The analyzer applies these lists only to unquoted query words and explicitly retains negation words. +Document words and their frequencies are retained. Stopword updates do not change indexed tokens and require no index refresh. +Chinese segmentation uses Jieba 0.42.1's packaged dictionary in accurate mode with `HMM=False`. diff --git a/src/powercontext/builtin/artifacts/lexical_data/en.txt b/src/powercontext/builtin/artifacts/lexical_data/en.txt new file mode 100644 index 0000000000..114db4f68b --- /dev/null +++ b/src/powercontext/builtin/artifacts/lexical_data/en.txt @@ -0,0 +1,228 @@ +i +me +my +myself +we +us +our +ours +ourselves +you +your +yours +yourself +yourselves +he +him +his +himself +she +her +hers +herself +it +its +itself +they +them +their +theirs +themselves +what +which +who +whom +this +that +these +those +am +is +are +was +were +be +been +being +have +has +had +having +do +does +did +doing +will +would +shall +should +can +could +may +might +must +ought +i'm +you're +he's +she's +it's +we're +they're +i've +you've +we've +they've +i'd +you'd +he'd +she'd +we'd +they'd +i'll +you'll +he'll +she'll +we'll +they'll +isn't +aren't +wasn't +weren't +hasn't +haven't +hadn't +doesn't +don't +didn't +won't +wouldn't +shan't +shouldn't +can't +cannot +couldn't +mustn't +let's +that's +who's +what's +here's +there's +when's +where's +why's +how's +daren't +needn't +doubtful +oughtn't +mightn't +a +an +the +and +but +if +or +because +as +until +while +of +at +by +for +with +about +against +between +into +through +during +before +after +above +below +to +from +up +down +in +out +on +off +over +under +again +further +then +once +here +there +when +where +why +how +all +any +both +each +few +more +most +other +some +such +no +nor +not +only +own +same +so +than +too +very +one +every +least +less +many +now +ever +never +say +says +said +also +get +go +goes +just +made +make +put +see +seen +whether +like +well +back +even +still +way +take +since +another +however +two +three +four +five +first +second +new +old +high +long diff --git a/src/powercontext/builtin/artifacts/lexical_data/zh.txt b/src/powercontext/builtin/artifacts/lexical_data/zh.txt new file mode 100644 index 0000000000..15dea1c3fc --- /dev/null +++ b/src/powercontext/builtin/artifacts/lexical_data/zh.txt @@ -0,0 +1,794 @@ +、 +。 +〈 +〉 +《 +》 +一 +一个 +一些 +一何 +一切 +一则 +一方面 +一旦 +一来 +一样 +一种 +一般 +一转眼 +七 +万一 +三 +上 +上下 +下 +不 +不仅 +不但 +不光 +不单 +不只 +不外乎 +不如 +不妨 +不尽 +不尽然 +不得 +不怕 +不惟 +不成 +不拘 +不料 +不是 +不比 +不然 +不特 +不独 +不管 +不至于 +不若 +不论 +不过 +不问 +与 +与其 +与其说 +与否 +与此同时 +且 +且不说 +且说 +两者 +个 +个别 +中 +临 +为 +为了 +为什么 +为何 +为止 +为此 +为着 +乃 +乃至 +乃至于 +么 +之 +之一 +之所以 +之类 +乌乎 +乎 +乘 +九 +也 +也好 +也罢 +了 +二 +二来 +于 +于是 +于是乎 +云云 +云尔 +五 +些 +亦 +人 +人们 +人家 +什 +什么 +什么样 +今 +介于 +仍 +仍旧 +从 +从此 +从而 +他 +他人 +他们 +他们们 +以 +以上 +以为 +以便 +以免 +以及 +以故 +以期 +以来 +以至 +以至于 +以致 +们 +任 +任何 +任凭 +会 +似的 +但 +但凡 +但是 +何 +何以 +何况 +何处 +何时 +余外 +作为 +你 +你们 +使 +使得 +例如 +依 +依据 +依照 +便于 +俺 +俺们 +倘 +倘使 +倘或 +倘然 +倘若 +借 +借傥然 +假使 +假如 +假若 +做 +像 +儿 +先不先 +光 +光是 +全体 +全部 +八 +六 +兮 +共 +关于 +关于具体地说 +其 +其一 +其中 +其二 +其他 +其余 +其它 +其次 +具体地说 +具体说来 +兼之 +内 +再 +再其次 +再则 +再有 +再者 +再者说 +再说 +冒 +冲 +况且 +几 +几时 +凡 +凡是 +凭 +凭借 +出于 +出来 +分 +分别 +则 +则甚 +别 +别人 +别处 +别是 +别的 +别管 +别说 +到 +前后 +前此 +前者 +加之 +加以 +区 +即 +即令 +即使 +即便 +即如 +即或 +即若 +却 +去 +又 +又及 +及 +及其 +及至 +反之 +反而 +反过来 +反过来说 +受到 +另 +另一方面 +另外 +另悉 +只 +只当 +只怕 +只是 +只有 +只消 +只要 +只限 +叫 +叮咚 +可 +可以 +可是 +可见 +各 +各个 +各位 +各种 +各自 +同 +同时 +后 +后者 +向 +向使 +向着 +吓 +吗 +否则 +吧 +吧哒 +含 +吱 +呀 +呃 +呕 +呗 +呜 +呜呼 +呢 +呵 +呵呵 +呸 +呼哧 +咋 +和 +咚 +咦 +咧 +咱 +咱们 +咳 +哇 +哈 +哈哈 +哉 +哎 +哎呀 +哎哟 +哗 +哟 +哦 +哩 +哪 +哪个 +哪些 +哪儿 +哪天 +哪年 +哪怕 +哪样 +哪边 +哪里 +哼 +哼唷 +唉 +唯有 +啊 +啐 +啥 +啦 +啪达 +啷当 +喂 +喏 +喔唷 +喽 +嗡 +嗡嗡 +嗬 +嗯 +嗳 +嘎 +嘎登 +嘘 +嘛 +嘻 +嘿 +嘿嘿 +四 +因 +因为 +因了 +因此 +因着 +因而 +固然 +在 +在下 +在于 +地 +基于 +处在 +多 +多么 +多少 +大 +大家 +她 +她们 +好 +如 +如上 +如上所述 +如下 +如何 +如其 +如同 +如是 +如果 +如此 +如若 +始而 +孰料 +孰知 +宁 +宁可 +宁愿 +宁肯 +它 +它们 +对 +对于 +对待 +对方 +对比 +将 +小 +尔 +尔后 +尔尔 +尚且 +就 +就是 +就是了 +就是说 +就算 +就要 +尽 +尽管 +尽管如此 +岂但 +己 +已 +已矣 +巴 +巴巴 +年 +并 +并且 +庶乎 +庶几 +开外 +开始 +归 +归齐 +当 +当地 +当然 +当着 +彼 +彼时 +彼此 +往 +待 +很 +得 +得了 +怎 +怎么 +怎么办 +怎么样 +怎奈 +怎样 +总之 +总的来看 +总的来说 +总的说来 +总而言之 +恰恰相反 +您 +惟其 +慢说 +我 +我们 +或 +或则 +或是 +或曰 +或者 +截至 +所 +所以 +所在 +所幸 +所有 +才 +才能 +打 +打从 +把 +抑或 +拿 +按 +按照 +换句话说 +换言之 +据 +据此 +接着 +故 +故此 +故而 +旁人 +无 +无宁 +无论 +既 +既往 +既是 +既然 +日 +时 +时候 +是 +是以 +是的 +更 +曾 +替 +替代 +最 +月 +有 +有些 +有关 +有及 +有时 +有的 +望 +朝 +朝着 +本 +本人 +本地 +本着 +本身 +来 +来着 +来自 +来说 +极了 +果然 +果真 +某 +某个 +某些 +某某 +根据 +欤 +正值 +正如 +正巧 +正是 +此 +此地 +此处 +此外 +此时 +此次 +此间 +毋宁 +每 +每当 +比 +比及 +比如 +比方 +没奈何 +沿 +沿着 +漫说 +点 +焉 +然则 +然后 +然而 +照 +照着 +犹且 +犹自 +甚且 +甚么 +甚或 +甚而 +甚至 +甚至于 +用 +用来 +由 +由于 +由是 +由此 +由此可见 +的 +的确 +的话 +直到 +相对而言 +省得 +看 +眨眼 +着 +着呢 +矣 +矣乎 +矣哉 +离 +秒 +称 +竟而 +第 +等 +等到 +等等 +简言之 +管 +类如 +紧接着 +纵 +纵令 +纵使 +纵然 +经 +经过 +结果 +给 +继之 +继后 +继而 +综上所述 +罢了 +者 +而 +而且 +而况 +而后 +而外 +而已 +而是 +而言 +能 +能否 +腾 +自 +自个儿 +自从 +自各儿 +自后 +自家 +自己 +自打 +自身 +至 +至于 +至今 +至若 +致 +般的 +若 +若夫 +若是 +若果 +若非 +莫不然 +莫如 +莫若 +虽 +虽则 +虽然 +虽说 +被 +要 +要不 +要不是 +要不然 +要么 +要是 +譬喻 +譬如 +让 +许多 +论 +设使 +设或 +设若 +诚如 +诚然 +该 +说 +说来 +请 +诸 +诸位 +诸如 +谁 +谁人 +谁料 +谁知 +贼死 +赖以 +赶 +起 +起见 +趁 +趁着 +越是 +距 +跟 +较 +较之 +边 +过 +还 +还是 +还有 +还要 +这 +这一来 +这个 +这么 +这么些 +这么样 +这么点儿 +这些 +这会儿 +这儿 +这就是说 +这时 +这样 +这次 +这般 +这边 +这里 +进而 +连 +连同 +逐步 +通过 +遵循 +遵照 +那 +那个 +那么 +那么些 +那么样 +那些 +那会儿 +那儿 +那时 +那样 +那般 +那边 +那里 +都 +鄙人 +鉴于 +针对 +阿 +除 +除了 +除外 +除开 +除此之外 +除非 +随 +随后 +随时 +随着 +难道说 +零 +非 +非但 +非徒 +非特 +非独 +靠 +顺 +顺着 +首先 +︿ +！ +＃ +＄ +％ +＆ +（ +） +＊ +＋ +， +０ +１ +２ +３ +４ +５ +６ +７ +８ +９ +： +； +＜ +＞ +？ +＠ +［ +］ +｛ +｜ +｝ +～ +￥ diff --git a/src/powercontext/builtin/artifacts/search.py b/src/powercontext/builtin/artifacts/search.py index 2676a83251..e726791c3e 100644 --- a/src/powercontext/builtin/artifacts/search.py +++ b/src/powercontext/builtin/artifacts/search.py @@ -16,13 +16,17 @@ from __future__ import annotations +import logging import math import re import sys import unicodedata from dataclasses import dataclass -from itertools import pairwise +from functools import lru_cache +from hashlib import sha256 +from pathlib import Path +import jieba from pydantic import BaseModel, ConfigDict, Field, StrictInt, field_validator from powercontext.artifacts.search import ChannelScore @@ -35,58 +39,71 @@ # headroom for RRF division and prevents more extreme ratios from underflowing. _MIN_NORMALIZED_RECALL_CHANNEL_WEIGHT = sys.float_info.min -# Query-only normalization: persisted Analyzer v1 projections remain unchanged. Negations -# are deliberately absent; domain constraints such as "without a backup" remain evidence. -_QUERY_FUNCTION_WORDS = frozenset([ - "a", - "an", - "the", - "and", - "or", - "for", - "of", - "to", - "in", - "on", - "at", - "by", - "from", - "with", - "as", - "is", - "are", - "was", - "were", - "be", - "been", - "being", - "what", - "which", - "who", - "how", - "why", - "when", - "where", - "do", - "does", - "did", - "can", - "could", - "should", - "would", - "i", - "we", - "you", - "it", - "its", - "our", - "your", - "this", - "that", - "these", - "those", - "please", +_NEGATIONS = frozenset([ + "no", + "not", + "never", + "neither", + "nor", + "without", + "cannot", + "can't", + "don't", + "doesn't", + "didn't", + "won't", + "wouldn't", + "shouldn't", + "couldn't", + "mustn't", + "isn't", + "aren't", + "wasn't", + "weren't", + "hasn't", + "haven't", + "hadn't", + "不", + "没", + "没有", + "未", + "无", + "非", + "别", + "莫", + "勿", + "不要", + "不能", + "不得", + "无需", + "不必", + "不是", + "并非", ]) +_HAN_RANGES = r"\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff\U00020000-\U0002fa1f" +_LEXICAL_RUN = re.compile(rf"([{_HAN_RANGES}]+)|[^{_HAN_RANGES}]+") +_WORD = re.compile(r"\w+(?:['’]\w+)*") # noqa: RUF001 - normalize typographic apostrophes within words. + + +@lru_cache(maxsize=1) +def _query_stopwords() -> frozenset[str]: + data = Path(__file__).with_name("lexical_data") + return ( + frozenset( + unicodedata.normalize("NFC", word).casefold() + for name in ("en.txt", "zh.txt") + for word in (data / name).read_text(encoding="utf-8").split() + ) + - _NEGATIONS + ) + + +@lru_cache(maxsize=1) +def _tokenizer() -> jieba.Tokenizer: + jieba.default_logger.setLevel(logging.WARNING) + return jieba.Tokenizer() + + _QUOTED_QUERY_TEXT = re.compile(r"""(`[^`]*`|"[^"]*"|“[^”]*”|‘[^’]*’|(? str: - """Apply Analyzer v1 and return space-delimited backend-safe terms.""" + """Analyze document words without filtering stopwords or losing word frequency.""" return " ".join(term for term, _start, _end in analyze_text_with_spans(value)) -def analyze_fts_query(value: str) -> str: - """Extract lexical evidence terms without changing stored text or semantic queries. - - Recognize only standalone, generic English execution instructions accompanying other - query content. Preserve quoted text, domain-specific directives and unknown phrasing. - Unquoted function words cannot supply the sole overlap during a relaxed recall-gate round. - """ - +def lexical_query_words(value: str) -> tuple[str, ...]: + """Filter unquoted stopwords while retaining quoted identifiers and negation.""" content: list[str] = [] - quoted_terms: set[str] = set() for index, part in enumerate(_QUOTED_QUERY_TEXT.split(unicodedata.normalize("NFC", value).casefold())): if index % 2: - content.append(part) - quoted_terms.update(analyze_text(part).split()) + content.extend(word for word, _start, _end in lexical_terms_with_spans(part[1:-1])) else: sentences = re.split(r"(?<=[.!?])\s+|\n+", part) - content.extend(sentence for sentence in sentences if not _is_execution_sentence(sentence)) - # A search consisting solely of a directive may be looking up that very policy. - terms = analyze_text(" ".join(content) if any(sentence.strip() for sentence in content) else value).split() - if len(set(terms)) > _FTS_SHORT_QUERY_MAX_TERMS: - meaningful = [term for term in terms if term in quoted_terms or term not in _QUERY_FUNCTION_WORDS] - if meaningful: - terms = meaningful - return " ".join(terms) + content.extend( + word + for sentence in sentences + if not _is_execution_sentence(sentence) + for word, _start, _end in lexical_terms_with_spans(sentence) + if word not in _query_stopwords() + ) + if not content and _is_execution_sentence(" ".join(value.casefold().split())): + content = [word for word, _start, _end in lexical_terms_with_spans(value) if word not in _query_stopwords()] + return tuple(content) + + +def analyze_fts_query(value: str) -> str: + """Analyze query evidence with the same word encoding as stored projections.""" + return " ".join(_encode_term(word) for word in lexical_query_words(value)) def _is_execution_sentence(value: str) -> bool: @@ -304,50 +320,33 @@ def _is_execution_sentence(value: str) -> bool: return False -def analyze_text_with_spans(value: str) -> tuple[tuple[str, int, int], ...]: - """Return Analyzer v1 terms with offsets in the NFC+casefold text.""" - +def lexical_terms_with_spans(value: str) -> tuple[tuple[str, int, int], ...]: + """Segment Chinese with Jieba, retaining Latin identifiers and normalized offsets.""" normalized = unicodedata.normalize("NFC", value).casefold() terms: list[tuple[str, int, int]] = [] - word_start: int | None = None - cjk: list[tuple[str, int]] = [] - - def flush_word(end: int) -> None: - nonlocal word_start - if word_start is not None: - terms.append((normalized[word_start:end], word_start, end)) - word_start = None - - def flush_cjk() -> None: - if not cjk: - return - terms.extend((f"u_{ord(character):x}", position, position + 1) for character, position in cjk) - terms.extend( - ( - f"b_{ord(left[0]):x}_{ord(right[0]):x}", - left[1], - right[1] + 1, - ) - for left, right in pairwise(cjk) - ) - cjk.clear() - - for position, character in enumerate(normalized): - if _is_cjk(character): - flush_word(position) - cjk.append((character, position)) - elif character.isalnum() or character == "_": - flush_cjk() - if word_start is None: - word_start = position + for run in _LEXICAL_RUN.finditer(normalized): + if run[1] is not None: + words = _tokenizer().tokenize(run[0], HMM=False) else: - flush_word(position) - flush_cjk() - flush_word(len(normalized)) - flush_cjk() + words = ((word[0].replace("’", "'"), word.start(), word.end()) for word in _WORD.finditer(run[0])) # noqa: RUF001 + terms.extend((word, run.start() + start, run.start() + end) for word, start, end in words) return tuple(terms) +def _encode_term(word: str) -> str: + # Keep ordinary Latin words compatible. Encode terms that unicode61/SPACE + # would split or discard, and escape literal identifiers using our prefixes. + if word.isascii() and word.isalnum() and 3 <= len(word) <= 80 and not word.startswith(("pcu", "pch")): + return word + raw = word.encode("utf-8") + return "pcu" + raw.hex() if len(raw) <= 40 else "pch" + sha256(raw).hexdigest() + + +def analyze_text_with_spans(value: str) -> tuple[tuple[str, int, int], ...]: + """Return backend-safe words and offsets in NFC+casefold text.""" + return tuple((_encode_term(word), start, end) for word, start, end in lexical_terms_with_spans(value)) + + def fts_query_requirements(value: str, /, *, floor: AdmissionFloor | None = None) -> tuple[tuple[str, ...], int]: """Return distinct lexical evidence terms and the shared admission threshold. @@ -390,20 +389,13 @@ def admits_fts_text(query: str, text: str, /, *, floor: AdmissionFloor | None = shared admission requirement exactly as ``fts_query_requirements`` documents. """ - query_terms, required_matches = fts_query_requirements(query, floor=floor) - if not query_terms: - return False - return len(set(query_terms).intersection(analyze_text(text).split())) >= required_matches + return admits_fts_projection(query, analyze_text(text), floor=floor) -def _is_cjk(character: str) -> bool: - point = ord(character) - return ( - 0x3400 <= point <= 0x4DBF - or 0x4E00 <= point <= 0x9FFF - or 0xF900 <= point <= 0xFAFF - or 0x20000 <= point <= 0x2FA1F - ) +def admits_fts_projection(query: str, projection: str, /, *, floor: AdmissionFloor | None = None) -> bool: + """Apply admission to already analyzed words without treating them as raw text.""" + query_terms, required_matches = fts_query_requirements(query, floor=floor) + return bool(query_terms) and len(set(query_terms).intersection(projection.split())) >= required_matches __all__ = [ diff --git a/src/powercontext/builtin/artifacts/topic_memory/fusion.py b/src/powercontext/builtin/artifacts/topic_memory/fusion.py index c1507f9958..c29846ec3e 100644 --- a/src/powercontext/builtin/artifacts/topic_memory/fusion.py +++ b/src/powercontext/builtin/artifacts/topic_memory/fusion.py @@ -26,7 +26,7 @@ AdmissionFloor, InvalidSearchScore, admits_fts_text, - analyze_text, + analyze_fts_query, analyze_text_with_spans, lexical_search_score, unit_l2_cosine_similarity, @@ -253,7 +253,7 @@ def _snippet(query: str, value: str, *, lexical: bool) -> str: def _lexical_focus(query: str, value: str) -> int: normalized, spans = _normalized_spans(value) - query_terms = set(analyze_text(query).split()) + query_terms = set(analyze_fts_query(query).split()) occurrences = sorted( (start, end - start, term) for term, start, end in analyze_text_with_spans(normalized) if term in query_terms ) diff --git a/src/powercontext/builtin/artifacts/topic_memory/relatedness.py b/src/powercontext/builtin/artifacts/topic_memory/relatedness.py index 3218acd3ee..c87ef10445 100644 --- a/src/powercontext/builtin/artifacts/topic_memory/relatedness.py +++ b/src/powercontext/builtin/artifacts/topic_memory/relatedness.py @@ -20,7 +20,7 @@ from dataclasses import dataclass from math import sqrt -from powercontext.builtin.artifacts.search import admits_fts_text, analyze_text +from powercontext.builtin.artifacts.search import admits_fts_text, lexical_query_words from powercontext.builtin.artifacts.topic_memory.fusion import admits_topic_memory_vector_distance from powercontext.builtin.artifacts.topic_memory.generation import MAX_TOPIC_MEMORY_STAGE_ITEMS, TopicMemoryProposal from powercontext.builtin.artifacts.topic_memory.models import ( @@ -45,7 +45,7 @@ def topic_memory_lexical_signature(content: TopicMemoryContent, /) -> str: weighted: Counter[str] = Counter() first: dict[str, tuple[int, int]] = {} for field_order, (weight, value) in enumerate(((8, content.title), (4, content.summary), (1, content.detail))): - for position, term in enumerate(analyze_text(value).split()): + for position, term in enumerate(lexical_query_words(value)): weighted[term] += weight first.setdefault(term, (field_order, position)) ordered = sorted(weighted, key=lambda term: (-weighted[term], first[term], term.encode())) diff --git a/src/powercontext/builtin/persistence/experience_index.py b/src/powercontext/builtin/persistence/experience_index.py index 8abf494cff..0a82d794e7 100644 --- a/src/powercontext/builtin/persistence/experience_index.py +++ b/src/powercontext/builtin/persistence/experience_index.py @@ -36,7 +36,9 @@ AdmissionCounts, AdmissionFloor, InvalidSearchScore, + admits_fts_projection, admits_fts_text, + analyze_text, lexical_search_score, ) from powercontext.builtin.artifacts.skill import ( @@ -240,6 +242,7 @@ async def rebuild_experience_projections(connection: AsyncConnection, /) -> None ARTIFACT_HEADS_TABLE.c.artifact_id, ARTIFACT_HEADS_TABLE.c.revision, ARTIFACTS_TABLE.c.content, + ARTIFACT_HEADS_TABLE.c.searchable_text, ) .join( ARTIFACTS_TABLE, @@ -254,12 +257,15 @@ async def rebuild_experience_projections(connection: AsyncConnection, /) -> None ).mappings() ) for row in rows: + searchable = experience_searchable_text(_content(row["content"])) + if row["searchable_text"] == searchable: + continue await _update_searchable_text( connection, scope_id=str(row["scope_id"]), artifact_id=str(row["artifact_id"]), revision=int(row["revision"]), - searchable_text=experience_searchable_text(_content(row["content"])), + searchable_text=searchable, ) @@ -274,6 +280,7 @@ async def rebuild_skill_projections(connection: AsyncConnection, /) -> None: ARTIFACT_HEADS_TABLE.c.artifact_id, ARTIFACT_HEADS_TABLE.c.revision, ARTIFACTS_TABLE.c.content, + ARTIFACT_HEADS_TABLE.c.searchable_text, ) .join( ARTIFACTS_TABLE, @@ -290,13 +297,16 @@ async def rebuild_skill_projections(connection: AsyncConnection, /) -> None: for row in rows: content = _skill_content(row["content"]) package = await _load_skill_package(connection, str(row["scope_id"]), content) + searchable = skill_searchable_text(content, package) + if row["searchable_text"] == searchable: + continue await _update_searchable_text( connection, scope_id=str(row["scope_id"]), family=Skill.family, artifact_id=str(row["artifact_id"]), revision=int(row["revision"]), - searchable_text=skill_searchable_text(content, package), + searchable_text=searchable, ) @@ -424,8 +434,8 @@ def skill_search_hits( hits: list[SkillSearchHit] = [] for row in rows: content = _skill_content(row["content"]) - searchable = row.get("searchable_text") or skill_search_text(content) - if not admits_fts_text(query, str(searchable), floor=admission): + searchable = row.get("searchable_text") or analyze_text(skill_search_text(content)) + if not admits_fts_projection(query, str(searchable), floor=admission): continue retrieval_score, channel_scores = _search_scores(row, required=require_scores or min_score is not None) if min_score is not None and (retrieval_score is None or retrieval_score < min_score): diff --git a/src/powercontext/builtin/persistence/lexical_projection.py b/src/powercontext/builtin/persistence/lexical_projection.py new file mode 100644 index 0000000000..a77c0d49be --- /dev/null +++ b/src/powercontext/builtin/persistence/lexical_projection.py @@ -0,0 +1,59 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Refresh changed derived words using existing index initialization and transactions.""" + +from sqlalchemy import Table, select, tuple_, update +from sqlalchemy.ext.asyncio import AsyncConnection + +from powercontext.builtin.artifacts.search import analyze_text +from powercontext.builtin.persistence.tables import TOPIC_MEMORY_ACTIVE_CHUNKS_TABLE, TOPIC_MEMORY_ACTIVE_TOPICS_TABLE + + +async def refresh_lexical_projections(connection: AsyncConnection, atomic_table: Table | None) -> None: + """Preserve identities, authoritative fields and vectors; write only changed words.""" + projections = [ + (TOPIC_MEMORY_ACTIVE_TOPICS_TABLE, ("title", "summary")), + (TOPIC_MEMORY_ACTIVE_CHUNKS_TABLE, ("chunk_text",)), + ] + if atomic_table is not None: + projections.append((atomic_table, ("kind", "text"))) + for table, fields in projections: + keys = tuple(table.primary_key.columns) + columns = tuple( + dict.fromkeys((*keys, table.c.revision, table.c.searchable_text, *(table.c[name] for name in fields))) + ) + after = None + changed = False + while True: + statement = select(*columns).order_by(*keys).limit(100) + if after is not None: + statement = statement.where(tuple_(*keys) > after) + rows = (await connection.execute(statement)).mappings().all() + if not rows: + break + for row in rows: + searchable = analyze_text("\n".join(row[name] for name in fields)) + if row["searchable_text"] != searchable: + await connection.execute( + update(table) + .where(*(key == row[key.name] for key in keys), table.c.revision == row["revision"]) + .values(searchable_text=searchable) + ) + changed = True + after = tuple(rows[-1][key.name] for key in keys) + if changed and table is not atomic_table and connection.dialect.name == "sqlite": + from powercontext.builtin.persistence.sqlite.topic_memory_index import SQLITE_TOPIC_MEMORY_FTS_MARKER_TABLE + + await connection.execute(SQLITE_TOPIC_MEMORY_FTS_MARKER_TABLE.delete()) diff --git a/src/powercontext/builtin/persistence/sqlite/experience_index.py b/src/powercontext/builtin/persistence/sqlite/experience_index.py index f720774c4a..87c1e4f258 100644 --- a/src/powercontext/builtin/persistence/sqlite/experience_index.py +++ b/src/powercontext/builtin/persistence/sqlite/experience_index.py @@ -19,7 +19,7 @@ from collections.abc import Mapping from typing import Any -from sqlalchemy import select, text +from sqlalchemy import text from sqlalchemy.ext.asyncio import AsyncConnection from powercontext.builtin.artifacts.experience import ( @@ -40,7 +40,6 @@ replace_skill_projection, skill_search_hits, ) -from powercontext.builtin.persistence.tables import ARTIFACT_HEADS_TABLE _CREATE_FTS_SQL = """ CREATE VIRTUAL TABLE IF NOT EXISTS pc_artifact_fts USING fts5( @@ -52,7 +51,32 @@ tokenize='unicode61' ) """ -_DELETE_ALL_FTS_SQL = "DELETE FROM pc_artifact_fts" +_RECONCILE_DELETE_FTS_SQL = """ +DELETE FROM pc_artifact_fts +WHERE NOT EXISTS ( + SELECT 1 FROM pc_artifact_heads AS h + WHERE h.scope_id = pc_artifact_fts.scope_id AND h.family = pc_artifact_fts.family + AND h.artifact_id = pc_artifact_fts.artifact_id AND h.revision = pc_artifact_fts.revision + AND h.family IN ('experience', 'skill') AND h.lifecycle_state = 'active' + AND h.searchable_text = pc_artifact_fts.searchable_text +) +OR rowid NOT IN ( + SELECT min(rowid) FROM pc_artifact_fts GROUP BY scope_id, family, artifact_id +) +""" +_RECONCILE_INSERT_FTS_SQL = """ +INSERT INTO pc_artifact_fts (scope_id, family, artifact_id, revision, searchable_text) +SELECT h.scope_id, h.family, h.artifact_id, h.revision, h.searchable_text +FROM pc_artifact_heads AS h +WHERE h.family IN ('experience', 'skill') AND h.lifecycle_state = 'active' + AND h.searchable_text IS NOT NULL + AND NOT EXISTS ( + SELECT 1 FROM pc_artifact_fts AS f + WHERE f.scope_id = h.scope_id AND f.family = h.family + AND f.artifact_id = h.artifact_id AND f.revision = h.revision + AND f.searchable_text = h.searchable_text + ) +""" _PROBE_FTS_SQL = "SELECT rowid FROM pc_artifact_fts WHERE pc_artifact_fts MATCH 'powercontext'" _DELETE_FTS_SQL = text( "DELETE FROM pc_artifact_fts WHERE scope_id = :scope_id AND family = :family AND artifact_id = :artifact_id" @@ -98,24 +122,8 @@ async def initialize(self, connection: AsyncConnection, /) -> None: await rebuild_experience_projections(connection) await rebuild_skill_projections(connection) await connection.exec_driver_sql(_CREATE_FTS_SQL) - await connection.exec_driver_sql(_DELETE_ALL_FTS_SQL) - rows = ( - await connection.execute( - select( - ARTIFACT_HEADS_TABLE.c.scope_id, - ARTIFACT_HEADS_TABLE.c.family, - ARTIFACT_HEADS_TABLE.c.artifact_id, - ARTIFACT_HEADS_TABLE.c.revision, - ARTIFACT_HEADS_TABLE.c.searchable_text, - ).where( - ARTIFACT_HEADS_TABLE.c.family.in_((Experience.family, Skill.family)), - ARTIFACT_HEADS_TABLE.c.lifecycle_state == "active", - ARTIFACT_HEADS_TABLE.c.searchable_text.is_not(None), - ) - ) - ).mappings() - for row in rows: - await self._insert_row(connection, row) + await connection.exec_driver_sql(_RECONCILE_DELETE_FTS_SQL) + await connection.exec_driver_sql(_RECONCILE_INSERT_FTS_SQL) await connection.exec_driver_sql(_PROBE_FTS_SQL) async def replace( diff --git a/src/powercontext/builtin/runtime/application.py b/src/powercontext/builtin/runtime/application.py index 8dbe0cfa0d..b15ed73837 100644 --- a/src/powercontext/builtin/runtime/application.py +++ b/src/powercontext/builtin/runtime/application.py @@ -80,7 +80,12 @@ PromptError, ) from powercontext.builtin.artifacts.prompt.service import PromptService -from powercontext.builtin.artifacts.search import AdmissionCounts, AdmissionFloor, analyze_text +from powercontext.builtin.artifacts.search import ( + AdmissionCounts, + AdmissionFloor, + analyze_text, + lexical_terms_with_spans, +) from powercontext.builtin.artifacts.skill import ( AgentKind, AgentSkillTarget, @@ -3530,7 +3535,7 @@ def _bounded_topic_memory_recall_query(query: str) -> str: """Focus internal Prepared Context recall without weakening the public search contract.""" trimmed = query.strip() - terms = tuple(dict.fromkeys(analyze_text(trimmed).split())) + terms = tuple(dict.fromkeys(word for word, _start, _end in lexical_terms_with_spans(trimmed))) if len(terms) <= MAX_TOPIC_MEMORY_QUERY_TERMS: return trimmed diff --git a/src/powercontext/builtin/runtime/composition.py b/src/powercontext/builtin/runtime/composition.py index b8015b896c..abc7826e2b 100644 --- a/src/powercontext/builtin/runtime/composition.py +++ b/src/powercontext/builtin/runtime/composition.py @@ -101,6 +101,7 @@ ensure_supported_atomic_memory_schema, ) from powercontext.builtin.persistence.experience_index import ensure_artifact_head_searchable_text +from powercontext.builtin.persistence.lexical_projection import refresh_lexical_projections from powercontext.builtin.persistence.oceanbase.atomic_memory_index import OceanBaseAtomicMemoryIndex from powercontext.builtin.persistence.oceanbase.experience_index import OceanBaseExperienceFTSIndex from powercontext.builtin.persistence.oceanbase.profile import OceanBaseConfig, OceanBaseProfile @@ -1017,6 +1018,9 @@ async def open_builtin_contexts( if not _topic_memory_worker: await _initialize_atomic_memory_authority(connection) await atomic_index.initialize(connection) + await refresh_lexical_projections( + connection, atomic_index.table if atomic_index.table in atomic_index.tables else None + ) await ensure_artifact_head_searchable_text(connection) await assert_atomic_memory_migration_ready(connection) await experience_index.initialize(connection) @@ -1093,6 +1097,9 @@ async def open_builtin_contexts( if not _topic_memory_worker: await _initialize_atomic_memory_authority(connection) await atomic_index.initialize(connection) + await refresh_lexical_projections( + connection, atomic_index.table if atomic_index.table in atomic_index.tables else None + ) await ensure_artifact_head_searchable_text(connection) await assert_atomic_memory_migration_ready(connection) await experience_index.initialize(connection) diff --git a/src/powercontext/builtin/runtime/topic_memory_processing.py b/src/powercontext/builtin/runtime/topic_memory_processing.py index 9d50c7d14b..9584b47135 100644 --- a/src/powercontext/builtin/runtime/topic_memory_processing.py +++ b/src/powercontext/builtin/runtime/topic_memory_processing.py @@ -35,7 +35,7 @@ from powercontext._logging import log_safely from powercontext.artifacts import ArtifactRef from powercontext.builtin.artifacts.prompt.service import current_prompt -from powercontext.builtin.artifacts.search import analyze_text +from powercontext.builtin.artifacts.search import lexical_terms_with_spans from powercontext.builtin.artifacts.topic_memory import ( MAX_TOPIC_MEMORY_QUERY_LENGTH, MAX_TOPIC_MEMORY_QUERY_TERMS, @@ -1444,7 +1444,7 @@ def _bounded_probe_query(probe: TopicMemoryProbe) -> str: """Close model-authored probes over the R4 query contract before I/O.""" raw = " ".join(value.strip() for value in (probe.query, *probe.keywords) if value.strip()) - terms = tuple(dict.fromkeys(analyze_text(raw).split())) + terms = tuple(dict.fromkeys(word for word, _start, _end in lexical_terms_with_spans(raw))) selected: list[str] = [] characters = 0 for term in terms[:MAX_TOPIC_MEMORY_QUERY_TERMS]: diff --git a/tests/builtin/artifacts/test_lexical_analysis.py b/tests/builtin/artifacts/test_lexical_analysis.py new file mode 100644 index 0000000000..a4758e60c1 --- /dev/null +++ b/tests/builtin/artifacts/test_lexical_analysis.py @@ -0,0 +1,93 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ruff: noqa: RUF001 - intentional multilingual and fullwidth input. + +"""Shared word retrieval, exact identifiers, quoting and content-safe normalization.""" + +import pytest + +from powercontext.builtin.artifacts.search import ( + AdmissionFloor, + admits_fts_text, + analyze_fts_query, + analyze_text, + lexical_terms_with_spans, +) + +FACT = "本项目的 PowerContext Server 固定在 127.0.0.1:8000；测试写入。" + + +@pytest.mark.parametrize( + "query", + [ + "PowerContext Server 地址", + "PowerContext Server 地址是什么？", + "的 PowerContext Server 地址是什么？", + "请告诉我 PowerContext Server 地址是什么？", + "我之前让你记住的 PowerContext Server 地址是什么？", + ], +) +def test_natural_question_forms_recall_the_same_lexical_fact(query): + assert admits_fts_text(query, FACT) + assert not admits_fts_text(query, "Dinner recipe baking instructions.") + + +def test_chinese_words_and_latin_identifiers_remain_distinct(): + assert admits_fts_text("alpha beta gamma u_7684", "alpha u_7684") + assert admits_fts_text("u_7684", "u_7684") + assert not admits_fts_text("u_7684", "的") + assert not admits_fts_text("的", "u_7684") + assert not admits_fts_text('"的"', "u_7684") + + +def test_stopwords_are_query_only_and_quoting_can_search_them(): + assert not analyze_fts_query("的 是 什么") + assert analyze_text("的") + assert admits_fts_text('"的"', "的") + assert admits_fts_text('"AND" "OR"', "AND and OR are operators.") + + +@pytest.mark.parametrize("word", ["不", "没有", "不要", "无", "not", "without", "can't"]) +def test_negation_is_searchable_evidence(word): + assert analyze_text(word) in analyze_fts_query(word) + + +def test_explicit_admission_still_applies_to_words(): + strict = AdmissionFloor(lexical_coverage=0.5, lexical_min_matched_terms=2) + assert not admits_fts_text("alpha beta gamma delta", "alpha") + assert not admits_fts_text("alpha beta gamma delta", "alpha", floor=strict) + assert admits_fts_text("alpha beta gamma delta", "alpha beta", floor=strict) + + +def test_word_spans_and_frequencies_preserve_the_normalized_original(): + text = "Ｃafé 地址 地址 u_7684" + # NFC preserves fullwidth letters; casefold only changes case. + normalized = "ｃafé 地址 地址 u_7684" + words = lexical_terms_with_spans(text) + assert [word for word, _start, _end in words].count("地址") == 2 + assert all(normalized[start:end] == word for word, start, end in words) + assert analyze_text(text).split().count(analyze_text("地址")) == 2 + + +@pytest.mark.parametrize("identifier", ["x9", "u_7684", "pcue79a84", "pch1234", "A" * 200]) +def test_encoded_projection_has_no_literal_identifier_alias(identifier): + assert admits_fts_text(identifier, identifier) + assert not admits_fts_text(identifier, analyze_text(identifier)) + + +def test_apostrophe_spelling_preserves_contractions_and_quoted_stopwords(): + assert analyze_fts_query("it’s") == analyze_fts_query("it's") == "" + assert admits_fts_text("don’t", "don't disclose passwords") + assert admits_fts_text('"it’s"', "it's documented") diff --git a/tests/e2e/test_lexical_projection_upgrade.py b/tests/e2e/test_lexical_projection_upgrade.py new file mode 100644 index 0000000000..5c47b968ed --- /dev/null +++ b/tests/e2e/test_lexical_projection_upgrade.py @@ -0,0 +1,187 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Offline refresh preserves authority and vectors, and skips compatible projections.""" + +import asyncio + +from pydantic import JsonValue +from sqlalchemy import select, text, update + +from powercontext.builtin.artifacts.memory import EmbeddingProfile +from powercontext.builtin.inference import EmbeddingResult +from powercontext.builtin.persistence.atomic_memory_index_schema import atomic_memory_current_table +from powercontext.builtin.persistence.tables import ( + ARTIFACT_HEADS_TABLE, + ARTIFACTS_TABLE, + SKILL_PACKAGES_TABLE, + SOURCES_TABLE, + TOPIC_MEMORY_ACTIVE_CHUNKS_TABLE, + TOPIC_MEMORY_ACTIVE_TOPICS_TABLE, +) +from powercontext.builtin.records import ArtifactWrite +from powercontext.builtin.runtime import BuiltinConfig, RuntimeConfig, open_builtin_runtime +from powercontext.builtin.scope import ScopeDraft +from tests.e2e.atomic_memory_migration_backend import migration_profile + + +class _Embedding: + profile = EmbeddingProfile(profile_id="lexical-upgrade", model="fixture", dimension=3) + + async def embed(self, texts, /): + return EmbeddingResult(vectors=tuple((1.0, 0.0, 0.0) for _ in texts)) + + +async def _snapshot(connection): + tables = ( + ARTIFACTS_TABLE, + SOURCES_TABLE, + SKILL_PACKAGES_TABLE, + ARTIFACT_HEADS_TABLE, + atomic_memory_current_table(), + TOPIC_MEMORY_ACTIVE_TOPICS_TABLE, + TOPIC_MEMORY_ACTIVE_CHUNKS_TABLE, + ) + return { + table.name: (await connection.execute(select(*(c for c in table.c if c.name != "searchable_text")))).all() + for table in tables + } + + +def test_startup_refreshes_changed_words_preserving_exact_records_and_vectors(migration_backend): + config = BuiltinConfig(database=migration_backend.config, runtime=RuntimeConfig(artifact_processing_families=())) + embedding = _Embedding() + marker = "部署地址" + contents: dict[str, dict[str, JsonValue]] = { + "atomic-memory": {"kind": "fact", "text": marker + " 127.0.0.1:8000"}, + "experience": {"situation": marker, "action": "配置服务", "outcome": "可访问", "lesson": "核对端口"}, + "skill": {"name": "deployment", "description": marker, "instructions": "配置服务", "validation": ["核对端口"]}, + "topic-memory": {"title": marker, "summary": "服务配置", "detail": "地址固定在 127.0.0.1:8000"}, + } + + async def scenario(): + async with open_builtin_runtime( + config, embedding_model=embedding, scheduler_path=migration_backend.directory / "scheduler" + ) as runtime: + assert runtime.scopes is not None + scope = ( + await runtime.scopes.create( + ScopeDraft(title="Upgrade", summary="Lexical acceptance", idempotency_key="upgrade") + ) + ).scope_id + application = runtime.artifacts.for_scope(scope) + originals = { + family: await runtime.records.for_scope(scope).create_artifact(family, ArtifactWrite(content=content)) + for family, content in contents.items() + } + originals = { + family: await runtime.records.for_scope(scope).get_artifact_revision( + family, value.artifact_id, value.revision + ) + for family, value in originals.items() + } + assert runtime.atomic_memory is not None + before_vectors = await runtime.atomic_memory.for_scope(scope).search(marker, mode="vector") + assert before_vectors.hits + + async with migration_profile(migration_backend.config) as profile, profile.database.transaction() as connection: + authority = await _snapshot(connection) + # Canonical revisions, states and vectors survive an analyzer change. + for table in ( + atomic_memory_current_table(), + TOPIC_MEMORY_ACTIVE_TOPICS_TABLE, + TOPIC_MEMORY_ACTIVE_CHUNKS_TABLE, + ): + await connection.execute(update(table).values(searchable_text="legacy stale terms")) + await connection.execute( + update(ARTIFACT_HEADS_TABLE) + .where(ARTIFACT_HEADS_TABLE.c.family.in_(("experience", "skill"))) + .values(searchable_text="legacy stale terms") + ) + async with open_builtin_runtime( + config, embedding_model=embedding, scheduler_path=migration_backend.directory / "scheduler" + ) as runtime: + application = runtime.artifacts.for_scope(scope) + for family, original in originals.items(): + result = await application.search(family, {"query": "请告诉我部署地址是什么", "mode": "text"}) + assert [match.artifact_ref.artifact_id for match in result.matches] == [original.artifact_id] + assert ( + await runtime.records.for_scope(scope).get_artifact_revision( + family, original.artifact_id, original.revision + ) + == original + ) + assert runtime.atomic_memory is not None + after_vectors = await runtime.atomic_memory.for_scope(scope).search(marker, mode="vector") + assert after_vectors == before_vectors + async with runtime.primary_database.transaction() as connection: + assert await _snapshot(connection) == authority + assert set( + (await connection.execute(select(atomic_memory_current_table().c.projection_format))).scalars() + ) == {"powercontext.atomic-memory.current.v1"} + fresh = await runtime.records.for_scope(scope).create_artifact( + "atomic-memory", ArtifactWrite(content={"kind": "fact", "text": "freshmarker"}) + ) + assert (await application.search("atomic-memory", {"query": "freshmarker"})).matches[ + 0 + ].artifact_ref.artifact_id == fresh.artifact_id + + asyncio.run(scenario()) + + +def test_startup_keeps_compatible_projection_and_fts_rows_unchanged(tmp_path): + from powercontext.builtin.persistence.sqlite import SQLiteConfig + + config = BuiltinConfig( + database=SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'compatible.db'}"), + runtime=RuntimeConfig(artifact_processing_families=()), + ) + + async def scenario(): + async with open_builtin_runtime(config, scheduler_path=tmp_path / "scheduler") as runtime: + assert runtime.scopes is not None + scope = ( + await runtime.scopes.create( + ScopeDraft(title="Compatible", summary="Existing index", idempotency_key="compatible") + ) + ).scope_id + original = await runtime.records.for_scope(scope).create_artifact( + "atomic-memory", ArtifactWrite(content={"kind": "fact", "text": "compatiblemarker"}) + ) + async with migration_profile(config.database) as profile, profile.database.transaction() as connection: + before = await _snapshot(connection) + schema = ( + await connection.execute(text("SELECT name, sql FROM sqlite_master WHERE type = 'table' ORDER BY name")) + ).all() + fts = (await connection.execute(text("SELECT rowid, * FROM pc_atomic_memory_current_fts"))).all() + await connection.execute( + text( + "CREATE TRIGGER refuse_projection_update BEFORE UPDATE ON pc_atomic_memory_current BEGIN SELECT RAISE(ABORT, 'compatible projection must not be rewritten'); END" + ) + ) + async with open_builtin_runtime(config, scheduler_path=tmp_path / "scheduler") as runtime: + result = await runtime.artifacts.for_scope(scope).search("atomic-memory", {"query": "compatiblemarker"}) + assert result.matches[0].artifact_ref.artifact_id == original.artifact_id + async with runtime.primary_database.transaction() as connection: + assert await _snapshot(connection) == before + assert ( + await connection.execute( + text("SELECT name, sql FROM sqlite_master WHERE type = 'table' ORDER BY name") + ) + ).all() == schema + assert ( + await connection.execute(text("SELECT rowid, * FROM pc_atomic_memory_current_fts")) + ).all() == fts + + asyncio.run(scenario()) diff --git a/tests/e2e/test_unified_lexical_search.py b/tests/e2e/test_unified_lexical_search.py new file mode 100644 index 0000000000..11edd0e1d0 --- /dev/null +++ b/tests/e2e/test_unified_lexical_search.py @@ -0,0 +1,134 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ruff: noqa: RUF001 - actual multilingual question forms. +"""One public recall contract on real SQLite, seekdb and OceanBase databases.""" + +import asyncio + +import httpx + +from powercontext.builtin.runtime import RuntimeConfig +from powercontext.server.factory import create_server_app +from powercontext.server.settings import McpConfig, ServerSettings + +FACT = "本项目的 PowerContext Server 固定在 127.0.0.1:8000；测试写入。" +QUESTIONS = ( + "PowerContext Server 地址", + "PowerContext Server 地址是什么？", + "的 PowerContext Server 地址是什么？", + "请告诉我 PowerContext Server 地址是什么？", + "我之前让你记住的 PowerContext Server 地址是什么？", +) + + +def test_public_word_recall_identifiers_scope_isolation_and_prepare(migration_backend): + async def scenario(): + app = create_server_app( + settings=ServerSettings( + database=migration_backend.config, + runtime=RuntimeConfig(artifact_processing_families=()), + mcp=McpConfig(enabled=False), + ), + scheduler_path=migration_backend.directory / "scheduler", + ) + async with ( + app.router.lifespan_context(app), + httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), + base_url="http://test", + ) as client, + ): + + async def scope(label): + response = await client.post( + "/v1/scopes", + json={ + "title": label, + "summary": "Lexical acceptance", + "idempotency_key": label, + }, + ) + response.raise_for_status() + return response.json()["scope_id"] + + async def remember(scope_id, text): + response = await client.post( + "/v1/memory/remember", + json={ + "scope_id": scope_id, + "kind": "fact", + "text": text, + }, + ) + response.raise_for_status() + return response.json()["records"][0]["artifact"] + + primary, other = await scope("primary"), await scope("other") + fact_ref = await remember(primary, FACT) + await remember(other, "PowerContext Server 固定在 10.0.0.9:9000。") + await remember(primary, "Dinner recipe baking instructions.") + identifier_ref = await remember(primary, "alpha u_7684") + long_identifier = "service_" + "a" * 200 + await remember(primary, long_identifier) + prohibition = await remember(primary, "不要保存密码") + await remember(primary, "保存密码") + operators = await remember(primary, "AND and OR operators.") + negation = await remember(primary, "not store passwords") + + async def search(query): + response = await client.post( + "/v1/memory/search", + json={ + "scope_id": primary, + "query": query, + "mode": "fts", + }, + ) + response.raise_for_status() + return response.json()["hits"] + + for question in QUESTIONS: + hits = await search(question) + assert [hit["memory"]["text"] for hit in hits] == [FACT] + prepared = await client.post( + "/v1/context/prepare", + json={ + "scope_id": primary, + "query": question, + "max_bytes": 8192, + }, + ) + prepared.raise_for_status() + assert prepared.json()["status"] == "ready" + assert "127.0.0.1:8000" in prepared.json()["content"] + assert "10.0.0.9" not in prepared.json()["content"] + assert [hit["memory"]["artifact"] for hit in await search("alpha u_7684")] == [identifier_ref] + assert [hit["memory"]["artifact"] for hit in await search("u_7684")] == [identifier_ref] + assert [hit["memory"]["artifact"] for hit in await search('"的"')] == [fact_ref] + assert [hit["memory"]["artifact"] for hit in await search("不要")] == [prohibition] + assert [hit["memory"]["artifact"] for hit in await search('"AND"')] == [operators] + assert [hit["memory"]["artifact"] for hit in await search('"OR"')] == [operators] + assert [hit["memory"]["artifact"] for hit in await search("not")] == [negation] + assert [hit["memory"]["text"] for hit in await search(long_identifier)] == [long_identifier] + assert await search("的 是 什么") == [] + assert await search("astronomy telescope galaxy") == [] + # Exact history remains readable through its returned authoritative identity. + exact = await client.get( + f"/v1/scopes/{primary}/artifacts/{fact_ref['family']}/{fact_ref['artifact_id']}/revisions/{fact_ref['revision']}" + ) + exact.raise_for_status() + assert exact.json()["content"]["text"] == FACT + + asyncio.run(scenario()) diff --git a/uv.lock b/uv.lock index 7dea1d5ceb..5c549c1e42 100644 --- a/uv.lock +++ b/uv.lock @@ -1403,6 +1403,12 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b2/a3/e137168c9c44d18eff0376253da9f1e9234d0239e0ee230d2fee6cea8e55/jeepney-0.9.0-py3-none-any.whl", hash = "sha256:97e5714520c16fc0a45695e5365a2e11b81ea79bba796e26f9f1d178cb182683", size = 49010, upload-time = "2025-02-27T18:51:00.104Z" }, ] +[[package]] +name = "jieba" +version = "0.42.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c6/cb/18eeb235f833b726522d7ebed54f2278ce28ba9438e3135ab0278d9792a2/jieba-0.42.1.tar.gz", hash = "sha256:055ca12f62674fafed09427f176506079bc135638a14e23e25be909131928db2", size = 19214172, upload-time = "2020-01-20T14:27:23.5Z" } + [[package]] name = "jinja2" version = "3.1.6" @@ -2731,6 +2737,7 @@ builtin = [ { name = "alembic" }, { name = "apscheduler" }, { name = "filelock" }, + { name = "jieba" }, { name = "jsonschema" }, { name = "packaging" }, { name = "pydantic-ai-slim", extra = ["anthropic", "openai"] }, @@ -2772,6 +2779,7 @@ seekdb = [ { name = "alembic" }, { name = "apscheduler" }, { name = "filelock" }, + { name = "jieba" }, { name = "jsonschema" }, { name = "packaging" }, { name = "pydantic-ai-slim", extra = ["anthropic", "openai"] }, @@ -2792,6 +2800,7 @@ server = [ { name = "fastmcp" }, { name = "filelock" }, { name = "httpx" }, + { name = "jieba" }, { name = "jinja2" }, { name = "jsonschema" }, { name = "markdown-it-py" }, @@ -2870,6 +2879,9 @@ requires-dist = [ { name = "httpx", extras = ["socks"], marker = "extra == 'cli'", specifier = ">=0.28,<1" }, { name = "httpx", extras = ["socks"], marker = "extra == 'client'", specifier = ">=0.28,<1" }, { name = "inquirerpy", marker = "extra == 'cli'", specifier = ">=0.3,<1" }, + { name = "jieba", marker = "extra == 'builtin'", specifier = "==0.42.1" }, + { name = "jieba", marker = "extra == 'seekdb'", specifier = "==0.42.1" }, + { name = "jieba", marker = "extra == 'server'", specifier = "==0.42.1" }, { name = "jinja2", marker = "extra == 'server'", specifier = ">=3.1.6,<4" }, { name = "jsonschema", marker = "extra == 'builtin'", specifier = ">=4.23,<5" }, { name = "jsonschema", marker = "extra == 'seekdb'", specifier = ">=4.23,<5" }, From 51643c4cfef6274a84f178b054d430774517038d Mon Sep 17 00:00:00 2001 From: Chojan Shang Date: Tue, 13 Oct 2026 20:56:33 +0800 Subject: [PATCH 2/9] build: package stopword license as distribution metadata --- .../LICENSE => LICENSES/stopwords.txt | 15 +++++++++++++++ pyproject.toml | 1 + .../builtin/artifacts/lexical_data/README.md | 11 ----------- 3 files changed, 16 insertions(+), 11 deletions(-) rename src/powercontext/builtin/artifacts/lexical_data/LICENSE => LICENSES/stopwords.txt (62%) delete mode 100644 src/powercontext/builtin/artifacts/lexical_data/README.md diff --git a/src/powercontext/builtin/artifacts/lexical_data/LICENSE b/LICENSES/stopwords.txt similarity index 62% rename from src/powercontext/builtin/artifacts/lexical_data/LICENSE rename to LICENSES/stopwords.txt index 866a3b472e..5e0c7371a2 100644 --- a/src/powercontext/builtin/artifacts/lexical_data/LICENSE +++ b/LICENSES/stopwords.txt @@ -1,3 +1,18 @@ +Bundled stopword lists +====================== + +The following files are redistributed under the MIT license below: + +- src/powercontext/builtin/artifacts/lexical_data/en.txt + Source: stopwords-iso/stopwords-en, raw/snowball-tartarus.txt + Revision: ccc8898188850d8fb019d5f69c14a6635c3bd115 + https://github.com/stopwords-iso/stopwords-en/tree/ccc8898188850d8fb019d5f69c14a6635c3bd115 + +- src/powercontext/builtin/artifacts/lexical_data/zh.txt + Source: stopwords-iso/stopwords-zh, stopwords-zh.txt + Revision: 0f62ec9b9061b9142ed87fa6b39d386945c7cf47 + https://github.com/stopwords-iso/stopwords-zh/tree/0f62ec9b9061b9142ed87fa6b39d386945c7cf47 + The MIT License (MIT) Copyright (c) 2016 Gene Diaz diff --git a/pyproject.toml b/pyproject.toml index ac43c15148..ef7d5681ab 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,7 @@ dynamic = ["version"] description = "PowerContext turns human-agent work into handoff-ready context." authors = [{ name = "PowerContext Team", email = "open_oceanbase@oceanbase.com" }] readme = "README.md" +license-files = ["LICENSE", "LICENSES/*"] keywords = ['python'] requires-python = ">=3.11,<4.0" dependencies = [ diff --git a/src/powercontext/builtin/artifacts/lexical_data/README.md b/src/powercontext/builtin/artifacts/lexical_data/README.md deleted file mode 100644 index 40cc124fce..0000000000 --- a/src/powercontext/builtin/artifacts/lexical_data/README.md +++ /dev/null @@ -1,11 +0,0 @@ -# Bundled query stopwords - -`en.txt` is the Snowball English list from `raw/snowball-tartarus.txt` in -[stopwords-iso/stopwords-en](https://github.com/stopwords-iso/stopwords-en/tree/ccc8898188850d8fb019d5f69c14a6635c3bd115). -`zh.txt` is `stopwords-zh.txt` from -[stopwords-iso/stopwords-zh](https://github.com/stopwords-iso/stopwords-zh/tree/0f62ec9b9061b9142ed87fa6b39d386945c7cf47). -Both are redistributed under the accompanying MIT license. The files retain their source content. - -The analyzer applies these lists only to unquoted query words and explicitly retains negation words. -Document words and their frequencies are retained. Stopword updates do not change indexed tokens and require no index refresh. -Chinese segmentation uses Jieba 0.42.1's packaged dictionary in accurate mode with `HMM=False`. From 93f089b5d513d38b65a14a99d6af9345ca0b9ecb Mon Sep 17 00:00:00 2001 From: Chojan Shang Date: Tue, 13 Oct 2026 21:04:41 +0800 Subject: [PATCH 3/9] fix(search): keep client imports independent of jieba --- .licenserc.yaml | 2 ++ e2e/bub/uv.lock | 3 +++ src/powercontext/builtin/artifacts/search.py | 7 ++++++- 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/.licenserc.yaml b/.licenserc.yaml index 5f70c0dd12..de4604d537 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -63,6 +63,8 @@ header: - '**/py.typed' - 'src/powercontext/http/_generated/**' # Vendored distributions retain their upstream licenses and copyright notices. + - 'LICENSES/**' + - 'src/powercontext/builtin/artifacts/lexical_data/*.txt' - 'src/powercontext/server/dashboard/static/vendor/**' - '.gitattributes' - '**/*.service.in' diff --git a/e2e/bub/uv.lock b/e2e/bub/uv.lock index 0a2d290bea..0a6848bbff 100644 --- a/e2e/bub/uv.lock +++ b/e2e/bub/uv.lock @@ -1635,6 +1635,9 @@ requires-dist = [ { name = "httpx", extras = ["socks"], marker = "extra == 'cli'", specifier = ">=0.28,<1" }, { name = "httpx", extras = ["socks"], marker = "extra == 'client'", specifier = ">=0.28,<1" }, { name = "inquirerpy", marker = "extra == 'cli'", specifier = ">=0.3,<1" }, + { name = "jieba", marker = "extra == 'builtin'", specifier = "==0.42.1" }, + { name = "jieba", marker = "extra == 'seekdb'", specifier = "==0.42.1" }, + { name = "jieba", marker = "extra == 'server'", specifier = "==0.42.1" }, { name = "jinja2", marker = "extra == 'server'", specifier = ">=3.1.6,<4" }, { name = "jsonschema", marker = "extra == 'builtin'", specifier = ">=4.23,<5" }, { name = "jsonschema", marker = "extra == 'seekdb'", specifier = ">=4.23,<5" }, diff --git a/src/powercontext/builtin/artifacts/search.py b/src/powercontext/builtin/artifacts/search.py index e726791c3e..8958f4ca65 100644 --- a/src/powercontext/builtin/artifacts/search.py +++ b/src/powercontext/builtin/artifacts/search.py @@ -25,12 +25,15 @@ from functools import lru_cache from hashlib import sha256 from pathlib import Path +from typing import TYPE_CHECKING -import jieba from pydantic import BaseModel, ConfigDict, Field, StrictInt, field_validator from powercontext.artifacts.search import ChannelScore +if TYPE_CHECKING: + import jieba + _FTS_MIN_QUERY_COVERAGE = 0.25 _FTS_MIN_MATCHED_TERMS = 2 _FTS_SHORT_QUERY_MAX_TERMS = 2 @@ -100,6 +103,8 @@ def _query_stopwords() -> frozenset[str]: @lru_cache(maxsize=1) def _tokenizer() -> jieba.Tokenizer: + import jieba + jieba.default_logger.setLevel(logging.WARNING) return jieba.Tokenizer() From 5aad4c6eb6b54f0d25f2ebf54a7361c72bf7bf5b Mon Sep 17 00:00:00 2001 From: Chojan Shang Date: Tue, 13 Oct 2026 21:14:14 +0800 Subject: [PATCH 4/9] test(search): focus coverage on public retrieval behavior --- .../artifacts/test_lexical_analysis.py | 93 ------------------- tests/e2e/test_lexical_projection_upgrade.py | 61 +++--------- tests/e2e/test_unified_lexical_search.py | 35 ++----- 3 files changed, 21 insertions(+), 168 deletions(-) delete mode 100644 tests/builtin/artifacts/test_lexical_analysis.py diff --git a/tests/builtin/artifacts/test_lexical_analysis.py b/tests/builtin/artifacts/test_lexical_analysis.py deleted file mode 100644 index a4758e60c1..0000000000 --- a/tests/builtin/artifacts/test_lexical_analysis.py +++ /dev/null @@ -1,93 +0,0 @@ -# Copyright (c) 2026 OceanBase. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# ruff: noqa: RUF001 - intentional multilingual and fullwidth input. - -"""Shared word retrieval, exact identifiers, quoting and content-safe normalization.""" - -import pytest - -from powercontext.builtin.artifacts.search import ( - AdmissionFloor, - admits_fts_text, - analyze_fts_query, - analyze_text, - lexical_terms_with_spans, -) - -FACT = "本项目的 PowerContext Server 固定在 127.0.0.1:8000；测试写入。" - - -@pytest.mark.parametrize( - "query", - [ - "PowerContext Server 地址", - "PowerContext Server 地址是什么？", - "的 PowerContext Server 地址是什么？", - "请告诉我 PowerContext Server 地址是什么？", - "我之前让你记住的 PowerContext Server 地址是什么？", - ], -) -def test_natural_question_forms_recall_the_same_lexical_fact(query): - assert admits_fts_text(query, FACT) - assert not admits_fts_text(query, "Dinner recipe baking instructions.") - - -def test_chinese_words_and_latin_identifiers_remain_distinct(): - assert admits_fts_text("alpha beta gamma u_7684", "alpha u_7684") - assert admits_fts_text("u_7684", "u_7684") - assert not admits_fts_text("u_7684", "的") - assert not admits_fts_text("的", "u_7684") - assert not admits_fts_text('"的"', "u_7684") - - -def test_stopwords_are_query_only_and_quoting_can_search_them(): - assert not analyze_fts_query("的 是 什么") - assert analyze_text("的") - assert admits_fts_text('"的"', "的") - assert admits_fts_text('"AND" "OR"', "AND and OR are operators.") - - -@pytest.mark.parametrize("word", ["不", "没有", "不要", "无", "not", "without", "can't"]) -def test_negation_is_searchable_evidence(word): - assert analyze_text(word) in analyze_fts_query(word) - - -def test_explicit_admission_still_applies_to_words(): - strict = AdmissionFloor(lexical_coverage=0.5, lexical_min_matched_terms=2) - assert not admits_fts_text("alpha beta gamma delta", "alpha") - assert not admits_fts_text("alpha beta gamma delta", "alpha", floor=strict) - assert admits_fts_text("alpha beta gamma delta", "alpha beta", floor=strict) - - -def test_word_spans_and_frequencies_preserve_the_normalized_original(): - text = "Ｃafé 地址 地址 u_7684" - # NFC preserves fullwidth letters; casefold only changes case. - normalized = "ｃafé 地址 地址 u_7684" - words = lexical_terms_with_spans(text) - assert [word for word, _start, _end in words].count("地址") == 2 - assert all(normalized[start:end] == word for word, start, end in words) - assert analyze_text(text).split().count(analyze_text("地址")) == 2 - - -@pytest.mark.parametrize("identifier", ["x9", "u_7684", "pcue79a84", "pch1234", "A" * 200]) -def test_encoded_projection_has_no_literal_identifier_alias(identifier): - assert admits_fts_text(identifier, identifier) - assert not admits_fts_text(identifier, analyze_text(identifier)) - - -def test_apostrophe_spelling_preserves_contractions_and_quoted_stopwords(): - assert analyze_fts_query("it’s") == analyze_fts_query("it's") == "" - assert admits_fts_text("don’t", "don't disclose passwords") - assert admits_fts_text('"it’s"', "it's documented") diff --git a/tests/e2e/test_lexical_projection_upgrade.py b/tests/e2e/test_lexical_projection_upgrade.py index 5c47b968ed..55685d31fa 100644 --- a/tests/e2e/test_lexical_projection_upgrade.py +++ b/tests/e2e/test_lexical_projection_upgrade.py @@ -12,21 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Offline refresh preserves authority and vectors, and skips compatible projections.""" +"""Upgrades preserve public search, exact revisions and vector retrieval.""" import asyncio from pydantic import JsonValue -from sqlalchemy import select, text, update +from sqlalchemy import update from powercontext.builtin.artifacts.memory import EmbeddingProfile from powercontext.builtin.inference import EmbeddingResult from powercontext.builtin.persistence.atomic_memory_index_schema import atomic_memory_current_table from powercontext.builtin.persistence.tables import ( ARTIFACT_HEADS_TABLE, - ARTIFACTS_TABLE, - SKILL_PACKAGES_TABLE, - SOURCES_TABLE, TOPIC_MEMORY_ACTIVE_CHUNKS_TABLE, TOPIC_MEMORY_ACTIVE_TOPICS_TABLE, ) @@ -43,23 +40,7 @@ async def embed(self, texts, /): return EmbeddingResult(vectors=tuple((1.0, 0.0, 0.0) for _ in texts)) -async def _snapshot(connection): - tables = ( - ARTIFACTS_TABLE, - SOURCES_TABLE, - SKILL_PACKAGES_TABLE, - ARTIFACT_HEADS_TABLE, - atomic_memory_current_table(), - TOPIC_MEMORY_ACTIVE_TOPICS_TABLE, - TOPIC_MEMORY_ACTIVE_CHUNKS_TABLE, - ) - return { - table.name: (await connection.execute(select(*(c for c in table.c if c.name != "searchable_text")))).all() - for table in tables - } - - -def test_startup_refreshes_changed_words_preserving_exact_records_and_vectors(migration_backend): +def test_upgrade_preserves_search_exact_records_and_vectors(migration_backend): config = BuiltinConfig(database=migration_backend.config, runtime=RuntimeConfig(artifact_processing_families=())) embedding = _Embedding() marker = "部署地址" @@ -96,7 +77,6 @@ async def scenario(): assert before_vectors.hits async with migration_profile(migration_backend.config) as profile, profile.database.transaction() as connection: - authority = await _snapshot(connection) # Canonical revisions, states and vectors survive an analyzer change. for table in ( atomic_memory_current_table(), @@ -125,11 +105,6 @@ async def scenario(): assert runtime.atomic_memory is not None after_vectors = await runtime.atomic_memory.for_scope(scope).search(marker, mode="vector") assert after_vectors == before_vectors - async with runtime.primary_database.transaction() as connection: - assert await _snapshot(connection) == authority - assert set( - (await connection.execute(select(atomic_memory_current_table().c.projection_format))).scalars() - ) == {"powercontext.atomic-memory.current.v1"} fresh = await runtime.records.for_scope(scope).create_artifact( "atomic-memory", ArtifactWrite(content={"kind": "fact", "text": "freshmarker"}) ) @@ -140,7 +115,7 @@ async def scenario(): asyncio.run(scenario()) -def test_startup_keeps_compatible_projection_and_fts_rows_unchanged(tmp_path): +def test_restart_preserves_compatible_search_and_exact_record(tmp_path): from powercontext.builtin.persistence.sqlite import SQLiteConfig config = BuiltinConfig( @@ -159,29 +134,17 @@ async def scenario(): original = await runtime.records.for_scope(scope).create_artifact( "atomic-memory", ArtifactWrite(content={"kind": "fact", "text": "compatiblemarker"}) ) - async with migration_profile(config.database) as profile, profile.database.transaction() as connection: - before = await _snapshot(connection) - schema = ( - await connection.execute(text("SELECT name, sql FROM sqlite_master WHERE type = 'table' ORDER BY name")) - ).all() - fts = (await connection.execute(text("SELECT rowid, * FROM pc_atomic_memory_current_fts"))).all() - await connection.execute( - text( - "CREATE TRIGGER refuse_projection_update BEFORE UPDATE ON pc_atomic_memory_current BEGIN SELECT RAISE(ABORT, 'compatible projection must not be rewritten'); END" - ) + original = await runtime.records.for_scope(scope).get_artifact_revision( + "atomic-memory", original.artifact_id, original.revision ) async with open_builtin_runtime(config, scheduler_path=tmp_path / "scheduler") as runtime: result = await runtime.artifacts.for_scope(scope).search("atomic-memory", {"query": "compatiblemarker"}) assert result.matches[0].artifact_ref.artifact_id == original.artifact_id - async with runtime.primary_database.transaction() as connection: - assert await _snapshot(connection) == before - assert ( - await connection.execute( - text("SELECT name, sql FROM sqlite_master WHERE type = 'table' ORDER BY name") - ) - ).all() == schema - assert ( - await connection.execute(text("SELECT rowid, * FROM pc_atomic_memory_current_fts")) - ).all() == fts + assert ( + await runtime.records.for_scope(scope).get_artifact_revision( + "atomic-memory", original.artifact_id, original.revision + ) + == original + ) asyncio.run(scenario()) diff --git a/tests/e2e/test_unified_lexical_search.py b/tests/e2e/test_unified_lexical_search.py index 11edd0e1d0..b0efa74590 100644 --- a/tests/e2e/test_unified_lexical_search.py +++ b/tests/e2e/test_unified_lexical_search.py @@ -27,13 +27,11 @@ QUESTIONS = ( "PowerContext Server 地址", "PowerContext Server 地址是什么？", - "的 PowerContext Server 地址是什么？", - "请告诉我 PowerContext Server 地址是什么？", "我之前让你记住的 PowerContext Server 地址是什么？", ) -def test_public_word_recall_identifiers_scope_isolation_and_prepare(migration_backend): +def test_public_word_recall_identifiers_and_prepare(migration_backend): async def scenario(): app = create_server_app( settings=ServerSettings( @@ -75,9 +73,8 @@ async def remember(scope_id, text): response.raise_for_status() return response.json()["records"][0]["artifact"] - primary, other = await scope("primary"), await scope("other") + primary = await scope("primary") fact_ref = await remember(primary, FACT) - await remember(other, "PowerContext Server 固定在 10.0.0.9:9000。") await remember(primary, "Dinner recipe baking instructions.") identifier_ref = await remember(primary, "alpha u_7684") long_identifier = "service_" + "a" * 200 @@ -102,33 +99,19 @@ async def search(query): for question in QUESTIONS: hits = await search(question) assert [hit["memory"]["text"] for hit in hits] == [FACT] - prepared = await client.post( - "/v1/context/prepare", - json={ - "scope_id": primary, - "query": question, - "max_bytes": 8192, - }, - ) - prepared.raise_for_status() - assert prepared.json()["status"] == "ready" - assert "127.0.0.1:8000" in prepared.json()["content"] - assert "10.0.0.9" not in prepared.json()["content"] - assert [hit["memory"]["artifact"] for hit in await search("alpha u_7684")] == [identifier_ref] + prepared = await client.post( + "/v1/context/prepare", + json={"scope_id": primary, "query": QUESTIONS[-1], "max_bytes": 8192}, + ) + prepared.raise_for_status() + assert prepared.json()["status"] == "ready" + assert "127.0.0.1:8000" in prepared.json()["content"] assert [hit["memory"]["artifact"] for hit in await search("u_7684")] == [identifier_ref] assert [hit["memory"]["artifact"] for hit in await search('"的"')] == [fact_ref] assert [hit["memory"]["artifact"] for hit in await search("不要")] == [prohibition] assert [hit["memory"]["artifact"] for hit in await search('"AND"')] == [operators] - assert [hit["memory"]["artifact"] for hit in await search('"OR"')] == [operators] assert [hit["memory"]["artifact"] for hit in await search("not")] == [negation] assert [hit["memory"]["text"] for hit in await search(long_identifier)] == [long_identifier] assert await search("的 是 什么") == [] - assert await search("astronomy telescope galaxy") == [] - # Exact history remains readable through its returned authoritative identity. - exact = await client.get( - f"/v1/scopes/{primary}/artifacts/{fact_ref['family']}/{fact_ref['artifact_id']}/revisions/{fact_ref['revision']}" - ) - exact.raise_for_status() - assert exact.json()["content"]["text"] == FACT asyncio.run(scenario()) From 2c77b6b09333b6a9cc0674b33c796932c4677aec Mon Sep 17 00:00:00 2001 From: Chojan Shang Date: Wed, 14 Oct 2026 00:57:00 +0800 Subject: [PATCH 5/9] docs(search): focus guidance on user behavior --- docs/en/docs/workflows/search-artifacts.md | 15 ++++----- docs/zh/docs/workflows/search-artifacts.md | 13 +++----- tests/e2e/test_lexical_projection_upgrade.py | 35 -------------------- 3 files changed, 11 insertions(+), 52 deletions(-) diff --git a/docs/en/docs/workflows/search-artifacts.md b/docs/en/docs/workflows/search-artifacts.md index 1a39e1f58e..86887e654e 100644 --- a/docs/en/docs/workflows/search-artifacts.md +++ b/docs/en/docs/workflows/search-artifacts.md @@ -85,15 +85,12 @@ with `artifact_search_not_supported` and field `include_scores`. Other Family-sp ## Shared word analysis -All built-in searchable Families use Jieba's packaged dictionary for Chinese with `HMM=False`, while Latin words -and identifiers remain whole. SQLite, seekdb and OceanBase rank the same backend-safe terms. Unquoted queries use -bundled Snowball English and stopwords-iso Chinese stopwords; negations remain evidence. Documents retain all words, -and quotations retain stopwords without requesting phrase matching. Admission and ranking controls keep their defaults. - -Record and projection formats remain v1. Stop old Server and Worker processes before upgrading. Normal initialization -compares derived words and updates changed rows, preserving content, history, identities and vectors. Compatible -projections and FTS helpers are reused; no extra database table or maintenance command is needed. Startup comparison -reads current searchable projections, with cost proportional to their text size. +Chinese text is segmented into words; Latin words and identifiers remain whole. Unquoted queries omit common English +and Chinese stopwords while retaining negations. Quote a stopword to search for it; quotations do not request phrase matching. + +Record formats remain v1. Before upgrading, stop existing Server and Worker processes. Initialization refreshes +searchable text for existing records without changing their content, history or vectors. This check runs at each startup +and can take longer as stored data grows. ## Experience and Skill text search diff --git a/docs/zh/docs/workflows/search-artifacts.md b/docs/zh/docs/workflows/search-artifacts.md index 4c3fa50385..f1c74d079f 100644 --- a/docs/zh/docs/workflows/search-artifacts.md +++ b/docs/zh/docs/workflows/search-artifacts.md @@ -82,14 +82,11 @@ Atomic 接受 `query`、`limit`、`mode`、`filters` 和 `include_scores: false` ## 统一分词 -内置可检索 Family 通过 Jieba 随包词典对中文分词，关闭 HMM，保留完整的拉丁词和标识符。 -SQLite、seekdb 和 OceanBase 对相同的后端安全词元排序。未加引号的查询使用随包的 Snowball 英文和 -stopwords-iso 中文停止词表，但保留否定词。文档保留所有词；引号保留停止词，不表示短语匹配。 -准入和排序参数沿用现有默认值。 - -记录和投影格式保持 v1。升级前停止旧 Server 和 Worker 进程。正常初始化比较派生词元，只更新变化的行， -保留正文、历史、身份和向量。已兼容的投影和 FTS 辅助索引直接沿用，无需新增数据库表或维护命令。 -初始化会读取当前可检索投影，成本随正文规模增长。 +中文按词分割，拉丁词和标识符保留为完整词。未加引号的查询会过滤常见中英文停止词，但保留否定词。 +要检索停止词，请给它加引号；引号不表示短语匹配。 + +记录格式保持 v1。升级前停止现有 Server 和 Worker 进程。初始化自动更新已有记录的检索词元， +正文、历史和向量保持不变。每次启动都会执行检查，已有数据越多，启动可能越慢。 ## Experience 和 Skill 文本检索 diff --git a/tests/e2e/test_lexical_projection_upgrade.py b/tests/e2e/test_lexical_projection_upgrade.py index 55685d31fa..0e2239d8d3 100644 --- a/tests/e2e/test_lexical_projection_upgrade.py +++ b/tests/e2e/test_lexical_projection_upgrade.py @@ -113,38 +113,3 @@ async def scenario(): ].artifact_ref.artifact_id == fresh.artifact_id asyncio.run(scenario()) - - -def test_restart_preserves_compatible_search_and_exact_record(tmp_path): - from powercontext.builtin.persistence.sqlite import SQLiteConfig - - config = BuiltinConfig( - database=SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'compatible.db'}"), - runtime=RuntimeConfig(artifact_processing_families=()), - ) - - async def scenario(): - async with open_builtin_runtime(config, scheduler_path=tmp_path / "scheduler") as runtime: - assert runtime.scopes is not None - scope = ( - await runtime.scopes.create( - ScopeDraft(title="Compatible", summary="Existing index", idempotency_key="compatible") - ) - ).scope_id - original = await runtime.records.for_scope(scope).create_artifact( - "atomic-memory", ArtifactWrite(content={"kind": "fact", "text": "compatiblemarker"}) - ) - original = await runtime.records.for_scope(scope).get_artifact_revision( - "atomic-memory", original.artifact_id, original.revision - ) - async with open_builtin_runtime(config, scheduler_path=tmp_path / "scheduler") as runtime: - result = await runtime.artifacts.for_scope(scope).search("atomic-memory", {"query": "compatiblemarker"}) - assert result.matches[0].artifact_ref.artifact_id == original.artifact_id - assert ( - await runtime.records.for_scope(scope).get_artifact_revision( - "atomic-memory", original.artifact_id, original.revision - ) - == original - ) - - asyncio.run(scenario()) From 65d40d6822284f9c7ea7c3beca0b2f67314bf012 Mon Sep 17 00:00:00 2001 From: Chojan Shang Date: Wed, 14 Oct 2026 08:47:36 +0800 Subject: [PATCH 6/9] fix(search): preserve omitted negative contractions --- src/powercontext/builtin/artifacts/search.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/powercontext/builtin/artifacts/search.py b/src/powercontext/builtin/artifacts/search.py index 8958f4ca65..aa393269ef 100644 --- a/src/powercontext/builtin/artifacts/search.py +++ b/src/powercontext/builtin/artifacts/search.py @@ -55,10 +55,15 @@ "doesn't", "didn't", "won't", + "shan't", "wouldn't", "shouldn't", "couldn't", "mustn't", + "daren't", + "needn't", + "oughtn't", + "mightn't", "isn't", "aren't", "wasn't", From ab9aaedae82c211d892530a494120d9611457187 Mon Sep 17 00:00:00 2001 From: Chojan Shang Date: Thu, 15 Oct 2026 11:45:26 +0800 Subject: [PATCH 7/9] fix(search): preserve topic recall evidence and valid content --- .../artifacts/topic_memory/relatedness.py | 15 +- .../builtin/artifacts/topic_memory/search.py | 31 +++ .../builtin/persistence/topic_memory.py | 19 +- .../builtin/runtime/application.py | 24 +-- .../runtime/topic_memory_processing.py | 60 +++--- .../artifacts/topic_memory/test_generation.py | 7 +- .../runtime/test_topic_memory_processing.py | 180 +++++++++++++++++- tests/e2e/test_unified_lexical_search.py | 48 ++++- 8 files changed, 303 insertions(+), 81 deletions(-) diff --git a/src/powercontext/builtin/artifacts/topic_memory/relatedness.py b/src/powercontext/builtin/artifacts/topic_memory/relatedness.py index c87ef10445..1527dea136 100644 --- a/src/powercontext/builtin/artifacts/topic_memory/relatedness.py +++ b/src/powercontext/builtin/artifacts/topic_memory/relatedness.py @@ -51,14 +51,13 @@ def topic_memory_lexical_signature(content: TopicMemoryContent, /) -> str: ordered = sorted(weighted, key=lambda term: (-weighted[term], first[term], term.encode())) selected: list[str] = [] characters = 0 - for term in ordered[:MAX_TOPIC_MEMORY_QUERY_TERMS]: + for word in ordered[:MAX_TOPIC_MEMORY_QUERY_TERMS]: + term = f'"{word}"' additional = len(term) + int(bool(selected)) if characters + additional > MAX_TOPIC_MEMORY_QUERY_LENGTH: continue selected.append(term) characters += additional - if not selected: - raise ValueError("Topic Memory content has no bounded lexical signature") # noqa: TRY003 return " ".join(selected) @@ -93,6 +92,7 @@ def topic_memory_related_components( # noqa: C901 *, secondary_candidates: tuple[frozenset[str], ...], vectors: tuple[EmbeddingVector | None, ...] = (), + signatures: tuple[str, ...] | None = None, ) -> tuple[tuple[int, ...], ...]: """Build bounded connected components from shared targets and bidirectional similarity.""" @@ -100,6 +100,8 @@ def topic_memory_related_components( # noqa: C901 raise ValueError("Topic Memory relatedness input exceeds its bounded contract") # noqa: TRY003 if vectors and len(vectors) != len(proposals): raise ValueError("Topic Memory relatedness vectors must cover every proposal") # noqa: TRY003 + if signatures is not None and len(signatures) != len(proposals): + raise ValueError("Topic Memory relatedness signatures must cover every proposal") # noqa: TRY003 parents = list(range(len(proposals))) def find(index: int) -> int: @@ -114,7 +116,8 @@ def union(left: int, right: int) -> None: if left_root != right_root: parents[max(left_root, right_root)] = min(left_root, right_root) - signatures = tuple(topic_memory_lexical_signature(proposal.content) for proposal in proposals) + if signatures is None: + signatures = tuple(topic_memory_lexical_signature(proposal.content) for proposal in proposals) for left in range(len(proposals)): for right in range(left + 1, len(proposals)): if ( @@ -128,7 +131,9 @@ def union(left: int, right: int) -> None: continue left_text = _content_text(proposals[left].content) right_text = _content_text(proposals[right].content) - lexical = admits_fts_text(signatures[left], right_text) and admits_fts_text(signatures[right], left_text) + lexical = bool(signatures[left] and signatures[right]) and ( + admits_fts_text(signatures[left], right_text) and admits_fts_text(signatures[right], left_text) + ) semantic = False left_vector = None if not vectors else vectors[left] right_vector = None if not vectors else vectors[right] diff --git a/src/powercontext/builtin/artifacts/topic_memory/search.py b/src/powercontext/builtin/artifacts/topic_memory/search.py index 31c6efff50..c98bcf69ff 100644 --- a/src/powercontext/builtin/artifacts/topic_memory/search.py +++ b/src/powercontext/builtin/artifacts/topic_memory/search.py @@ -22,7 +22,10 @@ from pydantic_core import InitErrorDetails from powercontext.artifacts.fusion import FusionSelection, RrfParameters, parse_fusion_parameters +from powercontext.builtin.artifacts.search import lexical_query_words, lexical_terms_with_spans from powercontext.builtin.artifacts.topic_memory.models import ( + MAX_TOPIC_MEMORY_QUERY_LENGTH, + MAX_TOPIC_MEMORY_QUERY_TERMS, TopicArtifactSearchRequest, TopicMemoryCapabilities, TopicMemoryMatchedBy, @@ -36,6 +39,34 @@ } +def bounded_topic_memory_query(query: str) -> str: + """Bound Topic recall without discarding quoted evidence.""" + + trimmed = query.strip() + words = tuple(dict.fromkeys(word for word, _start, _end in lexical_terms_with_spans(trimmed))) + if len(trimmed) <= MAX_TOPIC_MEMORY_QUERY_LENGTH and len(words) <= MAX_TOPIC_MEMORY_QUERY_TERMS: + return trimmed + + selected: list[str] = [] + characters = 0 + for word in dict.fromkeys(lexical_query_words(trimmed)): + # Quoting keeps an explicitly requested stopword searchable after reconstruction. + term = f'"{word}"' + additional = len(term) + int(bool(selected)) + if characters + additional > MAX_TOPIC_MEMORY_QUERY_LENGTH: + continue + selected.append(term) + characters += additional + if len(selected) == MAX_TOPIC_MEMORY_QUERY_TERMS: + break + if selected: + return " ".join(selected) + return ( + " ".join(words[:MAX_TOPIC_MEMORY_QUERY_TERMS])[:MAX_TOPIC_MEMORY_QUERY_LENGTH] + or trimmed[:MAX_TOPIC_MEMORY_QUERY_LENGTH] + ) + + def topic_search_error(path: tuple[str | int, ...], message: str) -> ValidationError: return ValidationError.from_exception_data( "TopicArtifactSearchRequest", diff --git a/src/powercontext/builtin/persistence/topic_memory.py b/src/powercontext/builtin/persistence/topic_memory.py index 75a4969808..9554a974e1 100644 --- a/src/powercontext/builtin/persistence/topic_memory.py +++ b/src/powercontext/builtin/persistence/topic_memory.py @@ -480,22 +480,21 @@ async def search( "query", f"must contain at most {MAX_TOPIC_MEMORY_QUERY_LENGTH} characters", ) - analyzed = analyze_text(query) used_mode = self._select_mode(mode, query_vector, embedding_profile) if artifact_request is not None: admission = artifact_request.admission.as_floor() fusion = self._search_fusion_parameters(used_mode, artifact_request) - query_terms = tuple(sorted(set(analyzed.split()))) - if used_mode in {"fts", "hybrid"} and len(query_terms) > MAX_TOPIC_MEMORY_QUERY_TERMS: - raise InvalidRepositoryArgumentError( - "query", - f"must contain at most {MAX_TOPIC_MEMORY_QUERY_TERMS} distinct Analyzer terms", - ) + query_terms: tuple[str, ...] = () + if used_mode in {"fts", "hybrid"}: + if len(set(analyze_text(query).split())) > MAX_TOPIC_MEMORY_QUERY_TERMS: + raise InvalidRepositoryArgumentError( + "query", + f"must contain at most {MAX_TOPIC_MEMORY_QUERY_TERMS} distinct Analyzer terms", + ) + query_terms = tuple(sorted(set(analyze_fts_query(query).split()))) query_vector = self._canonical_query_vector(used_mode, query_vector) await self._check_retrieval_shape(connection) - analyzed = analyze_fts_query(query) - query_terms = tuple(sorted(set(analyzed.split()))) - if not analyzed and used_mode == "fts": + if not query_terms and used_mode == "fts": return TopicMemorySearchResult( mode=used_mode, hits=(), artifacts=() if artifact_request is not None else None ) diff --git a/src/powercontext/builtin/runtime/application.py b/src/powercontext/builtin/runtime/application.py index b15ed73837..f99c3a5f6b 100644 --- a/src/powercontext/builtin/runtime/application.py +++ b/src/powercontext/builtin/runtime/application.py @@ -84,7 +84,6 @@ AdmissionCounts, AdmissionFloor, analyze_text, - lexical_terms_with_spans, ) from powercontext.builtin.artifacts.skill import ( AgentKind, @@ -125,6 +124,7 @@ TopicMemorySearchMode, TopicMemorySearchResult, ) +from powercontext.builtin.artifacts.topic_memory.search import bounded_topic_memory_query from powercontext.builtin.code.application import CodeApplication from powercontext.builtin.code.errors import CodeError from powercontext.builtin.code.models import CodeConfig, CodeQueryRequest, CodeQueryResult @@ -1492,7 +1492,7 @@ async def _topic_memory_hits( allow_embedding: bool, ) -> TopicMemoryRecallOutcome: configured = self._runtime._topic_memory_search is not None - bounded_query = _bounded_topic_memory_recall_query(query) + bounded_query = bounded_topic_memory_query(query) with self._runtime._stage( "topic_memory.search", attributes={ @@ -3529,23 +3529,3 @@ def _unavailable_evidence(resolution: HandoffResolution) -> tuple[HandoffCitatio def _utc_now() -> datetime: return datetime.now(UTC) - - -def _bounded_topic_memory_recall_query(query: str) -> str: - """Focus internal Prepared Context recall without weakening the public search contract.""" - - trimmed = query.strip() - terms = tuple(dict.fromkeys(word for word, _start, _end in lexical_terms_with_spans(trimmed))) - if len(terms) <= MAX_TOPIC_MEMORY_QUERY_TERMS: - return trimmed - - selected: list[str] = [] - characters = 0 - for term in terms[:MAX_TOPIC_MEMORY_QUERY_TERMS]: - separator = int(bool(selected)) - available = MAX_TOPIC_MEMORY_QUERY_LENGTH - characters - separator - if available < 1: - break - selected.append(term[:available]) - characters += separator + min(len(term), available) - return " ".join(selected) diff --git a/src/powercontext/builtin/runtime/topic_memory_processing.py b/src/powercontext/builtin/runtime/topic_memory_processing.py index 9584b47135..e1fd5304cd 100644 --- a/src/powercontext/builtin/runtime/topic_memory_processing.py +++ b/src/powercontext/builtin/runtime/topic_memory_processing.py @@ -35,10 +35,8 @@ from powercontext._logging import log_safely from powercontext.artifacts import ArtifactRef from powercontext.builtin.artifacts.prompt.service import current_prompt -from powercontext.builtin.artifacts.search import lexical_terms_with_spans from powercontext.builtin.artifacts.topic_memory import ( MAX_TOPIC_MEMORY_QUERY_LENGTH, - MAX_TOPIC_MEMORY_QUERY_TERMS, TOPIC_MEMORY_SOURCE_WINDOW_BINDING, PublishedTopicMemory, TopicMemory, @@ -46,6 +44,7 @@ TopicMemoryDraft, TopicMemoryProjection, TopicMemorySearchHit, + TopicMemorySearchMode, chunk_topic_memory_detail, prepare_topic_memory_projection, ) @@ -81,6 +80,7 @@ topic_memory_related_components, topic_memory_vector_centroid, ) +from powercontext.builtin.artifacts.topic_memory.search import bounded_topic_memory_query from powercontext.builtin.inference import ( EmbeddingModel, GenerationResult, @@ -793,10 +793,15 @@ async def _history( ranked: dict[tuple[str, int], tuple[float, int, TopicMemorySearchHit]] = {} hits_by_probe: list[tuple[TopicMemorySearchHit, ...]] = [] for probe_index, probe in enumerate(probes): - result = await self._search( - scope_id, - _bounded_probe_query(probe), - semantic_query=_bounded_semantic_probe_query(probe), + raw = " ".join(value.strip() for value in (probe.query, *probe.keywords) if value.strip()) + result = ( + await self._search( + scope_id, + bounded_topic_memory_query(raw), + semantic_query=raw[:MAX_TOPIC_MEMORY_QUERY_LENGTH].strip(), + ) + if raw + else () ) selected = _distinct_hits(hit for hit in result if hit.score >= self._history_threshold) if len(selected) < self._history_min: @@ -986,21 +991,30 @@ async def _coordinate( ) secondary: list[frozenset[str]] = [] vectors: list[tuple[float, ...] | None] = [] - for proposal in proposals: + signatures = tuple(topic_memory_lexical_signature(proposal.content) for proposal in proposals) + for proposal, signature in zip(proposals, signatures, strict=True): if proposal.candidate_id is not None: secondary.append(frozenset({proposal.candidate_id})) vectors.append(None) continue - signature = topic_memory_lexical_signature(proposal.content) vector = await self._signature_vector(proposal.content) + if not signature and vector is None: + secondary.append(frozenset()) + vectors.append(None) + continue profile = None if self._embedding_model is None else self._embedding_model.profile + query = ( + signature + or f"{proposal.content.title}\n{proposal.content.summary}"[:MAX_TOPIC_MEMORY_QUERY_LENGTH].strip() + ) + mode: TopicMemorySearchMode = "fts" if vector is None else "hybrid" if signature else "vector" async with self._database.transaction() as connection: result = await self._topics.search( connection, scope_id, - signature, + query, limit=self._history_max, - mode="hybrid" if vector is not None else "fts", + mode=mode, query_vector=vector, embedding_profile=profile, ) @@ -1020,6 +1034,7 @@ async def _coordinate( proposals, secondary_candidates=tuple(secondary), vectors=tuple(vectors), + signatures=signatures, ) coordinated: list[TopicMemoryProposal] = [] for component_index, component in enumerate(components, start=1): @@ -1440,31 +1455,6 @@ def _deterministic_plan(probes: tuple[TopicMemoryProbeCandidates, ...]) -> Topic return TopicMemoryPlannerOutput(items=tuple(items)) -def _bounded_probe_query(probe: TopicMemoryProbe) -> str: - """Close model-authored probes over the R4 query contract before I/O.""" - - raw = " ".join(value.strip() for value in (probe.query, *probe.keywords) if value.strip()) - terms = tuple(dict.fromkeys(word for word, _start, _end in lexical_terms_with_spans(raw))) - selected: list[str] = [] - characters = 0 - for term in terms[:MAX_TOPIC_MEMORY_QUERY_TERMS]: - separator = int(bool(selected)) - available = MAX_TOPIC_MEMORY_QUERY_LENGTH - characters - separator - if available < 1: - break - selected.append(term[:available]) - characters += separator + min(len(term), available) - if selected: - return " ".join(selected) - fallback = raw[:MAX_TOPIC_MEMORY_QUERY_LENGTH].strip() - return fallback or "_" - - -def _bounded_semantic_probe_query(probe: TopicMemoryProbe) -> str: - raw = " ".join(value.strip() for value in (probe.query, *probe.keywords) if value.strip()) - return raw[:MAX_TOPIC_MEMORY_QUERY_LENGTH].strip() or "_" - - def _distinct_hits(values: Iterable[TopicMemorySearchHit]) -> tuple[TopicMemorySearchHit, ...]: observed: set[tuple[str, int]] = set() result: list[TopicMemorySearchHit] = [] diff --git a/tests/builtin/artifacts/topic_memory/test_generation.py b/tests/builtin/artifacts/topic_memory/test_generation.py index 3296f8192f..5d563cb781 100644 --- a/tests/builtin/artifacts/topic_memory/test_generation.py +++ b/tests/builtin/artifacts/topic_memory/test_generation.py @@ -19,6 +19,7 @@ import pytest from pydantic import ValidationError +from powercontext.builtin.artifacts.search import lexical_query_words from powercontext.builtin.artifacts.topic_memory import TopicMemoryContent from powercontext.builtin.artifacts.topic_memory.generation import ( BudgetedTopicMemoryGenerator, @@ -162,7 +163,7 @@ def test_full_content_signature_scans_tail_and_remains_bounded() -> None: signature = topic_memory_lexical_signature(content) - assert "tailwinner" in signature.split() + assert "tailwinner" in lexical_query_words(signature) assert len(signature) <= 8_192 assert len(signature.split()) <= 64 @@ -170,9 +171,7 @@ def test_full_content_signature_scans_tail_and_remains_bounded() -> None: "front", " ".join(f"frequent{index} frequent{index}" for index in range(70)) + " unique-low-weight-tail", ) - assert "unique-low-weight-tail" not in topic_memory_lexical_signature(low_weight_tail).split() - with pytest.raises(ValueError, match="no bounded lexical signature"): - topic_memory_lexical_signature(TopicMemoryContent(title="---", summary="***", detail="###")) + assert "unique-low-weight-tail" not in lexical_query_words(topic_memory_lexical_signature(low_weight_tail)) def test_related_components_share_lexical_and_vector_admission() -> None: diff --git a/tests/builtin/runtime/test_topic_memory_processing.py b/tests/builtin/runtime/test_topic_memory_processing.py index e30193b152..20948209ec 100644 --- a/tests/builtin/runtime/test_topic_memory_processing.py +++ b/tests/builtin/runtime/test_topic_memory_processing.py @@ -62,6 +62,7 @@ TopicMemoryProbeInput, TopicMemoryProbeOutput, TopicMemoryProposal, + TopicMemoryReconcileOutput, TopicMemoryTemporaryInput, TopicMemoryTemporaryOutput, topic_memory_stage_fixed_prompt, @@ -80,7 +81,10 @@ from powercontext.builtin.persistence.processing_intents import ArtifactProcessingIntentRepository from powercontext.builtin.persistence.sources import SourceRepository from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile -from powercontext.builtin.persistence.sqlite.topic_memory_index import SQLiteTopicMemoryFTSIndex +from powercontext.builtin.persistence.sqlite.topic_memory_index import ( + SQLiteTopicMemoryFTSIndex, + SQLiteTopicMemoryVectorIndex, +) from powercontext.builtin.persistence.supervision import ArtifactProcessingLeaseRepository from powercontext.builtin.persistence.tables import ( BUILTIN_TABLES, @@ -263,9 +267,16 @@ def _stages( ) -async def _repositories(config: SQLiteConfig | None = None): - index = CompositeTopicMemoryIndex(SQLiteTopicMemoryFTSIndex()) - profile = SQLiteProfile.open(SQLiteConfig() if config is None else config, tables=BUILTIN_TABLES + index.tables) +async def _repositories(config: SQLiteConfig | None = None, *, embedding_profile: EmbeddingProfile | None = None): + index = CompositeTopicMemoryIndex( + SQLiteTopicMemoryFTSIndex(), + *((SQLiteTopicMemoryVectorIndex(embedding_profile),) if embedding_profile is not None else ()), + ) + profile = SQLiteProfile.open( + SQLiteConfig() if config is None else config, + tables=BUILTIN_TABLES + index.tables, + load_vector_extension=embedding_profile is not None, + ) opened = await profile.__aenter__() topics = TopicMemoryRepository(index=index) async with opened.database.transaction() as connection: @@ -327,6 +338,167 @@ async def scenario() -> None: asyncio.run(scenario()) +@pytest.mark.parametrize("with_vectors", [False, True]) +@pytest.mark.parametrize("bound_candidate", [False, True]) +def test_processor_publishes_stopword_only_content(with_vectors, bound_candidate) -> None: + class Embedding: + profile = EmbeddingProfile(profile_id="empty-lexical", model="fixture", dimension=2) + + async def embed(self, texts, /): + return EmbeddingResult(vectors=tuple((1.0, 0.0) for _ in texts)) + + async def scenario() -> None: + embedding = Embedding() if with_vectors else None + manager, profile, sources, topics = await _repositories( + embedding_profile=None if embedding is None else embedding.profile + ) + try: + content = TopicMemoryContent(title="New", summary="First", detail="One") + leases = ArtifactProcessingLeaseRepository() + historical = None + async with profile.database.transaction() as connection: + source = await sources.add( + connection, + "scope-a", + NoteSource(name="note", materialization=SourceMaterialization.CAPTURED, body="New First One"), + ) + if bound_candidate or with_vectors: + old_content = TopicMemoryContent( + title="historymarker", summary="Prior evidence", detail="Prior state" + ) + projection = prepare_topic_memory_projection(old_content) + if embedding is not None: + projection = prepare_topic_memory_projection( + old_content, + topic_embedding=(1.0, 0.0), + chunk_embeddings=((1.0, 0.0),) * len(projection.chunks), + embedding_profile=embedding.profile, + ) + historical = await topics.publish_create( + connection, + "scope-a", + "history", + TopicMemoryDraft(content=old_content, sources=(source.ref,)), + projection, + ) + term = await leases.start_single_process_term(connection, "holder") + proposal = TopicMemoryProposal( + content=content, + evidence_ids=("evidence-0001",), + candidate_id="candidate-0001" if bound_candidate else None, + ) + stages = _stages( + probe=TopicMemoryProbeOutput( + probes=( + TopicMemoryProbe( + query="historymarker" if bound_candidate else "unmatched", evidence_ids=("evidence-0001",) + ), + ) + ), + global_output=TopicMemoryGlobalOutput(proposals=(proposal,)), + reconcile=( + TopicMemoryReconcileOutput( + proposals=( + proposal.model_copy( + update={ + "candidate_id": "candidate-0001", + "proposal_id": "proposal-0001", + } + ), + ) + ), + ), + ) + processor = TopicMemoryProcessor( + database=profile.database, + sources=sources, + topics=topics, + stages=stages, + publisher=TopicMemoryAtomicPublisher(profile.database, sources, topics, leases=leases), + embedding_model=embedding, + id_factory=lambda: "created", + ) + + assert (await processor.process(_assignment(term.fence("single-process")))).outcome.value == "succeeded" + async with profile.database.transaction() as connection: + current = await topics.browse_current(connection, "scope-a", limit=10) + ref = current[0].artifact_ref + published = await topics.get_exact(connection, "scope-a", ref) + assert len(current) == 1 + assert published.topic.content == content + assert ref.artifact_id == ("history" if historical is not None else "created") + assert ref.revision == (2 if historical is not None else 1) + finally: + await manager.__aexit__(None, None, None) + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("query", ['"the"', "not"]) +def test_processor_updates_history_from_quoted_stopwords_and_negations(query) -> None: + async def scenario() -> None: + manager, profile, sources, topics = await _repositories() + try: + leases = ArtifactProcessingLeaseRepository() + content = TopicMemoryContent(title="the not", summary="historical evidence", detail="Historical state") + updated = TopicMemoryContent(title="Updated topic", summary="Remembered state", detail="Updated state") + async with profile.database.transaction() as connection: + await sources.add( + connection, + "scope-a", + NoteSource( + name="note", materialization=SourceMaterialization.CAPTURED, body="Update remembered state" + ), + ) + historical = await topics.publish_create( + connection, + "scope-a", + "history", + TopicMemoryDraft(content=content), + prepare_topic_memory_projection(content), + ) + term = await leases.start_single_process_term(connection, "holder") + processor = TopicMemoryProcessor( + database=profile.database, + sources=sources, + topics=topics, + stages=_stages( + probe=TopicMemoryProbeOutput( + probes=(TopicMemoryProbe(query=query, evidence_ids=("evidence-0001",)),) + ), + global_output=TopicMemoryGlobalOutput( + proposals=( + TopicMemoryProposal( + content=updated, candidate_id="candidate-0001", evidence_ids=("evidence-0001",) + ), + ) + ), + planner=TopicMemoryPlannerOutput(items=(TopicMemoryPlanItem(probe_ids=("probe-0001",)),)), + evolve=( + TopicMemoryEvolveOutput( + proposal=TopicMemoryProposal(content=updated, evidence_ids=("evidence-0001",)) + ), + ), + ), + publisher=TopicMemoryAtomicPublisher(profile.database, sources, topics, leases=leases), + ) + + assert (await processor.process(_assignment(term.fence("single-process")))).outcome.value == "succeeded" + async with profile.database.transaction() as connection: + current = await topics.browse_current(connection, "scope-a", limit=10) + assert len(current) == 1 + ref = current[0].artifact_ref + published = await topics.get_exact(connection, "scope-a", ref) + assert ref.artifact_id == historical.topic.artifact_id + assert ref.revision == 2 + assert published.topic.content == updated + assert published.topic.lineage.artifacts == (historical.topic.as_ref(),) + finally: + await manager.__aexit__(None, None, None) + + asyncio.run(scenario()) + + def test_processor_zero_probe_noop_and_exact_history_update() -> None: async def scenario() -> None: manager, profile, sources, topics = await _repositories() diff --git a/tests/e2e/test_unified_lexical_search.py b/tests/e2e/test_unified_lexical_search.py index b0efa74590..0ebc04c1f5 100644 --- a/tests/e2e/test_unified_lexical_search.py +++ b/tests/e2e/test_unified_lexical_search.py @@ -18,8 +18,11 @@ import asyncio import httpx +import pytest -from powercontext.builtin.runtime import RuntimeConfig +from powercontext.builtin.records import ArtifactWrite +from powercontext.builtin.runtime import BuiltinConfig, PrepareContextRequest, RuntimeConfig, open_builtin_runtime +from powercontext.builtin.scope import ScopeDraft from powercontext.server.factory import create_server_app from powercontext.server.settings import McpConfig, ServerSettings @@ -30,6 +33,49 @@ "我之前让你记住的 PowerContext Server 地址是什么？", ) +STOPWORD_PREFIX = ( + "i me my myself we us our ours ourselves you your yours yourself yourselves he him his himself " + "she her hers herself it its itself they them their theirs themselves what which who whom this " + "that these those am is are was were be been being have has had having do does did doing will " + "would shall should can could may might must ought" +) + + +@pytest.mark.parametrize("evidence", ["deploymentmarker", '"the"', "not"]) +def test_prepared_topic_recall_preserves_evidence_after_stopwords(migration_backend, evidence): + async def scenario(): + config = BuiltinConfig( + database=migration_backend.config, runtime=RuntimeConfig(artifact_processing_families=()) + ) + async with open_builtin_runtime(config, scheduler_path=migration_backend.directory / "scheduler") as runtime: + assert runtime.scopes is not None + scope = ( + await runtime.scopes.create( + ScopeDraft( + title="Bounded recall", summary="Preserve effective query evidence", idempotency_key="bounded" + ) + ) + ).scope_id + original = await runtime.records.for_scope(scope).create_artifact( + "topic-memory", + ArtifactWrite( + content={"title": "deploymentmarker", "summary": "the not", "detail": "Durable deployment evidence"} + ), + ) + + result = await runtime.context.for_scope(scope).prepare( + PrepareContextRequest.model_validate({ + "query": f"{STOPWORD_PREFIX} {evidence}", + "max_bytes": 8192, + "assembly": {"sections": [{"family": "topic-memory", "limit": 8}]}, + }) + ) + + assert result.status == "ready" + assert result.content is not None and original.artifact_id in result.content + + asyncio.run(scenario()) + def test_public_word_recall_identifiers_and_prepare(migration_backend): async def scenario(): From 65b44047b83d1e5e8569bcbc12686c019766aae6 Mon Sep 17 00:00:00 2001 From: Chojan Shang Date: Thu, 15 Oct 2026 11:45:30 +0800 Subject: [PATCH 8/9] perf(search): avoid redundant projection scans at startup --- docs/en/docs/operate/troubleshoot.md | 2 +- docs/en/docs/workflows/search-artifacts.md | 6 +- docs/zh/docs/operate/troubleshoot.md | 2 +- docs/zh/docs/workflows/search-artifacts.md | 4 +- .../builtin/persistence/lexical_projection.py | 91 +++++++++++++------ .../persistence/sqlite/experience_index.py | 9 +- .../builtin/persistence/tables.py | 9 +- .../persistence/test_experience_index.py | 29 ++++++ tests/e2e/test_lexical_projection_upgrade.py | 12 +++ 9 files changed, 121 insertions(+), 43 deletions(-) diff --git a/docs/en/docs/operate/troubleshoot.md b/docs/en/docs/operate/troubleshoot.md index fb5457c358..9063303f22 100644 --- a/docs/en/docs/operate/troubleshoot.md +++ b/docs/en/docs/operate/troubleshoot.md @@ -267,7 +267,7 @@ so the previous database remains available for recovery: ```bash obloader -D --csv \ - --table 'pc_scopes,pc_source_journal_heads,pc_sources,pc_artifacts,pc_source_cursors,pc_memory_source_windows,pc_artifact_processing_leases,pc_artifact_processing_binding_states,pc_artifact_processing_pending,pc_artifact_processing_auto_wave_targets,pc_artifact_processing_sequences,pc_artifact_processing_intents,pc_topic_memory_processing_targets,pc_artifact_processing_schema,pc_artifact_processing_migration_receipts,pc_topic_memory_work_budgets,pc_topic_memory_retrieval_shape,pc_connector_checkpoints,pc_source_definition_manifests,pc_external_skill_registrations,pc_skill_packages,pc_agent_skill_targets,pc_skill_publications,pc_model_usage_daily,pc_recall_token_daily,pc_recall_effort_daily,pc_receipt_migration_review,pc_decision_observations' \ + --table 'pc_scopes,pc_source_journal_heads,pc_sources,pc_artifacts,pc_source_cursors,pc_memory_source_windows,pc_artifact_processing_leases,pc_artifact_processing_binding_states,pc_artifact_processing_pending,pc_artifact_processing_auto_wave_targets,pc_artifact_processing_sequences,pc_artifact_processing_intents,pc_topic_memory_processing_targets,pc_artifact_processing_schema,pc_artifact_processing_migration_receipts,pc_topic_memory_work_budgets,pc_topic_memory_retrieval_shape,pc_connector_checkpoints,pc_source_definition_manifests,pc_external_skill_registrations,pc_skill_packages,pc_agent_skill_targets,pc_skill_publications,pc_model_usage_daily,pc_recall_token_daily,pc_recall_effort_daily,pc_receipt_migration_review,pc_decision_observations,pc_lexical_projection_versions' \ -f ``` diff --git a/docs/en/docs/workflows/search-artifacts.md b/docs/en/docs/workflows/search-artifacts.md index 86887e654e..d9a253d675 100644 --- a/docs/en/docs/workflows/search-artifacts.md +++ b/docs/en/docs/workflows/search-artifacts.md @@ -88,9 +88,9 @@ with `artifact_search_not_supported` and field `include_scores`. Other Family-sp Chinese text is segmented into words; Latin words and identifiers remain whole. Unquoted queries omit common English and Chinese stopwords while retaining negations. Quote a stopword to search for it; quotations do not request phrase matching. -Record formats remain v1. Before upgrading, stop existing Server and Worker processes. Initialization refreshes -searchable text for existing records without changing their content, history or vectors. This check runs at each startup -and can take longer as stored data grows. +Record formats remain v1. Before upgrading, stop existing Server and Worker processes. The first startup after a +document analyzer upgrade updates searchable text without changing record content, history or vectors. +This startup can take longer as stored data grows. ## Experience and Skill text search diff --git a/docs/zh/docs/operate/troubleshoot.md b/docs/zh/docs/operate/troubleshoot.md index a664e93bc6..19feea4395 100644 --- a/docs/zh/docs/operate/troubleshoot.md +++ b/docs/zh/docs/operate/troubleshoot.md @@ -258,7 +258,7 @@ collation，但不会包含数据库 URL 或凭据。 ```bash obloader -D --csv \ - --table 'pc_scopes,pc_source_journal_heads,pc_sources,pc_artifacts,pc_source_cursors,pc_memory_source_windows,pc_artifact_processing_leases,pc_artifact_processing_binding_states,pc_artifact_processing_pending,pc_artifact_processing_auto_wave_targets,pc_artifact_processing_sequences,pc_artifact_processing_intents,pc_topic_memory_processing_targets,pc_artifact_processing_schema,pc_artifact_processing_migration_receipts,pc_topic_memory_work_budgets,pc_topic_memory_retrieval_shape,pc_connector_checkpoints,pc_source_definition_manifests,pc_external_skill_registrations,pc_skill_packages,pc_agent_skill_targets,pc_skill_publications,pc_model_usage_daily,pc_recall_token_daily,pc_recall_effort_daily,pc_receipt_migration_review,pc_decision_observations' \ + --table 'pc_scopes,pc_source_journal_heads,pc_sources,pc_artifacts,pc_source_cursors,pc_memory_source_windows,pc_artifact_processing_leases,pc_artifact_processing_binding_states,pc_artifact_processing_pending,pc_artifact_processing_auto_wave_targets,pc_artifact_processing_sequences,pc_artifact_processing_intents,pc_topic_memory_processing_targets,pc_artifact_processing_schema,pc_artifact_processing_migration_receipts,pc_topic_memory_work_budgets,pc_topic_memory_retrieval_shape,pc_connector_checkpoints,pc_source_definition_manifests,pc_external_skill_registrations,pc_skill_packages,pc_agent_skill_targets,pc_skill_publications,pc_model_usage_daily,pc_recall_token_daily,pc_recall_effort_daily,pc_receipt_migration_review,pc_decision_observations,pc_lexical_projection_versions' \ -f ``` diff --git a/docs/zh/docs/workflows/search-artifacts.md b/docs/zh/docs/workflows/search-artifacts.md index f1c74d079f..07a49b8a47 100644 --- a/docs/zh/docs/workflows/search-artifacts.md +++ b/docs/zh/docs/workflows/search-artifacts.md @@ -85,8 +85,8 @@ Atomic 接受 `query`、`limit`、`mode`、`filters` 和 `include_scores: false` 中文按词分割，拉丁词和标识符保留为完整词。未加引号的查询会过滤常见中英文停止词，但保留否定词。 要检索停止词，请给它加引号；引号不表示短语匹配。 -记录格式保持 v1。升级前停止现有 Server 和 Worker 进程。初始化自动更新已有记录的检索词元， -正文、历史和向量保持不变。每次启动都会执行检查，已有数据越多，启动可能越慢。 +记录格式保持 v1。升级前停止现有 Server 和 Worker 进程。分词器升级后，首次启动会更新已有记录的检索词元， +正文、历史和向量保持不变。已有数据越多，首次启动可能越慢。 ## Experience 和 Skill 文本检索 diff --git a/src/powercontext/builtin/persistence/lexical_projection.py b/src/powercontext/builtin/persistence/lexical_projection.py index a77c0d49be..4175c25ed3 100644 --- a/src/powercontext/builtin/persistence/lexical_projection.py +++ b/src/powercontext/builtin/persistence/lexical_projection.py @@ -15,45 +15,78 @@ """Refresh changed derived words using existing index initialization and transactions.""" from sqlalchemy import Table, select, tuple_, update +from sqlalchemy.dialects.mysql import insert as mysql_insert +from sqlalchemy.dialects.sqlite import insert as sqlite_insert from sqlalchemy.ext.asyncio import AsyncConnection from powercontext.builtin.artifacts.search import analyze_text -from powercontext.builtin.persistence.tables import TOPIC_MEMORY_ACTIVE_CHUNKS_TABLE, TOPIC_MEMORY_ACTIVE_TOPICS_TABLE +from powercontext.builtin.persistence.tables import ( + LEXICAL_PROJECTION_VERSIONS_TABLE, + TOPIC_MEMORY_ACTIVE_CHUNKS_TABLE, + TOPIC_MEMORY_ACTIVE_TOPICS_TABLE, +) + +# Bump when document segmentation or token encoding changes; query stopwords do not affect projections. +_ANALYZER_VERSION = "jieba-0.42.1-words-v1" async def refresh_lexical_projections(connection: AsyncConnection, atomic_table: Table | None) -> None: - """Preserve identities, authoritative fields and vectors; write only changed words.""" + """Refresh each analyzer version once without changing records, identities or vectors.""" + versions = dict((await connection.execute(select(LEXICAL_PROJECTION_VERSIONS_TABLE))).tuples().all()) projections = [ (TOPIC_MEMORY_ACTIVE_TOPICS_TABLE, ("title", "summary")), (TOPIC_MEMORY_ACTIVE_CHUNKS_TABLE, ("chunk_text",)), ] if atomic_table is not None: projections.append((atomic_table, ("kind", "text"))) + topic_changed = False for table, fields in projections: - keys = tuple(table.primary_key.columns) - columns = tuple( - dict.fromkeys((*keys, table.c.revision, table.c.searchable_text, *(table.c[name] for name in fields))) - ) - after = None - changed = False - while True: - statement = select(*columns).order_by(*keys).limit(100) - if after is not None: - statement = statement.where(tuple_(*keys) > after) - rows = (await connection.execute(statement)).mappings().all() - if not rows: - break - for row in rows: - searchable = analyze_text("\n".join(row[name] for name in fields)) - if row["searchable_text"] != searchable: - await connection.execute( - update(table) - .where(*(key == row[key.name] for key in keys), table.c.revision == row["revision"]) - .values(searchable_text=searchable) - ) - changed = True - after = tuple(rows[-1][key.name] for key in keys) - if changed and table is not atomic_table and connection.dialect.name == "sqlite": - from powercontext.builtin.persistence.sqlite.topic_memory_index import SQLITE_TOPIC_MEMORY_FTS_MARKER_TABLE - - await connection.execute(SQLITE_TOPIC_MEMORY_FTS_MARKER_TABLE.delete()) + if versions.get(table.name) == _ANALYZER_VERSION: + continue + changed = await _refresh_projection(connection, table, fields) + topic_changed |= changed and table is not atomic_table + values = {"projection": table.name, "analyzer_version": _ANALYZER_VERSION} + if connection.dialect.name == "sqlite": + marker = ( + sqlite_insert(LEXICAL_PROJECTION_VERSIONS_TABLE) + .values(values) + .on_conflict_do_update(index_elements=["projection"], set_={"analyzer_version": _ANALYZER_VERSION}) + ) + else: + marker = ( + mysql_insert(LEXICAL_PROJECTION_VERSIONS_TABLE) + .values(values) + .on_duplicate_key_update(analyzer_version=_ANALYZER_VERSION) + ) + await connection.execute(marker) + if topic_changed and connection.dialect.name == "sqlite": + from powercontext.builtin.persistence.sqlite.topic_memory_index import SQLITE_TOPIC_MEMORY_FTS_MARKER_TABLE + + await connection.execute(SQLITE_TOPIC_MEMORY_FTS_MARKER_TABLE.delete()) + + +async def _refresh_projection(connection: AsyncConnection, table: Table, fields: tuple[str, ...]) -> bool: + """Page one projection and write only changed tokens in the caller's transaction.""" + keys = tuple(table.primary_key.columns) + columns = tuple( + dict.fromkeys((*keys, table.c.revision, table.c.searchable_text, *(table.c[name] for name in fields))) + ) + after = None + changed = False + while True: + statement = select(*columns).order_by(*keys).limit(100) + if after is not None: + statement = statement.where(tuple_(*keys) > after) + rows = (await connection.execute(statement)).mappings().all() + if not rows: + return changed + for row in rows: + searchable = analyze_text("\n".join(row[name] for name in fields)) + if row["searchable_text"] != searchable: + await connection.execute( + update(table) + .where(*(key == row[key.name] for key in keys), table.c.revision == row["revision"]) + .values(searchable_text=searchable) + ) + changed = True + after = tuple(rows[-1][key.name] for key in keys) diff --git a/src/powercontext/builtin/persistence/sqlite/experience_index.py b/src/powercontext/builtin/persistence/sqlite/experience_index.py index 87c1e4f258..6fa3766b27 100644 --- a/src/powercontext/builtin/persistence/sqlite/experience_index.py +++ b/src/powercontext/builtin/persistence/sqlite/experience_index.py @@ -70,12 +70,9 @@ FROM pc_artifact_heads AS h WHERE h.family IN ('experience', 'skill') AND h.lifecycle_state = 'active' AND h.searchable_text IS NOT NULL - AND NOT EXISTS ( - SELECT 1 FROM pc_artifact_fts AS f - WHERE f.scope_id = h.scope_id AND f.family = h.family - AND f.artifact_id = h.artifact_id AND f.revision = h.revision - AND f.searchable_text = h.searchable_text - ) +EXCEPT +SELECT scope_id, family, artifact_id, revision, searchable_text +FROM pc_artifact_fts """ _PROBE_FTS_SQL = "SELECT rowid FROM pc_artifact_fts WHERE pc_artifact_fts MATCH 'powercontext'" _DELETE_FTS_SQL = text( diff --git a/src/powercontext/builtin/persistence/tables.py b/src/powercontext/builtin/persistence/tables.py index 69c5191fa4..7b1a306a0f 100644 --- a/src/powercontext/builtin/persistence/tables.py +++ b/src/powercontext/builtin/persistence/tables.py @@ -1256,6 +1256,13 @@ def _entry_text_type(): RECURRENCE_TABLES = (RECURRENCE_MATCH_TABLE, RECURRENCE_OBSERVATION_TABLE) +LEXICAL_PROJECTION_VERSIONS_TABLE = Table( + "pc_lexical_projection_versions", + SHARED_METADATA, + Column("projection", String(128), primary_key=True), + Column("analyzer_version", String(64), nullable=False), +) + BUILTIN_TABLES = ( SCOPE_TABLES + SHARED_TABLES @@ -1264,5 +1271,5 @@ def _entry_text_type(): + STATISTICS_TABLES + DECISION_OBSERVATION_TABLES + RECURRENCE_TABLES - + (ARTIFACT_TAGS_TABLE, DREAM_RUNS_TABLE, RECEIPT_MIGRATION_REVIEW_TABLE) + + (ARTIFACT_TAGS_TABLE, DREAM_RUNS_TABLE, RECEIPT_MIGRATION_REVIEW_TABLE, LEXICAL_PROJECTION_VERSIONS_TABLE) ) diff --git a/tests/builtin/persistence/test_experience_index.py b/tests/builtin/persistence/test_experience_index.py index 7a12817381..2fd8803a68 100644 --- a/tests/builtin/persistence/test_experience_index.py +++ b/tests/builtin/persistence/test_experience_index.py @@ -301,12 +301,41 @@ async def scenario() -> None: .values(searchable_text=None) ) await connection.exec_driver_sql("DELETE FROM pc_artifact_fts") + await connection.exec_driver_sql( + "INSERT INTO pc_artifact_fts (scope_id, family, artifact_id, revision, searchable_text) " + "SELECT scope_id, family, artifact_id, revision, searchable_text FROM pc_artifact_heads " + "WHERE family = 'skill'" + ) + await connection.exec_driver_sql( + "INSERT INTO pc_artifact_fts (scope_id, family, artifact_id, revision, searchable_text) " + "SELECT scope_id, family, artifact_id, revision, searchable_text FROM pc_artifact_heads " + "WHERE family = 'skill'" + ) + await connection.exec_driver_sql( + "INSERT INTO pc_artifact_fts (scope_id, family, artifact_id, revision, searchable_text) " + "SELECT scope_id, family, artifact_id, revision - 1, 'staleprojection' FROM pc_artifact_heads " + "WHERE family = 'experience'" + ) + await connection.exec_driver_sql( + "INSERT INTO pc_artifact_fts VALUES ('project', 'experience', 'orphan', 1, 'orphanprojection')" + ) assert await contexts.search_experience("project", "falconcurrent", 8) == () async with contexts.database.transaction() as connection: await contexts.experience_index.initialize(connection) rebuilt = await contexts.search_experience("project", "falconcurrent", 8) assert tuple(hit.artifact_ref for hit in rebuilt) == (replaced.result_artifact,) + assert await contexts.search_experience("project", "staleprojection", 8) == () + assert await contexts.search_experience("project", "orphanprojection", 8) == () + assert tuple(hit.artifact_ref for hit in await contexts.search_skills("project", "regenerate", 8)) == ( + skill_approval.result_artifact, + ) + + async with contexts.database.transaction() as connection: + await contexts.experience_index.initialize(connection) + assert tuple( + hit.artifact_ref for hit in await contexts.search_experience("project", "falconcurrent", 8) + ) == (replaced.result_artifact,) asyncio.run(scenario()) diff --git a/tests/e2e/test_lexical_projection_upgrade.py b/tests/e2e/test_lexical_projection_upgrade.py index 0e2239d8d3..155dae193a 100644 --- a/tests/e2e/test_lexical_projection_upgrade.py +++ b/tests/e2e/test_lexical_projection_upgrade.py @@ -24,6 +24,7 @@ from powercontext.builtin.persistence.atomic_memory_index_schema import atomic_memory_current_table from powercontext.builtin.persistence.tables import ( ARTIFACT_HEADS_TABLE, + LEXICAL_PROJECTION_VERSIONS_TABLE, TOPIC_MEMORY_ACTIVE_CHUNKS_TABLE, TOPIC_MEMORY_ACTIVE_TOPICS_TABLE, ) @@ -78,6 +79,7 @@ async def scenario(): async with migration_profile(migration_backend.config) as profile, profile.database.transaction() as connection: # Canonical revisions, states and vectors survive an analyzer change. + await connection.execute(LEXICAL_PROJECTION_VERSIONS_TABLE.delete()) for table in ( atomic_memory_current_table(), TOPIC_MEMORY_ACTIVE_TOPICS_TABLE, @@ -111,5 +113,15 @@ async def scenario(): assert (await application.search("atomic-memory", {"query": "freshmarker"})).matches[ 0 ].artifact_ref.artifact_id == fresh.artifact_id + current_vectors = await runtime.atomic_memory.for_scope(scope).search(marker, mode="vector") + + async with open_builtin_runtime( + config, embedding_model=embedding, scheduler_path=migration_backend.directory / "scheduler" + ) as runtime: + for family, original in originals.items(): + result = await runtime.artifacts.for_scope(scope).search(family, {"query": marker, "mode": "text"}) + assert [match.artifact_ref.artifact_id for match in result.matches] == [original.artifact_id] + assert runtime.atomic_memory is not None + assert await runtime.atomic_memory.for_scope(scope).search(marker, mode="vector") == current_vectors asyncio.run(scenario()) From 924623ad4b60a2bd8957bb3216699d7b1a707ba2 Mon Sep 17 00:00:00 2001 From: Chojan Shang Date: Fri, 16 Oct 2026 00:05:13 +0800 Subject: [PATCH 9/9] fix(search): preserve quoted contractions --- src/powercontext/builtin/artifacts/search.py | 5 ++++- tests/builtin/artifacts/memory/test_fusion.py | 16 ++++++++++++++++ .../artifacts/test_text_search_admission.py | 2 +- 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/src/powercontext/builtin/artifacts/search.py b/src/powercontext/builtin/artifacts/search.py index 50f2dbaf0f..67d6b608eb 100644 --- a/src/powercontext/builtin/artifacts/search.py +++ b/src/powercontext/builtin/artifacts/search.py @@ -114,7 +114,10 @@ def _tokenizer() -> jieba.Tokenizer: return jieba.Tokenizer() -_QUOTED_QUERY_TEXT = re.compile(r"""(`[^`]*`|"[^"]*"|“[^”]*”|‘[^’]*’|(? None: @@ -224,6 +225,21 @@ def test_quoted_function_words_remain_lexical_evidence(query) -> None: assert admit_fts_candidates(query, (operators, arithmetic)) == (operators,) +@pytest.mark.parametrize( + ("query", "text"), + [ + ("'I'm'", "I'm"), + ("'we're'", "we're"), + ("‘don’t’", "don't"), # noqa: RUF001 - real query quotes and apostrophe. + ], +) +def test_quoted_contractions_remain_lexical_evidence(query, text) -> None: + contraction = channel_hit("contraction", text=text) + unrelated = channel_hit("unrelated", text="Deployment evidence.") + + assert admit_fts_candidates(query, (contraction, unrelated)) == (contraction,) + + def test_vector_admission_converts_unit_l2_distance_to_cosine_threshold() -> None: boundary = (2.0 * (1.0 - 0.3)) ** 0.5 accepted = channel_hit("accepted", distance=boundary) diff --git a/tests/builtin/artifacts/test_text_search_admission.py b/tests/builtin/artifacts/test_text_search_admission.py index 3194985c24..166012db61 100644 --- a/tests/builtin/artifacts/test_text_search_admission.py +++ b/tests/builtin/artifacts/test_text_search_admission.py @@ -44,7 +44,7 @@ def test_answer_format_does_not_displace_topic_evidence(suffix: str) -> None: ), ("If no reliable backup is present, stop deployment.", "no reliable backup stop deployment"), ("Answer briefly and cite rollback_mode v2.7 without approval.", "rollback_mode v2.7 without approval"), - ("请保留 release_gate_v3 并且不要部署未审核的版本。", "release_gate_v3 不要部署未审核的版本"), + ("请保留 release_gate_v3 并且不要部署未审核的版本。", "release_gate_v3 不要部署未审核版本"), ], ) def test_query_normalization_preserves_quotes_domain_conditions_and_identifiers(content: str, protected: str) -> None: