feat(search): unify word analysis across backends

.licenserc.yaml

@@ -63,6 +63,8 @@ header:
     - '**/py.typed'
     - 'src/powercontext/http/_generated/**'
     # Vendored distributions retain their upstream licenses and copyright notices.
+    - 'LICENSES/**'
+    - 'src/powercontext/builtin/artifacts/lexical_data/*.txt'
     - 'src/powercontext/server/dashboard/static/vendor/**'
     - '.gitattributes'
     - '**/*.service.in'

LICENSES/stopwords.txt

@@ -0,0 +1,36 @@
+Bundled stopword lists
+======================
+
+The following files are redistributed under the MIT license below:
+
+- src/powercontext/builtin/artifacts/lexical_data/en.txt
+  Source: stopwords-iso/stopwords-en, raw/snowball-tartarus.txt
+  Revision: ccc8898188850d8fb019d5f69c14a6635c3bd115
+  https://github.com/stopwords-iso/stopwords-en/tree/ccc8898188850d8fb019d5f69c14a6635c3bd115
+
+- src/powercontext/builtin/artifacts/lexical_data/zh.txt
+  Source: stopwords-iso/stopwords-zh, stopwords-zh.txt
+  Revision: 0f62ec9b9061b9142ed87fa6b39d386945c7cf47
+  https://github.com/stopwords-iso/stopwords-zh/tree/0f62ec9b9061b9142ed87fa6b39d386945c7cf47
+
+The MIT License (MIT)
+
+Copyright (c) 2016 Gene Diaz
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

docs/en/docs/operate/troubleshoot.md

@@ -268,7 +268,7 @@ so the previous database remains available for recovery:
 
    ```bash
    obloader <connection-options> -D <new-database> --csv \
-      --table 'pc_scopes,pc_source_journal_heads,pc_sources,pc_artifacts,pc_source_cursors,pc_memory_source_windows,pc_artifact_processing_leases,pc_artifact_processing_binding_states,pc_artifact_processing_pending,pc_artifact_processing_auto_wave_targets,pc_artifact_processing_sequences,pc_artifact_processing_intents,pc_topic_memory_processing_targets,pc_artifact_processing_schema,pc_artifact_processing_migration_receipts,pc_topic_memory_work_budgets,pc_topic_memory_retrieval_shape,pc_connector_checkpoints,pc_source_definition_manifests,pc_external_skill_registrations,pc_skill_packages,pc_agent_skill_targets,pc_skill_publications,pc_model_usage_daily,pc_recall_token_daily,pc_recall_effort_daily,pc_receipt_migration_review,pc_decision_observations' \
+      --table 'pc_scopes,pc_source_journal_heads,pc_sources,pc_artifacts,pc_source_cursors,pc_memory_source_windows,pc_artifact_processing_leases,pc_artifact_processing_binding_states,pc_artifact_processing_pending,pc_artifact_processing_auto_wave_targets,pc_artifact_processing_sequences,pc_artifact_processing_intents,pc_topic_memory_processing_targets,pc_artifact_processing_schema,pc_artifact_processing_migration_receipts,pc_topic_memory_work_budgets,pc_topic_memory_retrieval_shape,pc_connector_checkpoints,pc_source_definition_manifests,pc_external_skill_registrations,pc_skill_packages,pc_agent_skill_targets,pc_skill_publications,pc_model_usage_daily,pc_recall_token_daily,pc_recall_effort_daily,pc_receipt_migration_review,pc_decision_observations,pc_lexical_projection_versions' \
      -f <export-directory>
    ```
 

docs/en/docs/workflows/search-artifacts.md

@@ -70,6 +70,15 @@ Thresholds are applied after admission and score calculation, before the final l
 scoring candidates to fill a result after `min_score` removes hits. Enabling scores changes only response metadata;
 it does not change candidate admission, identities, ordering, or thresholds.
 
+## Shared word analysis
+
+Chinese text is segmented into words; Latin words and identifiers remain whole. Unquoted queries omit common English
+and Chinese stopwords while retaining negations. Quote a stopword to search for it; quotations do not request phrase matching.
+
+Record formats remain v1. Before upgrading, stop existing Server and Worker processes. The first startup after a
+document analyzer upgrade updates searchable text without changing record content, history or vectors.
+This startup can take longer as stored data grows.
+
 ## Experience and Skill text search
 
 Both Families search one `text` channel. Their admission fields are:

docs/zh/docs/operate/troubleshoot.md

@@ -259,7 +259,7 @@ collation，但不会包含数据库 URL 或凭据。
 
    ```bash
    obloader <connection-options> -D <new-database> --csv \
-      --table 'pc_scopes,pc_source_journal_heads,pc_sources,pc_artifacts,pc_source_cursors,pc_memory_source_windows,pc_artifact_processing_leases,pc_artifact_processing_binding_states,pc_artifact_processing_pending,pc_artifact_processing_auto_wave_targets,pc_artifact_processing_sequences,pc_artifact_processing_intents,pc_topic_memory_processing_targets,pc_artifact_processing_schema,pc_artifact_processing_migration_receipts,pc_topic_memory_work_budgets,pc_topic_memory_retrieval_shape,pc_connector_checkpoints,pc_source_definition_manifests,pc_external_skill_registrations,pc_skill_packages,pc_agent_skill_targets,pc_skill_publications,pc_model_usage_daily,pc_recall_token_daily,pc_recall_effort_daily,pc_receipt_migration_review,pc_decision_observations' \
+      --table 'pc_scopes,pc_source_journal_heads,pc_sources,pc_artifacts,pc_source_cursors,pc_memory_source_windows,pc_artifact_processing_leases,pc_artifact_processing_binding_states,pc_artifact_processing_pending,pc_artifact_processing_auto_wave_targets,pc_artifact_processing_sequences,pc_artifact_processing_intents,pc_topic_memory_processing_targets,pc_artifact_processing_schema,pc_artifact_processing_migration_receipts,pc_topic_memory_work_budgets,pc_topic_memory_retrieval_shape,pc_connector_checkpoints,pc_source_definition_manifests,pc_external_skill_registrations,pc_skill_packages,pc_agent_skill_targets,pc_skill_publications,pc_model_usage_daily,pc_recall_token_daily,pc_recall_effort_daily,pc_receipt_migration_review,pc_decision_observations,pc_lexical_projection_versions' \
      -f <export-directory>
    ```
 

docs/zh/docs/workflows/search-artifacts.md

@@ -67,6 +67,14 @@ Topic 和 Atomic 向量或混合检索还需要兼容的 Embedding 模型和索
 准入和评分完成后，先应用 `min_score`，再截取最终 `limit`。阈值过滤后不会用更低分的候选补齐数量。
 开启评分只增加响应元数据，不改变候选准入、结果身份、顺序或阈值行为。
 
+## 统一分词
+
+中文按词分割，拉丁词和标识符保留为完整词。未加引号的查询会过滤常见中英文停止词，但保留否定词。
+要检索停止词，请给它加引号；引号不表示短语匹配。
+
+记录格式保持 v1。升级前停止现有 Server 和 Worker 进程。分词器升级后，首次启动会更新已有记录的检索词元，
+正文、历史和向量保持不变。已有数据越多，首次启动可能越慢。
+
 ## Experience 和 Skill 文本检索
 
 这两个 Family 都使用一个 `text` 通道，支持以下准入字段：

pyproject.toml

@@ -18,6 +18,7 @@ dynamic = ["version"]
 description = "PowerContext turns human-agent work into handoff-ready context."
 authors = [{ name = "PowerContext Team", email = "open_oceanbase@oceanbase.com" }]
 readme = "README.md"
+license-files = ["LICENSE", "LICENSES/*"]
 keywords = ['python']
 requires-python = ">=3.11,<4.0"
 dependencies = [
@@ -51,6 +52,7 @@ builtin = [
     "apscheduler>=3.11,<4",
     "filelock>=3.16,<4",
     "jsonschema>=4.23,<5",
+    "jieba==0.42.1",
     "packaging>=24,<27",
     "pydantic-ai-slim[anthropic,openai]>=2.27.1,<3",
     "pydantic-settings>=2.7,<3",

src/powercontext/builtin/artifacts/lexical_data/en.txt

@@ -0,0 +1,228 @@
+i
+me
+my
+myself
+we
+us
+our
+ours
+ourselves
+you
+your
+yours
+yourself
+yourselves
+he
+him
+his
+himself
+she
+her
+hers
+herself
+it
+its
+itself
+they
+them
+their
+theirs
+themselves
+what
+which
+who
+whom
+this
+that
+these
+those
+am
+is
+are
+was
+were
+be
+been
+being
+have
+has
+had
+having
+do
+does
+did
+doing
+will
+would
+shall
+should
+can
+could
+may
+might
+must
+ought
+i'm
+you're
+he's
+she's
+it's
+we're
+they're
+i've
+you've
+we've
+they've
+i'd
+you'd
+he'd
+she'd
+we'd
+they'd
+i'll
+you'll
+he'll
+she'll
+we'll
+they'll
+isn't
+aren't
+wasn't
+weren't
+hasn't
+haven't
+hadn't
+doesn't
+don't
+didn't
+won't
+wouldn't
+shan't
+shouldn't
+can't
+cannot
+couldn't
+mustn't
+let's
+that's
+who's
+what's
+here's
+there's
+when's
+where's
+why's
+how's
+daren't
+needn't
+doubtful
+oughtn't
+mightn't
+a
+an
+the
+and
+but
+if
+or
+because
+as
+until
+while
+of
+at
+by
+for
+with
+about
+against
+between
+into
+through
+during
+before
+after
+above
+below
+to
+from
+up
+down
+in
+out
+on
+off
+over
+under
+again
+further
+then
+once
+here
+there
+when
+where
+why
+how
+all
+any
+both
+each
+few
+more
+most
+other
+some
+such
+no
+nor
+not
+only
+own
+same
+so
+than
+too
+very
+one
+every
+least
+less
+many
+now
+ever
+never
+say
+says
+said
+also
+get
+go
+goes
+just
+made
+make
+put
+see
+seen
+whether
+like
+well
+back
+even
+still
+way
+take
+since
+another
+however
+two
+three
+four
+five
+first
+second
+new
+old
+high
+long