diff --git a/docs/knowledge-operations.md b/docs/knowledge-operations.md index 4f9c9f6d8..e3a5833cf 100644 --- a/docs/knowledge-operations.md +++ b/docs/knowledge-operations.md @@ -92,6 +92,20 @@ order regardless of the cue; `selection.ranking_applied` is false for those call Ordinary collection of other authorities and records remains available without the relevance opt-in. `KnowledgeStore.collect` delegates these same options. +The default `relevance_method="weighted-lexical-v1"` searches the JSON +representation of every field, including strings. Explicitly select +`relevance_method="weighted-raw-string-v1"` (CLI: +`--relevance-method weighted-raw-string-v1`) to match original string values +instead: an actual newline or tab no longer creates a literal `n` or `t` match +from its JSON escape. Literal backslash-letter text remains searchable. +Nonstring fields, including the combined tags/applicability value, retain their +legacy JSON representation. Both methods keep the same substring matching, +distinct cue words, weights, positive-score admission and identity tie order. +The nondefault method requires `relevance`; unsupported methods are rejected. +Explicit IDs still bypass ranking. This option changes only owner discovery +ranking, not query matching or optional context-resource scoring. It is a +lexical-presence contract option, not evidence of general retrieval quality. + `context_bytes` requests a separate context view for returned generated packaged skills. It requires an explicit positive integer budget, with no token estimate or default. It can be used without `relevance`; optional resources then match the diff --git a/evidence/indexer-255-raw-string-20261030/README.md b/evidence/indexer-255-raw-string-20261030/README.md new file mode 100644 index 000000000..65dde0892 --- /dev/null +++ b/evidence/indexer-255-raw-string-20261030/README.md @@ -0,0 +1,65 @@ +# Opt-in raw-string lexical contract + +Related tracker: [Issue #255](https://github.com/InferMatrixLab/InferRSI/issues/255). +The single frozen comparison and independent saved-output audit passed **only the +synthetic lexical-presence contract**. No general task accuracy, latency, model +or hardware efficacy, default promotion or tracker completion is inferred. + +## Try the existing interface with the new explicit method + +From the source checkout with the package dependencies installed: + +```bash +PYTHONPATH=src python -m inferrsi collect 'cpu.stat.local' \ + --relevance cgroupv2 --relevance-method weighted-raw-string-v1 \ + --context-bytes 65536 --limit 1 --json +``` + +Omit `--relevance-method` for unchanged legacy selection. The query remains a +filter; the relevance cue ranks packaged skill owners. Context resources remain +whole files with exact owner revision and SHA-256. The fixed real-corpus example +returned the same complete 7,348-byte cgroup-resource-controls core in both +modes; this is an interface demonstration, not a quality improvement claim. + +Synthetic cue `n t`, with unchanged title weight 4: + +| Original title | Legacy score | Opt-in score | +|---|---:|---:| +| Actual newline + tab | 8 | 0 | +| Literal backslash-n + backslash-t | 8 | 8 | + +The phantom owner is no longer admitted. All declared genuine controls remain. +See the [readable before/after](results/readable-demo.txt) and +[independent audit](results/independent-postrun-review.json). + +## Retained evidence + +- Frozen baseline: `1052925ff22f9a850cebc64afd0083b35ed1fae9`. +- Frozen candidate: `728984c524cf696d063ff7ef0688ae126c8546a6`. +- [Exact protocol](plan/protocol.md), original [candidate patch](patch/frozen-candidate.diff) + and [Git bundle](patch/frozen-candidate.bundle). The Git bundle requires the + publicly retained baseline commit; it preserves the original local candidate + identity rather than substituting the later publication commit. +- [Complete frozen study archive](results/frozen-study.tar.gz), SHA-256 + `3074dbc24d198ff3d8ec440f839578340222b85600a5ff67c9d4186f88ec2b23`. + All 36 retained files are byte-preserved, including original fixtures, + prospective freezes, raw outputs, ordinary test logs, initial development + failure, superseded pre-review freeze, provenance, source/module fingerprints, + independent reviews and the one-run receipt. No scored comparison was repeated. +- [Current-main integration check](results/integration-check.json): clean application + to `436c950cf06ded742fef61ff4303493ac5175d4b`, 108 affected tests passed, + 2 skipped and Ruff passed. This does not rebase or replace the frozen study. + +## Results and boundaries + +All 14 scalar and 2 ordering fixtures matched their independently authored labels. +Five phantom field-token contributions were removed across two scalar cases; +15 original-string and 11 expected nonstring-JSON positive contributions were +retained. Six nonstring field controls and 21 compatibility requests preserved +legacy behavior, including optional context contents, hashes, revisions and +omissions. Defaults, explicit-ID membership/order, filters, query and budgets +are unchanged. Full ordinary CPU checks on the frozen candidate passed 2707 tests +and 284 subtests with 12 skips; these are not task-efficacy evidence. + +See [publication provenance](plan/publication.md) for the distinction between +new configured authorship, inherited historical evidence and CI qualification. diff --git a/evidence/indexer-255-raw-string-20261030/patch/frozen-candidate.bundle b/evidence/indexer-255-raw-string-20261030/patch/frozen-candidate.bundle new file mode 100644 index 000000000..2f6596c0b Binary files /dev/null and b/evidence/indexer-255-raw-string-20261030/patch/frozen-candidate.bundle differ diff --git a/evidence/indexer-255-raw-string-20261030/patch/frozen-candidate.diff b/evidence/indexer-255-raw-string-20261030/patch/frozen-candidate.diff new file mode 100644 index 000000000..14ae5479e --- /dev/null +++ b/evidence/indexer-255-raw-string-20261030/patch/frozen-candidate.diff @@ -0,0 +1,377 @@ +diff --git a/docs/knowledge-operations.md b/docs/knowledge-operations.md +index 4f9c9f6d..e3a5833c 100644 +--- a/docs/knowledge-operations.md ++++ b/docs/knowledge-operations.md +@@ -92,6 +92,20 @@ order regardless of the cue; `selection.ranking_applied` is false for those call + Ordinary collection of other authorities and records remains available without + the relevance opt-in. `KnowledgeStore.collect` delegates these same options. + ++The default `relevance_method="weighted-lexical-v1"` searches the JSON ++representation of every field, including strings. Explicitly select ++`relevance_method="weighted-raw-string-v1"` (CLI: ++`--relevance-method weighted-raw-string-v1`) to match original string values ++instead: an actual newline or tab no longer creates a literal `n` or `t` match ++from its JSON escape. Literal backslash-letter text remains searchable. ++Nonstring fields, including the combined tags/applicability value, retain their ++legacy JSON representation. Both methods keep the same substring matching, ++distinct cue words, weights, positive-score admission and identity tie order. ++The nondefault method requires `relevance`; unsupported methods are rejected. ++Explicit IDs still bypass ranking. This option changes only owner discovery ++ranking, not query matching or optional context-resource scoring. It is a ++lexical-presence contract option, not evidence of general retrieval quality. ++ + `context_bytes` requests a separate context view for returned generated packaged + skills. It requires an explicit positive integer budget, with no token estimate + or default. It can be used without `relevance`; optional resources then match the +diff --git a/src/inferrsi/_skill_selection.py b/src/inferrsi/_skill_selection.py +index d06f492c..e58c5ed5 100644 +--- a/src/inferrsi/_skill_selection.py ++++ b/src/inferrsi/_skill_selection.py +@@ -4,6 +4,8 @@ import json + import posixpath + import re + ++RELEVANCE_METHODS = ("weighted-lexical-v1", "weighted-raw-string-v1") ++ + + class SkillIndexError(ValueError): + pass +@@ -21,6 +23,16 @@ def lexical_score(cue: str, fields: list[tuple[int, object]]) -> int: + ) + + ++def raw_string_lexical_score(cue: str, fields: list[tuple[int, object]]) -> int: ++ """Match original strings; retain legacy JSON matching for nonstring fields.""" ++ tokens = words(cue) ++ score = 0 ++ for weight, value in fields: ++ text = value if isinstance(value, str) else json.dumps(value, ensure_ascii=False) ++ score += weight * sum(token in text.casefold() for token in tokens) ++ return score ++ ++ + def indexed_skill(record: dict, view: dict, content: dict[str, bytes]) -> tuple: + """Validate the existing projection against this exact canonical bundle.""" + registry_path = "data/skill_library/skill-index.json" +@@ -82,8 +94,13 @@ def indexed_skill(record: dict, view: dict, content: dict[str, bytes]) -> tuple: + return core, required, optional + + +-def skill_score(cue: str, record: dict, core: bytes) -> int: +- return lexical_score(cue, [ ++def skill_score( ++ cue: str, record: dict, core: bytes, *, method: str = "weighted-lexical-v1" ++) -> int: ++ if method not in RELEVANCE_METHODS: ++ raise ValueError("Unknown relevance method") ++ scorer = raw_string_lexical_score if method == "weighted-raw-string-v1" else lexical_score ++ return scorer(cue, [ + (4, record["title"]), + (3, record["summary"]), + (2, [record.get("tags", []), record.get("applicability", {})]), +diff --git a/src/inferrsi/arguments.py b/src/inferrsi/arguments.py +index 17c614df..46b5e933 100644 +--- a/src/inferrsi/arguments.py ++++ b/src/inferrsi/arguments.py +@@ -8,6 +8,7 @@ import sys + from pathlib import Path + + from . import __version__ ++from ._skill_selection import RELEVANCE_METHODS + from .catalog import BACKENDS, KNOWLEDGE_REPOS, KNOWLEDGE_SCOPES, REPO_TYPES + from .runspec import RunSpec + +@@ -191,6 +192,10 @@ def parser() -> argparse.ArgumentParser: + collect.add_argument("--max-bytes", type=int) + collect.add_argument("--material", action="store_true") + collect.add_argument("--relevance", help="Opt-in lexical cue for packaged skill selection") ++ collect.add_argument( ++ "--relevance-method", choices=RELEVANCE_METHODS, default="weighted-lexical-v1", ++ help="Packaged skill ranking method; nondefault method requires --relevance", ++ ) + collect.add_argument( + "--context-bytes", type=int, + help="Opt-in context view; total UTF-8 resource-body byte budget", +diff --git a/src/inferrsi/cli.py b/src/inferrsi/cli.py +index 9b883658..349e3df3 100644 +--- a/src/inferrsi/cli.py ++++ b/src/inferrsi/cli.py +@@ -193,6 +193,7 @@ def main(argv: list[str] | None = None) -> int: + max_bytes=args.max_bytes, + material=args.material, + relevance=args.relevance, ++ relevance_method=args.relevance_method, + context_bytes=args.context_bytes, + ) + if args.json: +diff --git a/src/inferrsi/knowledge_operations.py b/src/inferrsi/knowledge_operations.py +index 1c78418f..d776964d 100644 +--- a/src/inferrsi/knowledge_operations.py ++++ b/src/inferrsi/knowledge_operations.py +@@ -9,7 +9,7 @@ from datetime import date, datetime + from importlib.resources import files + from pathlib import Path, PurePosixPath + +-from ._skill_selection import context_view, indexed_skill, skill_score, words ++from ._skill_selection import RELEVANCE_METHODS, context_view, indexed_skill, skill_score, words + from .catalog import KNOWLEDGE_SCOPES, SCENARIO_ALIASES, knowledge_records + + AUTHORITIES = ("package", "checkout", "store") +@@ -490,6 +490,7 @@ class KnowledgeReader: + max_bytes: int | None = None, + material: bool = False, + relevance: str | None = None, ++ relevance_method: str = "weighted-lexical-v1", + context_bytes: int | None = None, + ) -> dict: + if type(limit) is not int or limit < 1: +@@ -504,6 +505,10 @@ class KnowledgeReader: + raise ValueError("Unknown content kind") + if relevance is not None and (not isinstance(relevance, str) or not words(relevance)): + raise ValueError("relevance requires at least one word") ++ if relevance_method not in RELEVANCE_METHODS: ++ raise ValueError("Unknown relevance method") ++ if relevance is None and relevance_method != "weighted-lexical-v1": ++ raise ValueError("relevance_method requires relevance") + if context_bytes is not None and (type(context_bytes) is not int or context_bytes < 1): + raise ValueError("context_bytes must be a positive integer") + if revisions is None: +@@ -645,7 +650,9 @@ class KnowledgeReader: + continue + try: + core, _, _ = indexed_skill(record, view, bundle.content) +- score = skill_score(relevance, record, bundle.content[core]) ++ score = skill_score( ++ relevance, record, bundle.content[core], method=relevance_method ++ ) + except (ValueError, KeyError, TypeError) as exc: + omitted.append({ + "id": reference, "reason": "index_unavailable", "detail": str(exc), +@@ -673,7 +680,7 @@ class KnowledgeReader: + } + if relevance is not None: + result["selection"] = { +- "method": "weighted-lexical-v1", "cue": relevance, ++ "method": relevance_method, "cue": relevance, + "scope": "generated-packaged-skills", "ranking_applied": ids is None, + } + result["limitations"][0] = ( +diff --git a/tests/test_raw_string_selection.py b/tests/test_raw_string_selection.py +new file mode 100644 +index 00000000..32b4ec72 +--- /dev/null ++++ b/tests/test_raw_string_selection.py +@@ -0,0 +1,213 @@ ++"""Ordinary synthetic compatibility controls, not a retrieval-quality benchmark.""" ++ ++import copy ++import hashlib ++import json ++ ++import pytest ++ ++from inferrsi._skill_selection import ( ++ indexed_skill, ++ lexical_score, ++ raw_string_lexical_score, ++ skill_score, ++) ++from inferrsi.cli import main ++from inferrsi.knowledge_operations import KnowledgeReader, _hash, _json ++from inferrsi.knowledge_store import KnowledgeStore ++ ++RAW = "weighted-raw-string-v1" ++LEGACY = "weighted-lexical-v1" ++ ++ ++def synthetic_reader(entries): ++ """Exercise the real package adapter using a complete in-memory test library.""" ++ registry_path = "data/skill_library/skill-index.json" ++ registry = {"schema_version": 1, "skills": {}} ++ library, records = {}, {} ++ for name, fields in entries.items(): ++ path = f"methods/{name}" ++ prefix = f"data/skill_library/{path}/" ++ registry["skills"][name] = {"path": path, "backend": "common"} ++ library[prefix + "SKILL.md"] = fields.get("core", "").encode() ++ for relative, text in fields.get("resources", {}).items(): ++ library[prefix + relative] = text.encode() ++ identity = f"inferrsi.common.skills.{name.replace('-', '_')}" ++ records[identity] = { ++ "id": identity, "title": fields.get("title", ""), ++ "summary": fields.get("summary", ""), "tags": fields.get("tags", []), ++ "applicability": fields.get("applicability", {}), ++ "sources": [{"type": "internal", "issue": "test", "artifact": "synthetic"}], ++ "status": "candidate", "verification": "synthetic-only", "repo": "inferrsi", ++ "backend": "common", "scenario": "skills.discovery", "tier": "contribution", ++ "index_type": "packaged-skill", "skill_documents": {"original": prefix + "SKILL.md"}, ++ "skill_metadata": {"name": name, "references": fields.get("references", [])}, ++ } ++ library[registry_path] = _json(registry) ++ for record in records.values(): ++ prefix = record["skill_documents"]["original"].removesuffix("SKILL.md") ++ record["source_files"] = { ++ "src/inferrsi/" + path: _hash(data) ++ for path, data in library.items() if path.startswith(prefix) or path == registry_path ++ } ++ reader = KnowledgeReader() ++ reader._records = records ++ reader._library = library, "sha256:" + _hash(_json({ ++ path: _hash(data) for path, data in sorted(library.items()) ++ })) ++ return reader ++ ++ ++def without_method(result): ++ value = copy.deepcopy(result) ++ value.get("selection", {}).pop("method", None) ++ return value ++ ++ ++@pytest.mark.parametrize("value", [None, True, 13.5, ["\n\t", None], {"n": "\r"}]) ++def test_nonstring_values_retain_exact_legacy_json_matching(value): ++ cue = "n t r null true 13 5" ++ assert raw_string_lexical_score(cue, [(2, value)]) == lexical_score(cue, [(2, value)]) ++ ++ ++def test_raw_string_matching_does_not_change_the_legacy_helper(): ++ assert raw_string_lexical_score("n t", [(4, "\n\t")]) == 0 ++ assert lexical_score("n t", [(4, "\n\t")]) == 8 ++ assert raw_string_lexical_score("n t", [(4, r"\n\t")]) == 8 ++ assert raw_string_lexical_score("RUN run STRASSE", [(3, "running Straße")]) == 6 ++ ++ ++def test_skill_adapter_keeps_field_weights_and_combined_nonstring_metadata(): ++ record = {"title": "n", "summary": "n", "tags": ["\n"], "applicability": {}} ++ assert skill_score("n N", record, b"n", method=RAW) == 10 ++ assert skill_score("n N", record, b"n") == 10 ++ with pytest.raises(ValueError, match="Unknown relevance method"): ++ skill_score("n", record, b"n", method="unknown") ++ ++ ++def test_opt_in_changes_only_raw_string_owner_ranking_and_positive_admission(): ++ reader = synthetic_reader({ ++ "a-phantom": {"title": "\n", "summary": "\n", "core": "\n"}, ++ "b-hit": {"title": "n"}, "c-hit": {"title": "n"}, ++ }) ++ legacy = reader.collect(relevance="n") ++ assert legacy == reader.collect(relevance="n", relevance_method=LEGACY) ++ assert [r["title"] for r in legacy["items"]] == ["\n", "n", "n"] ++ raw = reader.collect(relevance="n", relevance_method=RAW) ++ expected = ["inferrsi.common.skills.b_hit", "inferrsi.common.skills.c_hit"] ++ assert [r["id"] for r in raw["items"]] == expected ++ assert raw["selection"] == { ++ "method": RAW, "cue": "n", "scope": "generated-packaged-skills", "ranking_applied": True, ++ } ++ assert reader.collect(relevance="n", relevance_method=RAW, limit=1)["items"] == raw["items"][:1] ++ assert reader.collect() == reader.collect(relevance_method=LEGACY) ++ ++ ++@pytest.mark.parametrize("options", [ ++ {"query": "missing"}, {"query": "synthetic n"}, ++ {"kind": "skill"}, {"repo": "vllm"}, {"backend": "cpu"}, ++ {"tier": "peer"}, {"scenario": "diffusion.t2i"}, {"material": True}, ++]) ++def test_query_and_filters_preserve_ordinary_semantics(options): ++ reader = synthetic_reader({"item": {"title": "running", "core": "n"}}) ++ legacy = reader.collect(relevance="n", **options) ++ raw = reader.collect(relevance="n", relevance_method=RAW, **options) ++ assert without_method(raw) == without_method(legacy) ++ ++ ++def test_explicit_ids_keep_membership_order_aliases_revisions_and_item_budgets(): ++ reader = synthetic_reader({"a": {"title": "\n"}, "b": {"title": "n"}}) ++ a, b = sorted(reader.records()) ++ options = {"ids": [b, a, "package:" + a], "relevance": "absent", "material": True} ++ legacy = reader.collect(**options) ++ raw = reader.collect(**options, relevance_method=RAW) ++ assert without_method(raw) == without_method(legacy) ++ assert [r["id"] for r in raw["items"]] == [b, a] ++ assert raw["selection"]["ranking_applied"] is False ++ for extra in ( ++ {"limit": 1}, {"max_bytes": len(_json(raw["items"][0]))}, ++ {"revisions": {a: reader.read(a).revision}}, {"revisions": {a: "sha256:" + "0" * 64}}, ++ ): ++ assert without_method(reader.collect(**options, **extra, relevance_method=RAW)) == ( ++ without_method(reader.collect(**options, **extra)) ++ ) ++ ++ ++@pytest.mark.parametrize("explicit", [False, True]) ++def test_context_required_hashes_atomic_budgets_and_optional_legacy_order(explicit): ++ reader = synthetic_reader({"owner": { ++ "title": "n", "core": "Core é", "references": ["refs.md"], ++ "resources": {"refs.md": "Required", "LICENSE": "Legal", "a.md": "\n", "b.md": "n"}, ++ }}) ++ identity, = reader.records() ++ bundle = reader.read(identity) ++ view = {**bundle.view(), "authority": "package"} ++ _, required, _ = indexed_skill(reader.records()[identity], view, bundle.content) ++ required_bytes = sum(len(bundle.content[path]) for path in required) ++ options = {"relevance": "n"} ++ if explicit: ++ options["ids"] = [identity] ++ for budget in (required_bytes - 1, required_bytes, required_bytes + 1, required_bytes + 2): ++ legacy = reader.collect(**options, context_bytes=budget) ++ raw = reader.collect(**options, context_bytes=budget, relevance_method=RAW) ++ assert without_method(raw) == without_method(legacy) ++ if budget < required_bytes: ++ assert not raw["contexts"] ++ assert raw["context_omissions"][0]["reason"] == "context_limit" ++ continue ++ context, = raw["contexts"] ++ assert context["revision"] == bundle.revision ++ assert context["content_bytes"] == budget ++ resources = context["resources"] ++ assert [r["path"] for r in resources[:len(required)]] == required ++ for resource in resources: ++ data = reader.resource(identity, resource["path"], revision=context["revision"]) ++ assert resource["content"].encode() == data ++ assert resource["sha256"] == hashlib.sha256(data).hexdigest() ++ if budget > required_bytes: ++ # JSON's newline contribution must still admit the optional a.md first. ++ assert resources[len(required)]["path"].endswith("/a.md") ++ ++ ++@pytest.mark.parametrize("options", [ ++ {"relevance_method": RAW}, {"relevance_method": "unknown"}, ++ {"relevance": "n", "relevance_method": "unknown"}, ++ {"relevance": "n", "relevance_method": None}, ++ {"relevance": "n", "relevance_method": []}, ++ {"relevance": "!!!", "relevance_method": RAW}, ++]) ++def test_method_validation_happens_before_reader_access(monkeypatch, options): ++ reader = KnowledgeReader() ++ ++ def unexpected_access(): ++ raise AssertionError("invalid request accessed owners") ++ ++ monkeypatch.setattr(reader, "owners", unexpected_access) ++ with pytest.raises(ValueError): ++ reader.collect(**options) ++ ++ ++def test_raw_method_keeps_fail_closed_index_validation(): ++ reader = synthetic_reader({"owner": {"title": "n"}}) ++ identity, = reader.records() ++ reader.records()[identity]["source_files"] = {} ++ result = reader.collect(relevance="n", relevance_method=RAW) ++ assert result["no_match"] ++ assert result["omissions"][0]["reason"] == "index_unavailable" ++ ++ ++def test_cli_and_store_delegate_the_explicit_method(monkeypatch, capsys, tmp_path): ++ reader = synthetic_reader({"owner": {"title": "n"}}) ++ monkeypatch.setattr("inferrsi.cli._knowledge_reader", lambda args: reader) ++ assert main(["collect", "--relevance", "n", "--relevance-method", RAW, "--json"]) == 0 ++ assert json.loads(capsys.readouterr().out)["selection"]["method"] == RAW ++ with pytest.raises(SystemExit): ++ main(["collect", "--relevance", "n", "--relevance-method", "unknown"]) ++ with pytest.raises(SystemExit) as error: ++ main(["collect", "--relevance-method", RAW]) ++ assert error.value.code == 2 ++ store = KnowledgeStore(tmp_path, skill_root=None) ++ monkeypatch.setattr(store, "reader", lambda: reader) ++ assert store.collect(relevance="n", relevance_method=RAW) == reader.collect( ++ relevance="n", relevance_method=RAW ++ ) diff --git a/evidence/indexer-255-raw-string-20261030/plan/integrity.json b/evidence/indexer-255-raw-string-20261030/plan/integrity.json new file mode 100644 index 000000000..5c93339e6 --- /dev/null +++ b/evidence/indexer-255-raw-string-20261030/plan/integrity.json @@ -0,0 +1,12 @@ +{ + "README.md": "4830238c7cbddd4a209a0673aaeedd45fb979e86691d82d3124414cc5a04155b", + "patch/frozen-candidate.bundle": "ca30a22261cca91e520e6040522040171e04a22ff27bb526fc9b0e152e21c1e5", + "patch/frozen-candidate.diff": "5e94426574d7105f4354e76c10daa27aa0ac59365517372fa31e302c624e85ff", + "plan/protocol.md": "a91c63b85b3e168c06a4fb2986b92256a14679406b955baa6853e743712c3dfc", + "plan/publication.md": "5675418b722b852cbedc2f916d9b4052a4b7044bc867c0f4ffe53cf5a499cfc6", + "results/frozen-study.tar.gz": "3074dbc24d198ff3d8ec440f839578340222b85600a5ff67c9d4186f88ec2b23", + "results/independent-postrun-review.json": "0aa02a6745e91fb818baabfef5a5a9cf1eb7cdb3be1fb098cc54f01257429e9f", + "results/integration-check.json": "e696cbcdd4fee200d1e4ca82484e1689293a2bb71669802d2823bdad5bc7b399", + "results/readable-demo.txt": "bb47077e52ce03e6412caa938fd229b59c5a65357162560237ce1eb643dfd7d1", + "validator/README.md": "0ef597749b676eeb1df2ea1ad8c12c116f76890c172d985e1ab073d8653991bb" +} diff --git a/evidence/indexer-255-raw-string-20261030/plan/protocol.md b/evidence/indexer-255-raw-string-20261030/plan/protocol.md new file mode 100644 index 000000000..02583d97f --- /dev/null +++ b/evidence/indexer-255-raw-string-20261030/plan/protocol.md @@ -0,0 +1,36 @@ +# #255 raw-string lexical-presence contract study — prospective protocol + +Draft for coordinator review, 2026-10-30T09:08Z. No candidate or scoring yet. + +## Authority, baseline and objective + +Continue owner-authorized #255 small constructed accuracy cases, without algorithm rewrite, corpus changes, paid/model calls or hardware. Public-query availability is NOT a prerequisite. Main verified through GitHub at preparation: `1052925ff22f9a850cebc64afd0083b35ed1fae9`, tree `bf2fc3880768c6c453e862b31c5fc658f8a400a8`. Freeze this historical baseline; a concurrent merge does not silently replace it. + +Documented lexical contract: `docs/knowledge-operations.md:82–91` awards distinct casefolded cue words for substring presence in original title/description/core, weighted4/3/2/1. `_skill_selection.py:16–20` currently serializes all fields to JSON, introducing alphabetic escape prefixes absent from original string values. Objective is eliminate that specific false-positive contribution, not optimize semantic task success or improve natural-language retrieval generally. + +## One candidate and protected paths + +Add one explicitly named/versioned opt-in ranking method `weighted-raw-string-v1` (final name must freeze before implementation). For string fields only, compute membership in the original `value.casefold()`; for nonstring objects retain EXACT legacy `json.dumps(value, ensure_ascii=False).casefold()`. Keep cue regex/casefold/unique token set, substring morphology, field weights, score sums, positive-score admission, identity tie-break, filters, query, limits and resource semantics unchanged. + +Default legacy method unchanged. Explicit-ID membership/order unchanged. Shared optional-resource scoring remains legacy for all modes; do not globally change `lexical_score` and accidentally alter default context delivery. No changes to metadata, generated indexes, skill text, query rewriting, method weights, services or schema. No token-boundary or stemming change. Historical v2/v3 failures retained separately. + +## Independent fixtures and freezes + +A fresh no-history independent author drafts exactly 14 scalar contract cases plus two small synthetic-record ordering fixtures (count declared before drafting) before seeing candidate code/scores, old DEV/heldout or catalog. Cases include actual newline/tab/carriage-return versus literal backslash escapes; genuine and absent cues; Unicode/casefold; duplicates; substring morphology; weights/ties; null/numeric/list/dict fields. Expected membership and scores are manually justified from original bytes under the stated contract, with JSON representation explicitly retained for nonstrings. Case author records exact count before drafting. No cases are replaced after scorer exposure. Reviewer audits labels and freezes their hash; candidate author may then see these as development/contract fixtures, not heldout. + +Freeze protocol, fixtures, baseline source/corpus identities, runner and model/provenance preflight before candidate scoring. Coordinator must confirm required model-role provenance; unknown model identity must be recorded unknown, never invented. This work uses existing agent reasoning only, no separate model/API experiment. Candidate commit/hash frozen before executing its comparison. Old sealed semantic heldout remains unused and undisclosed. + +## Evaluation and gates + +One local CPU comparison only after protocol approval/preflight: +1. All frozen scalar/per-field contract labels must match candidate exactly (100%); report legacy mismatches separately as false-positive, false-negative or weight error. +2. At least one predeclared serialization-created false-positive contribution must be corrected; genuine substring contributions and nonstring behavior must be unchanged across all corresponding controls. Do not report a vague aggregate accuracy percentage without counts and denominator. +3. Both fixed ranking fixtures must match contract-derived order; no loss of the predeclared true-hit controls. Label results synthetic/adversarial and source-conditional to the contract. +4. Default output, explicit-ID order/membership, ordinary query/filter behavior, unsupported-method rejection, mandatory resource completeness, hashes/revisions and exact/insufficient context budgets must pass existing plus new isolated controls. Optional-resource selection must remain byte-identical to legacy for identical selected owners. +5. A fixed existing-corpus diagnostic is optional only if separately specified/frozen BEFORE execution; it is NOT needed to demonstrate this scalar contract discrepancy and cannot become posthoc selection of wins. No old DEV or heldout reuse to claim validation. + +No latency-effect claim: only report available execution duration/output bytes as diagnostic overhead, never as paired performance evidence. No timing campaign/retries/threshold tuning. Any candidate contract or compatibility failure means FAIL and stops; unavailable safe execution/preflight means INCONCLUSIVE with no unprotected fallback. Preserve all outputs and errors. No second candidate automatically. + +## Interpretation / integration boundary + +PASS supports correcting phantom lexical evidence in this opt-in mode on explicit synthetic contracts, not general semantic retrieval superiority, real-user distribution, hardware efficacy or whole #255 closure. Fixed-corpus quality, broader task relevance and actual caller utility remain separately bounded. Runtime publication needs scoped source review and applicable CI; evidence may be delivered even if candidate fails. Default method may not be promoted under this protocol. diff --git a/evidence/indexer-255-raw-string-20261030/plan/publication.md b/evidence/indexer-255-raw-string-20261030/plan/publication.md new file mode 100644 index 000000000..80511c5ca --- /dev/null +++ b/evidence/indexer-255-raw-string-20261030/plan/publication.md @@ -0,0 +1,28 @@ +# Publication wrapper, not a new experiment + +The source patch is integrated onto verified main +`436c950cf06ded742fef61ff4303493ac5175d4b`. The six candidate files are byte-equal +to the frozen candidate. The additional wrapper, archive and inventory entry only +publish retained evidence; they do not alter gates, fixtures, scoring or defaults. + +The new material source/docs/test author is the explicitly configured +`gpt-6-astra` / `xhigh` collaboration agent +`/root/implement_raw_string_contract`. This reports the actual supplied new-task +configuration, not an independent hidden-backend attestation. The PR declaration +must name the current publication head, not the earlier experiment head. + +Protocol/fixture authors and the independent reviewer are separately attributed +inside the retained archive; their unknown model configurations stay unknown. +They did not author this candidate's protected source/document changes. The +post-run audit script is explicitly a supplemental saved-output auditor, not a +prospectively frozen scorer. No historical PR authorship is retrospectively +certified by this new declaration or by green CI. + +The original acceptance receipt records a prior configuration block and remains +byte-preserved. The later new-task configuration/delegation is recorded separately. +Historical v2/v3 failures remain separate; no old sealed heldout was accessed. + +Original full local suite used Python 3.12.14. The integration check also used +Python 3.12.14. Hosted Python 3.10/3.11 full-suite and clean-wheel checks are a +separate publication gate and must be read at the actual PR head. No paid/API +model call, hardware experiment or new dependency purchase was performed. diff --git a/evidence/indexer-255-raw-string-20261030/results/frozen-study.tar.gz b/evidence/indexer-255-raw-string-20261030/results/frozen-study.tar.gz new file mode 100644 index 000000000..3d7c88c06 Binary files /dev/null and b/evidence/indexer-255-raw-string-20261030/results/frozen-study.tar.gz differ diff --git a/evidence/indexer-255-raw-string-20261030/results/independent-postrun-review.json b/evidence/indexer-255-raw-string-20261030/results/independent-postrun-review.json new file mode 100644 index 000000000..f92b616be --- /dev/null +++ b/evidence/indexer-255-raw-string-20261030/results/independent-postrun-review.json @@ -0,0 +1,24 @@ +{ + "reviewed_at_utc": "2026-10-30T10:03:01.156127+00:00", + "decision": "PASS_WITHIN_FROZEN_SYNTHETIC_CONTRACT_SCOPE", + "baseline_sha": "1052925ff22f9a850cebc64afd0083b35ed1fae9", + "candidate_sha": "728984c524cf696d063ff7ef0688ae126c8546a6", + "runner_sha256": "7abe02af59ae06a5f2e7956d5236573172810b84025223714ded593c1e6c8061", + "fixtures_sha256": "70086b95ba74c182cb195c32d71fed7aed747bec975079160f10bf8fff673e18", + "method": "Independent retained-output label/arithmetic/resource/hash/source/module audit; no scorer/retrieval/candidate execution.", + "scalar_cases": 14, + "ranking_fixtures": 2, + "phantom_field_token_contributions_removed": 5, + "genuine_field_token_controls_retained": 26, + "nonstring_controls_unchanged": 6, + "compatibility_requests": 21, + "source_and_module_identity_verified": true, + "default_parity_verified": true, + "context_hash_bytes_required_core_ref_license_and_budget_verified": true, + "fixed_cli_demo": "Source resource hashes and bytes verified; interface-only, no semantic relevance or latency inference.", + "limits": [ + "Synthetic adversarial lexical contract, not general task accuracy or user distribution.", + "No default-method promotion or #255 closure implied.", + "Historical failures remain distinct; old heldout unused." + ] +} diff --git a/evidence/indexer-255-raw-string-20261030/results/integration-check.json b/evidence/indexer-255-raw-string-20261030/results/integration-check.json new file mode 100644 index 000000000..b2b4fd219 --- /dev/null +++ b/evidence/indexer-255-raw-string-20261030/results/integration-check.json @@ -0,0 +1,39 @@ +{ + "checked_at_utc": "2026-10-30T10:25:27.327562+00:00", + "scope": "Local integration readiness check only; no new scoring study, full current-main CI qualification, publication or merge.", + "base_sha": "436c950cf06ded742fef61ff4303493ac5175d4b", + "candidate_sha": "728984c524cf696d063ff7ef0688ae126c8546a6", + "patch_sha256": "5e94426574d7105f4354e76c10daa27aa0ac59365517372fa31e302c624e85ff", + "tested_tree": "fa8af8bbe8f5acd61f751d4139dee6fc34b3cd39", + "patch_application": "CLEAN_NO_CONFLICT", + "candidate_changed_file_bytes": "All six equal to frozen candidate hashes", + "tests": { + "command": "python -m pytest tests/test_raw_string_selection.py tests/test_collect_selection.py tests/test_knowledge_operations.py tests/test_knowledge_store.py", + "passed": 108, + "skipped": 2, + "seconds": 63.51, + "python": "3.12.14" + }, + "ruff": { + "command": "python -m ruff check src tests tools", + "result": "PASS" + }, + "full_suite_current_main": "NOT_RUN; prior frozen candidate full suite remains separately recorded", + "installed_wheel_and_python310_311": "NOT_RUN", + "frozen_candidate_baseline_evidence_delivery": "Unchanged; before/after SHA256SUMS verification passed", + "default_behavior_change": false, + "scored_study_rerun": false, + "remote_actions": false, + "publication_permission": "PENDING", + "logs": { + "frozen-evidence-integrity-before.txt": "91c1281080b882ea3bd430d1148a412a6792b49f6a1dbf152fef0913718dc3da", + "affected-tests.txt": "66a8c737e986d446fde6e54634a8279e69f15db43a9fda5f4d0ab4d02fb9aad6", + "ruff.txt": "82b3e6a6c090a57601d22943bd23fca9218d1031dbe5a7b754092f9a156b4f18", + "delivery-integrity-before.txt": "39854bb83a5e000ebb29b4e38cd4e8aad61bc1fedfe841ae6f3ffb27f64c5b6d", + "delivery-integrity-after.txt": "39854bb83a5e000ebb29b4e38cd4e8aad61bc1fedfe841ae6f3ffb27f64c5b6d", + "source-identity.json": "4302e13827c79ace78b820ef5aabcc995896ec36eecbf2843d2c6d3f1d9afb48", + "affected-tests.exit-code": "9a271f2a916b0b6ee6cecb2426f0b3206ef074578be55d9bc94f6f3fe3ab86aa", + "frozen-evidence-integrity-after.txt": "91c1281080b882ea3bd430d1148a412a6792b49f6a1dbf152fef0913718dc3da", + "ruff.exit-code": "9a271f2a916b0b6ee6cecb2426f0b3206ef074578be55d9bc94f6f3fe3ab86aa" + } +} diff --git a/evidence/indexer-255-raw-string-20261030/results/readable-demo.txt b/evidence/indexer-255-raw-string-20261030/results/readable-demo.txt new file mode 100644 index 000000000..b8b57dfe9 --- /dev/null +++ b/evidence/indexer-255-raw-string-20261030/results/readable-demo.txt @@ -0,0 +1,36 @@ +SYNTHETIC CONTRACT DEMO — not general retrieval accuracy +Cue: n t; fields retain substring matching and weights 4/3/2/1. +A title with actual newline+tab has neither literal n nor literal t. +A title with literal backslash-n/backslash-t contains both letters. + +Record legacy score raw-string score +synthetic/y.controls 8 0 +synthetic/z.literal 8 8 +synthetic/a.split 7 7 +synthetic/b.split 7 7 +synthetic/m.metadata 5 5 +synthetic/x.empty 0 0 + +Legacy selected: synthetic/y.controls -> synthetic/z.literal -> synthetic/a.split -> synthetic/b.split -> synthetic/m.metadata +Opt-in selected: synthetic/z.literal -> synthetic/a.split -> synthetic/b.split -> synthetic/m.metadata + +FIXED REAL-CORPUS CLI EXAMPLE — interface/context demonstration only +Query: cpu.stat.local; task cue: cgroupv2; limit: 1; context budget: 65536 bytes. + +Legacy: exit=0; method=weighted-lexical-v1 + Selected: package:inferrsi.common.skills.cgroup_resource_controls + Revision: sha256:aebe3d1e6af70b9eb1cc1f04292e977e0bb507a3b614a5f36dfe24361c6f8b18 + Context bytes: 7348 + Complete core: data/skill_library/hardware/host/cgroup-resource-controls/SKILL.md + SHA-256: fd1f5a7e6a55714730044155f5b38c1d700383d4e16c3e91963bbfbe1c773af5 + Context omissions: [] + +Raw opt-in: exit=0; method=weighted-raw-string-v1 + Selected: package:inferrsi.common.skills.cgroup_resource_controls + Revision: sha256:aebe3d1e6af70b9eb1cc1f04292e977e0bb507a3b614a5f36dfe24361c6f8b18 + Context bytes: 7348 + Complete core: data/skill_library/hardware/host/cgroup-resource-controls/SKILL.md + SHA-256: fd1f5a7e6a55714730044155f5b38c1d700383d4e16c3e91963bbfbe1c773af5 + Context omissions: [] + +Default behavior is unchanged. No efficacy, latency or whole-Issue completion claim. diff --git a/evidence/indexer-255-raw-string-20261030/validator/README.md b/evidence/indexer-255-raw-string-20261030/validator/README.md new file mode 100644 index 000000000..79dca6fa4 --- /dev/null +++ b/evidence/indexer-255-raw-string-20261030/validator/README.md @@ -0,0 +1,22 @@ +# Inspect retained evidence without rescoring + +The archive retains the exact comparison runner (SHA-256 +`7abe02af59ae06a5f2e7956d5236573172810b84025223714ded593c1e6c8061`) and fixture +(SHA-256 `70086b95ba74c182cb195c32d71fed7aed747bec975079160f10bf8fff673e18`). +Original protocol/runner/candidate freezes and source/module proofs are included. +No timing or quality conclusion follows from diagnostic duration/output bytes. + +Extract into a separate scratch directory, then verify retained bytes: + +```bash +tar -xzf /evidence/indexer-255-raw-string-20261030/results/frozen-study.tar.gz +cd issue255-raw-string-results +sha256sum --check SHA256SUMS +python validator/show_demo.py results +``` + +The demo reads saved JSON only. Do not repeat the frozen comparison; the one-run +receipt and independent retained-output audit are the accepted historical result. +`independent_saved_audit.py` is explicitly a post-run supplemental audit, not a +replacement scorer. `plan/integrity.json` in the publication wrapper authenticates +all wrapper/archive files; it does not re-adjudicate the contract result. diff --git a/src/inferrsi/_skill_selection.py b/src/inferrsi/_skill_selection.py index d06f492c0..e58c5ed5c 100644 --- a/src/inferrsi/_skill_selection.py +++ b/src/inferrsi/_skill_selection.py @@ -4,6 +4,8 @@ import posixpath import re +RELEVANCE_METHODS = ("weighted-lexical-v1", "weighted-raw-string-v1") + class SkillIndexError(ValueError): pass @@ -21,6 +23,16 @@ def lexical_score(cue: str, fields: list[tuple[int, object]]) -> int: ) +def raw_string_lexical_score(cue: str, fields: list[tuple[int, object]]) -> int: + """Match original strings; retain legacy JSON matching for nonstring fields.""" + tokens = words(cue) + score = 0 + for weight, value in fields: + text = value if isinstance(value, str) else json.dumps(value, ensure_ascii=False) + score += weight * sum(token in text.casefold() for token in tokens) + return score + + def indexed_skill(record: dict, view: dict, content: dict[str, bytes]) -> tuple: """Validate the existing projection against this exact canonical bundle.""" registry_path = "data/skill_library/skill-index.json" @@ -82,8 +94,13 @@ def indexed_skill(record: dict, view: dict, content: dict[str, bytes]) -> tuple: return core, required, optional -def skill_score(cue: str, record: dict, core: bytes) -> int: - return lexical_score(cue, [ +def skill_score( + cue: str, record: dict, core: bytes, *, method: str = "weighted-lexical-v1" +) -> int: + if method not in RELEVANCE_METHODS: + raise ValueError("Unknown relevance method") + scorer = raw_string_lexical_score if method == "weighted-raw-string-v1" else lexical_score + return scorer(cue, [ (4, record["title"]), (3, record["summary"]), (2, [record.get("tags", []), record.get("applicability", {})]), diff --git a/src/inferrsi/arguments.py b/src/inferrsi/arguments.py index 17c614dfc..46b5e9336 100644 --- a/src/inferrsi/arguments.py +++ b/src/inferrsi/arguments.py @@ -8,6 +8,7 @@ from pathlib import Path from . import __version__ +from ._skill_selection import RELEVANCE_METHODS from .catalog import BACKENDS, KNOWLEDGE_REPOS, KNOWLEDGE_SCOPES, REPO_TYPES from .runspec import RunSpec @@ -191,6 +192,10 @@ def parser() -> argparse.ArgumentParser: collect.add_argument("--max-bytes", type=int) collect.add_argument("--material", action="store_true") collect.add_argument("--relevance", help="Opt-in lexical cue for packaged skill selection") + collect.add_argument( + "--relevance-method", choices=RELEVANCE_METHODS, default="weighted-lexical-v1", + help="Packaged skill ranking method; nondefault method requires --relevance", + ) collect.add_argument( "--context-bytes", type=int, help="Opt-in context view; total UTF-8 resource-body byte budget", diff --git a/src/inferrsi/cli.py b/src/inferrsi/cli.py index 9b8836587..349e3df34 100644 --- a/src/inferrsi/cli.py +++ b/src/inferrsi/cli.py @@ -193,6 +193,7 @@ def main(argv: list[str] | None = None) -> int: max_bytes=args.max_bytes, material=args.material, relevance=args.relevance, + relevance_method=args.relevance_method, context_bytes=args.context_bytes, ) if args.json: diff --git a/src/inferrsi/knowledge_operations.py b/src/inferrsi/knowledge_operations.py index 1c78418fe..d776964da 100644 --- a/src/inferrsi/knowledge_operations.py +++ b/src/inferrsi/knowledge_operations.py @@ -9,7 +9,7 @@ from importlib.resources import files from pathlib import Path, PurePosixPath -from ._skill_selection import context_view, indexed_skill, skill_score, words +from ._skill_selection import RELEVANCE_METHODS, context_view, indexed_skill, skill_score, words from .catalog import KNOWLEDGE_SCOPES, SCENARIO_ALIASES, knowledge_records AUTHORITIES = ("package", "checkout", "store") @@ -490,6 +490,7 @@ def collect( max_bytes: int | None = None, material: bool = False, relevance: str | None = None, + relevance_method: str = "weighted-lexical-v1", context_bytes: int | None = None, ) -> dict: if type(limit) is not int or limit < 1: @@ -504,6 +505,10 @@ def collect( raise ValueError("Unknown content kind") if relevance is not None and (not isinstance(relevance, str) or not words(relevance)): raise ValueError("relevance requires at least one word") + if relevance_method not in RELEVANCE_METHODS: + raise ValueError("Unknown relevance method") + if relevance is None and relevance_method != "weighted-lexical-v1": + raise ValueError("relevance_method requires relevance") if context_bytes is not None and (type(context_bytes) is not int or context_bytes < 1): raise ValueError("context_bytes must be a positive integer") if revisions is None: @@ -645,7 +650,9 @@ def select(reference: str, bundle: Bundle, view: dict) -> None: continue try: core, _, _ = indexed_skill(record, view, bundle.content) - score = skill_score(relevance, record, bundle.content[core]) + score = skill_score( + relevance, record, bundle.content[core], method=relevance_method + ) except (ValueError, KeyError, TypeError) as exc: omitted.append({ "id": reference, "reason": "index_unavailable", "detail": str(exc), @@ -673,7 +680,7 @@ def select(reference: str, bundle: Bundle, view: dict) -> None: } if relevance is not None: result["selection"] = { - "method": "weighted-lexical-v1", "cue": relevance, + "method": relevance_method, "cue": relevance, "scope": "generated-packaged-skills", "ranking_applied": ids is None, } result["limitations"][0] = ( diff --git a/tests/test_evidence_inventory.py b/tests/test_evidence_inventory.py index dc5fd64f6..22764106e 100644 --- a/tests/test_evidence_inventory.py +++ b/tests/test_evidence_inventory.py @@ -145,6 +145,6 @@ def test_checked_in_inventory(): repo = Path(__file__).resolve().parents[1] result = validate_inventory(repo) assert result["errors"] == [] - assert len(result["bundles"]) == 14 - assert result["checked_hashes"] == 311 + assert len(result["bundles"]) == 15 + assert result["checked_hashes"] == 321 assert result["unavailable"] # Historical raw artifacts are not in this checkout. diff --git a/tests/test_raw_string_selection.py b/tests/test_raw_string_selection.py new file mode 100644 index 000000000..32b4ec724 --- /dev/null +++ b/tests/test_raw_string_selection.py @@ -0,0 +1,213 @@ +"""Ordinary synthetic compatibility controls, not a retrieval-quality benchmark.""" + +import copy +import hashlib +import json + +import pytest + +from inferrsi._skill_selection import ( + indexed_skill, + lexical_score, + raw_string_lexical_score, + skill_score, +) +from inferrsi.cli import main +from inferrsi.knowledge_operations import KnowledgeReader, _hash, _json +from inferrsi.knowledge_store import KnowledgeStore + +RAW = "weighted-raw-string-v1" +LEGACY = "weighted-lexical-v1" + + +def synthetic_reader(entries): + """Exercise the real package adapter using a complete in-memory test library.""" + registry_path = "data/skill_library/skill-index.json" + registry = {"schema_version": 1, "skills": {}} + library, records = {}, {} + for name, fields in entries.items(): + path = f"methods/{name}" + prefix = f"data/skill_library/{path}/" + registry["skills"][name] = {"path": path, "backend": "common"} + library[prefix + "SKILL.md"] = fields.get("core", "").encode() + for relative, text in fields.get("resources", {}).items(): + library[prefix + relative] = text.encode() + identity = f"inferrsi.common.skills.{name.replace('-', '_')}" + records[identity] = { + "id": identity, "title": fields.get("title", ""), + "summary": fields.get("summary", ""), "tags": fields.get("tags", []), + "applicability": fields.get("applicability", {}), + "sources": [{"type": "internal", "issue": "test", "artifact": "synthetic"}], + "status": "candidate", "verification": "synthetic-only", "repo": "inferrsi", + "backend": "common", "scenario": "skills.discovery", "tier": "contribution", + "index_type": "packaged-skill", "skill_documents": {"original": prefix + "SKILL.md"}, + "skill_metadata": {"name": name, "references": fields.get("references", [])}, + } + library[registry_path] = _json(registry) + for record in records.values(): + prefix = record["skill_documents"]["original"].removesuffix("SKILL.md") + record["source_files"] = { + "src/inferrsi/" + path: _hash(data) + for path, data in library.items() if path.startswith(prefix) or path == registry_path + } + reader = KnowledgeReader() + reader._records = records + reader._library = library, "sha256:" + _hash(_json({ + path: _hash(data) for path, data in sorted(library.items()) + })) + return reader + + +def without_method(result): + value = copy.deepcopy(result) + value.get("selection", {}).pop("method", None) + return value + + +@pytest.mark.parametrize("value", [None, True, 13.5, ["\n\t", None], {"n": "\r"}]) +def test_nonstring_values_retain_exact_legacy_json_matching(value): + cue = "n t r null true 13 5" + assert raw_string_lexical_score(cue, [(2, value)]) == lexical_score(cue, [(2, value)]) + + +def test_raw_string_matching_does_not_change_the_legacy_helper(): + assert raw_string_lexical_score("n t", [(4, "\n\t")]) == 0 + assert lexical_score("n t", [(4, "\n\t")]) == 8 + assert raw_string_lexical_score("n t", [(4, r"\n\t")]) == 8 + assert raw_string_lexical_score("RUN run STRASSE", [(3, "running Straße")]) == 6 + + +def test_skill_adapter_keeps_field_weights_and_combined_nonstring_metadata(): + record = {"title": "n", "summary": "n", "tags": ["\n"], "applicability": {}} + assert skill_score("n N", record, b"n", method=RAW) == 10 + assert skill_score("n N", record, b"n") == 10 + with pytest.raises(ValueError, match="Unknown relevance method"): + skill_score("n", record, b"n", method="unknown") + + +def test_opt_in_changes_only_raw_string_owner_ranking_and_positive_admission(): + reader = synthetic_reader({ + "a-phantom": {"title": "\n", "summary": "\n", "core": "\n"}, + "b-hit": {"title": "n"}, "c-hit": {"title": "n"}, + }) + legacy = reader.collect(relevance="n") + assert legacy == reader.collect(relevance="n", relevance_method=LEGACY) + assert [r["title"] for r in legacy["items"]] == ["\n", "n", "n"] + raw = reader.collect(relevance="n", relevance_method=RAW) + expected = ["inferrsi.common.skills.b_hit", "inferrsi.common.skills.c_hit"] + assert [r["id"] for r in raw["items"]] == expected + assert raw["selection"] == { + "method": RAW, "cue": "n", "scope": "generated-packaged-skills", "ranking_applied": True, + } + assert reader.collect(relevance="n", relevance_method=RAW, limit=1)["items"] == raw["items"][:1] + assert reader.collect() == reader.collect(relevance_method=LEGACY) + + +@pytest.mark.parametrize("options", [ + {"query": "missing"}, {"query": "synthetic n"}, + {"kind": "skill"}, {"repo": "vllm"}, {"backend": "cpu"}, + {"tier": "peer"}, {"scenario": "diffusion.t2i"}, {"material": True}, +]) +def test_query_and_filters_preserve_ordinary_semantics(options): + reader = synthetic_reader({"item": {"title": "running", "core": "n"}}) + legacy = reader.collect(relevance="n", **options) + raw = reader.collect(relevance="n", relevance_method=RAW, **options) + assert without_method(raw) == without_method(legacy) + + +def test_explicit_ids_keep_membership_order_aliases_revisions_and_item_budgets(): + reader = synthetic_reader({"a": {"title": "\n"}, "b": {"title": "n"}}) + a, b = sorted(reader.records()) + options = {"ids": [b, a, "package:" + a], "relevance": "absent", "material": True} + legacy = reader.collect(**options) + raw = reader.collect(**options, relevance_method=RAW) + assert without_method(raw) == without_method(legacy) + assert [r["id"] for r in raw["items"]] == [b, a] + assert raw["selection"]["ranking_applied"] is False + for extra in ( + {"limit": 1}, {"max_bytes": len(_json(raw["items"][0]))}, + {"revisions": {a: reader.read(a).revision}}, {"revisions": {a: "sha256:" + "0" * 64}}, + ): + assert without_method(reader.collect(**options, **extra, relevance_method=RAW)) == ( + without_method(reader.collect(**options, **extra)) + ) + + +@pytest.mark.parametrize("explicit", [False, True]) +def test_context_required_hashes_atomic_budgets_and_optional_legacy_order(explicit): + reader = synthetic_reader({"owner": { + "title": "n", "core": "Core é", "references": ["refs.md"], + "resources": {"refs.md": "Required", "LICENSE": "Legal", "a.md": "\n", "b.md": "n"}, + }}) + identity, = reader.records() + bundle = reader.read(identity) + view = {**bundle.view(), "authority": "package"} + _, required, _ = indexed_skill(reader.records()[identity], view, bundle.content) + required_bytes = sum(len(bundle.content[path]) for path in required) + options = {"relevance": "n"} + if explicit: + options["ids"] = [identity] + for budget in (required_bytes - 1, required_bytes, required_bytes + 1, required_bytes + 2): + legacy = reader.collect(**options, context_bytes=budget) + raw = reader.collect(**options, context_bytes=budget, relevance_method=RAW) + assert without_method(raw) == without_method(legacy) + if budget < required_bytes: + assert not raw["contexts"] + assert raw["context_omissions"][0]["reason"] == "context_limit" + continue + context, = raw["contexts"] + assert context["revision"] == bundle.revision + assert context["content_bytes"] == budget + resources = context["resources"] + assert [r["path"] for r in resources[:len(required)]] == required + for resource in resources: + data = reader.resource(identity, resource["path"], revision=context["revision"]) + assert resource["content"].encode() == data + assert resource["sha256"] == hashlib.sha256(data).hexdigest() + if budget > required_bytes: + # JSON's newline contribution must still admit the optional a.md first. + assert resources[len(required)]["path"].endswith("/a.md") + + +@pytest.mark.parametrize("options", [ + {"relevance_method": RAW}, {"relevance_method": "unknown"}, + {"relevance": "n", "relevance_method": "unknown"}, + {"relevance": "n", "relevance_method": None}, + {"relevance": "n", "relevance_method": []}, + {"relevance": "!!!", "relevance_method": RAW}, +]) +def test_method_validation_happens_before_reader_access(monkeypatch, options): + reader = KnowledgeReader() + + def unexpected_access(): + raise AssertionError("invalid request accessed owners") + + monkeypatch.setattr(reader, "owners", unexpected_access) + with pytest.raises(ValueError): + reader.collect(**options) + + +def test_raw_method_keeps_fail_closed_index_validation(): + reader = synthetic_reader({"owner": {"title": "n"}}) + identity, = reader.records() + reader.records()[identity]["source_files"] = {} + result = reader.collect(relevance="n", relevance_method=RAW) + assert result["no_match"] + assert result["omissions"][0]["reason"] == "index_unavailable" + + +def test_cli_and_store_delegate_the_explicit_method(monkeypatch, capsys, tmp_path): + reader = synthetic_reader({"owner": {"title": "n"}}) + monkeypatch.setattr("inferrsi.cli._knowledge_reader", lambda args: reader) + assert main(["collect", "--relevance", "n", "--relevance-method", RAW, "--json"]) == 0 + assert json.loads(capsys.readouterr().out)["selection"]["method"] == RAW + with pytest.raises(SystemExit): + main(["collect", "--relevance", "n", "--relevance-method", "unknown"]) + with pytest.raises(SystemExit) as error: + main(["collect", "--relevance-method", RAW]) + assert error.value.code == 2 + store = KnowledgeStore(tmp_path, skill_root=None) + monkeypatch.setattr(store, "reader", lambda: reader) + assert store.collect(relevance="n", relevance_method=RAW) == reader.collect( + relevance="n", relevance_method=RAW + ) diff --git a/tools/evidence_inventory.json b/tools/evidence_inventory.json index c0fb3f82e..e3dcd9ef5 100644 --- a/tools/evidence_inventory.json +++ b/tools/evidence_inventory.json @@ -1,6 +1,10 @@ { "version": 1, "bundles": { + "indexer-255-raw-string-20261030": { + "kind": "campaign", + "manifests": [{"path": "plan/integrity.json", "schema": "bundle-sha256"}] + }, "indexer-255-token-v3-20261030": { "kind": "campaign", "manifests": [{"path": "plan/publication-integrity.json", "schema": "bundle-sha256"}]