From 11f2e33bb551d502043f50ef65ef19f82e5756cc Mon Sep 17 00:00:00 2001 From: Rohan Dsouza Date: Fri, 16 Oct 2026 20:59:34 +0530 Subject: [PATCH 1/7] Phase 10 task 1: the ablation of the fused ranking bench.retrieval takes --ablate, which also ranks each question with the fused query less one arm at a time: bm25-content, content-cue, and bm25. bench/ablation.py builds those queries. With every arm, its query is the query of ranking="fused". --- bench/README.md | 1 + bench/ablation.py | 114 +++++++++++++++++++++++++++++++++++ bench/retrieval.py | 27 ++++++--- bench/tests/test_ablation.py | 32 ++++++++++ 4 files changed, 167 insertions(+), 7 deletions(-) create mode 100644 bench/ablation.py create mode 100644 bench/tests/test_ablation.py diff --git a/bench/README.md b/bench/README.md index adec5cc..8e07313 100644 --- a/bench/README.md +++ b/bench/README.md @@ -37,6 +37,7 @@ The options are these: | `--location` | `local` or `cloud` | `local` | | `--url` | the address of the local engine | `http://localhost:3000` | | `--limit` | the first N scored questions | every scored question | +| `--ablate` | also rank with the fused query less one arm at a time: `bm25-content`, `content-cue`, and `bm25`; needs `--driver hotdata` | off | With `--location cloud`, the runner reads the connection from `HOTDATA_API_KEY`, `HOTDATA_WORKSPACE`, and `HOTDATA_API_URL`, so set them in `.env` first. It ignores `--url`. diff --git a/bench/ablation.py b/bench/ablation.py new file mode 100644 index 0000000..b9b7f57 --- /dev/null +++ b/bench/ablation.py @@ -0,0 +1,114 @@ +"""The fused ranking of `HotdataStore` with a chosen set of its arms, for the ablation. + +Each ablated ranking builds the query of `hotmemory.hotdata._fused_query` with only the +named arms: `bm25`, `content`, and `cue`. With every arm, the query is the same as that of +`ranking="fused"`. +""" + +from __future__ import annotations + +import builtins +from collections.abc import Sequence + +from hotmemory import Filter, Hit +from hotmemory.hotdata import ( + CUE_TABLE, + READ_COLUMNS, + RRF_CONSTANT, + HotdataStore, + _bm25_text, + _depth, + _record, + _ref, + _tables, + _text, + _union, + _vector, +) + +ARMS = ("bm25", "content", "cue") +ABLATIONS = { + "bm25-content": ("bm25", "content"), + "content-cue": ("content", "cue"), + "bm25": ("bm25",), +} + + +class ArmStore(HotdataStore): + """A `HotdataStore` whose search fuses only the arms in `arms`.""" + + def __init__(self, *args: object, arms: Sequence[str], **kwargs: object) -> None: + if not arms or set(arms) - set(ARMS): + raise ValueError(f"arms must be a non-empty subset of {ARMS}, got {arms!r}") + super().__init__(*args, ranking="fused", **kwargs) # type: ignore[arg-type] + self._arms = tuple(arms) + + def search( + self, + query: str | None, + prefixes: Sequence[Sequence[str]], + filter: Filter | None = None, + k: int = 10, + ) -> builtins.list[Hit]: + if query is None or k == 0: + return super().search(query, prefixes, filter, k) + where = self._listed(prefixes, filter) + tables = _tables(filter) + vector = _vector(self._embed([query])[0]) + sql = arm_query(tables, where, vector, _bm25_text(query), k, self._arms) + return [Hit(_record(row), float(row["distance"])) for row in self._sql(sql).to_records()] + + +def arm_query( + tables: Sequence[str], where: str, vector: str, text: str, k: int, arms: Sequence[str] +) -> str: + """Return the fused query of `hotmemory.hotdata._fused_query` with only `arms`.""" + depth = _depth(k) + rank = "ROW_NUMBER() OVER (ORDER BY {order}) AS rank" + parts = [] + rankings = [] + if "bm25" in arms: + for table in tables: + name = f"text_{table}" + parts.append( + f"{name} AS (SELECT id, {rank.format(order='score DESC, id ASC')} FROM " + f"bm25_search('{_ref(table)}', 'content', {_text(text)}, {depth}) WHERE {where})" + ) + rankings.append(name) + if "content" in arms: + nearest = " UNION ALL ".join( + f"SELECT scanned.id, cosine_distance(scanned.content_embedding, {vector}) AS distance " + f"FROM {_ref(table)} scanned WHERE {where}" + for table in tables + ) + parts.append( + f"content_rank AS (SELECT candidate.id, " + f"{rank.format(order='candidate.distance ASC, candidate.id ASC')} FROM (" + f"SELECT * FROM ({nearest}) ORDER BY distance ASC, id ASC LIMIT {depth}) candidate)" + ) + rankings.append("content_rank") + if "cue" in arms: + parts.append( + f"cue_rank AS (SELECT candidate.id, " + f"{rank.format(order='candidate.distance ASC, candidate.id ASC')} FROM (" + f"SELECT scanned.id, cosine_distance(scanned.cues_embedding, {vector}) AS distance " + f"FROM {_ref(CUE_TABLE)} scanned WHERE scanned.id IN ({_union(tables, 'id', where)}) " + f"ORDER BY distance ASC, id ASC LIMIT {depth}) candidate)" + ) + rankings.append("cue_rank") + ranked = " UNION ALL ".join( + f"SELECT id, rank FROM {name} WHERE rank <= {depth}" for name in rankings + ) + parts.append( + f"fused AS (SELECT id, sum(1.0 / ({RRF_CONSTANT} + rank)) AS score " + f"FROM ({ranked}) GROUP BY id)" + ) + columns = ", ".join(READ_COLUMNS) + rows = _union(tables, f"{columns}, content_embedding", where) + return ( + f"WITH {', '.join(parts)} " + f"SELECT {', '.join(f'r.{column}' for column in READ_COLUMNS)}, " + f"cosine_distance(r.content_embedding, {vector}) AS distance FROM fused " + f"JOIN ({rows}) r ON r.id = fused.id " + f"ORDER BY fused.score DESC, distance ASC, r.created_at DESC, r.id ASC LIMIT {k}" + ) diff --git a/bench/retrieval.py b/bench/retrieval.py index 3e8866b..c33f912 100644 --- a/bench/retrieval.py +++ b/bench/retrieval.py @@ -37,6 +37,7 @@ DROPPED = "Remote end closed connection without response" Mode = Literal["isolated", "shared"] RANKINGS = ("fused", "vector") +ABLATED = ("bm25-content", "content-cue", "bm25") @dataclass @@ -160,6 +161,7 @@ def hotdata_target( import pyarrow as pa from hotdata_framework import HotdataClient + from bench.ablation import ABLATIONS, ArmStore from hotmemory.hotdata import EPISODE_TABLE, HotdataStore started = time.perf_counter() @@ -176,14 +178,16 @@ def hotdata_target( dimensions=embed.DIMENSIONS, client=client, ) + options: dict[str, Any] = { + "embedder": embedder, + "model": embed.MODEL, + "dimensions": embed.DIMENSIONS, + } stores: dict[str, Store] = { - name: HotdataStore( - client, - provisioned.database, - embedder=embedder, - model=embed.MODEL, - dimensions=embed.DIMENSIONS, - ranking=name, # type: ignore[arg-type] + name: ( + ArmStore(client, provisioned.database, arms=ABLATIONS[name], **options) + if name in ABLATIONS + else HotdataStore(client, provisioned.database, ranking=name, **options) # type: ignore[arg-type] ) for name in rankings } @@ -215,12 +219,21 @@ def main(argv: Sequence[str] | None = None) -> Path: parser.add_argument("--driver", choices=("hotdata", "memory"), default="hotdata") parser.add_argument("--mode", choices=("isolated", "shared"), default="isolated") parser.add_argument("--ranking", choices=(*RANKINGS, "both"), default="both") + parser.add_argument( + "--ablate", + action="store_true", + help="also rank with the fused query less one arm at a time", + ) parser.add_argument("--location", choices=("local", "cloud"), default="local") parser.add_argument("--url", default="http://localhost:3000") parser.add_argument("--limit", type=int, default=None, help="the first N scored questions") args = parser.parse_args(argv) rankings = RANKINGS if args.ranking == "both" else (args.ranking,) + if args.ablate: + if args.driver == "memory": + parser.error("--ablate needs --driver hotdata") + rankings = (*rankings, *ABLATED) questions = [question for question in load(DATA_DIR / DATA_FILE) if question.is_scored] questions = questions[: args.limit] commit = _commit() diff --git a/bench/tests/test_ablation.py b/bench/tests/test_ablation.py new file mode 100644 index 0000000..ad6bc45 --- /dev/null +++ b/bench/tests/test_ablation.py @@ -0,0 +1,32 @@ +import pytest + +from bench.ablation import ABLATIONS, ARMS, ArmStore, arm_query +from bench.retrieval import ABLATED +from hotmemory.hotdata import EPISODE_TABLE, MEMORY_TABLE, _fused_query + +TABLES = (MEMORY_TABLE, EPISODE_TABLE) +VECTOR = "ARRAY[0.1, 0.2]" + + +def test_every_arm_gives_the_fused_query() -> None: + assert arm_query(TABLES, "TRUE", VECTOR, '"cat"', 5, ARMS) == _fused_query( + TABLES, "TRUE", VECTOR, '"cat"', 5 + ) + + +@pytest.mark.parametrize("name", list(ABLATIONS)) +def test_an_ablation_leaves_out_its_arms(name: str) -> None: + sql = arm_query(TABLES, "TRUE", VECTOR, '"cat"', 5, ABLATIONS[name]) + assert ("bm25_search" in sql) == ("bm25" in ABLATIONS[name]) + assert ("content_rank AS" in sql) == ("content" in ABLATIONS[name]) + assert ("cue_rank AS" in sql) == ("cue" in ABLATIONS[name]) + + +def test_the_runner_names_each_ablation() -> None: + assert tuple(ABLATIONS) == ABLATED + + +@pytest.mark.parametrize("arms", [(), ("bm25", "title")]) +def test_arms_must_be_a_subset(arms: tuple[str, ...]) -> None: + with pytest.raises(ValueError, match="arms must be"): + ArmStore(None, "db", embedder=lambda texts: [], model="m", dimensions=2, arms=arms) From d8d46c9373b81b9563cb75959621520627c8aabc Mon Sep 17 00:00:00 2001 From: Rohan Dsouza Date: Fri, 16 Oct 2026 22:15:18 +0530 Subject: [PATCH 2/7] Phase 10 tasks 2 to 4: the vector ranking by default, and capture with chunks HotdataStore ranks by content vector alone by default, as MemoryStore does, and ranking="fused" stays as an option. Memory.capture takes chunk_chars with episode: it cuts the text at blank lines into chunks, stores chunk n under -, and each fact cites every chunk. The extractor still reads the whole text. The skill capture script takes --chunk-chars, and the bench rankings default to vector. docs/benchmarks.md reports the ablation of task 1 and the memory arm with each ranking. --- CHANGELOG.md | 6 ++ bench/README.md | 4 +- bench/e2e.py | 8 +-- bench/tests/test_e2e.py | 4 +- docs/benchmarks.md | 67 +++++++++++++++++++- docs/contracts.md | 49 ++++++++++++--- docs/guarantees.md | 4 ++ docs/internal/architecture.md | 26 +++++--- skills/hotmemory/SKILL.md | 13 ++++ skills/hotmemory/scripts/capture.py | 12 +++- src/hotmemory/_chunks.py | 14 ++++- src/hotmemory/contract.py | 70 ++++++++++++++++++++- src/hotmemory/hotdata.py | 16 ++--- tests/test_chunks.py | 37 ++++++++++- tests/test_contract.py | 97 +++++++++++++++++++++++++++++ tests/test_oracle.py | 80 +++++++++++++++++++++++- tests/test_skill.py | 6 +- 17 files changed, 472 insertions(+), 41 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ab8d214..6af657b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -66,3 +66,9 @@ each one, and its failure message points to this file. - The block of `recall` and `profile` shows a source that is the id of a record in the namespace of its line as `@`. Before, it showed the full id. The stored sources do not change. +- `HotdataStore` ranks by content vector alone by default, as `MemoryStore` does. Before, + the default was `ranking="fused"`, which stays as an option. On LongMemEval-S, the BM25 + arm of the fused ranking lowered the retrieval score. +- `Memory.capture` takes `chunk_chars`, with `episode`. It cuts the text at blank lines + into chunks and stores chunk n as an episode under `-`. The extractor still + reads the whole text, and each fact names every chunk in `sources`. diff --git a/bench/README.md b/bench/README.md index 8e07313..fa091f7 100644 --- a/bench/README.md +++ b/bench/README.md @@ -82,8 +82,8 @@ The options are these: |---|---|---| | `--sample` | `a` or `b` | `a` | | `--arms` | a comma-separated list of `full`, `retrieval`, and `memory` | every arm | -| `--retrieval-ranking` | `fused` or `vector`, the ranking of the retrieval arm | `fused` | -| `--memory-ranking` | `fused` or `vector`, the ranking of `capture` and `recall` | `fused` | +| `--retrieval-ranking` | `vector` or `fused`, the ranking of the retrieval arm | `vector` | +| `--memory-ranking` | `vector` or `fused`, the ranking of `capture` and `recall` | `vector` | | `--recall-k` | the `k` of the memory recall | `30` | | `--recall-budget` | the `budget` of the memory recall, in characters | `8000` | | `--extractor` | a variant name in `VARIANTS` of `bench/prompts.py` | `baseline` | diff --git a/bench/e2e.py b/bench/e2e.py index a488ec0..04bc05b 100644 --- a/bench/e2e.py +++ b/bench/e2e.py @@ -90,8 +90,8 @@ class Config: `recall`, and needs `session_episodes` unless it is `none`. """ - retrieval_ranking: str = "fused" - memory_ranking: str = "fused" + retrieval_ranking: str = "vector" + memory_ranking: str = "vector" k: int = RECALL_K budget: int = RECALL_BUDGET extractor: str = "baseline" @@ -789,8 +789,8 @@ def main(argv: Sequence[str] | None = None) -> Path | None: action="store_true", help="run the 18 questions of the smoke sample on MemoryStore, in each worker", ) - parser.add_argument("--retrieval-ranking", choices=RANKINGS, default="fused") - parser.add_argument("--memory-ranking", choices=RANKINGS, default="fused") + parser.add_argument("--retrieval-ranking", choices=RANKINGS, default="vector") + parser.add_argument("--memory-ranking", choices=RANKINGS, default="vector") parser.add_argument("--recall-k", type=int, default=RECALL_K, help="k of the memory recall") parser.add_argument( "--recall-budget", type=int, default=RECALL_BUDGET, help="characters of the memory block" diff --git a/bench/tests/test_e2e.py b/bench/tests/test_e2e.py index e5eeeef..3d3872b 100644 --- a/bench/tests/test_e2e.py +++ b/bench/tests/test_e2e.py @@ -397,14 +397,14 @@ def test_kept_name_holds_what_decides_the_memory() -> None: config = Config(extractor="detail", session_episodes=True, episodes="search", budget=64000) name = kept_name("a", config, "b5443ac") assert name.startswith("hotmemory-kept-a-detail") - assert name.endswith("-episodes-fused-b5443ac") + assert name.endswith("-episodes-vector-b5443ac") after_capture = replace(config, episodes="cited", budget=32000, k=10) assert kept_name("a", after_capture, "b5443ac") == name for other in ( kept_name("b", config, "b5443ac"), kept_name("a", config, "1234567"), kept_name("a", replace(config, extractor="baseline"), "b5443ac"), - kept_name("a", replace(config, memory_ranking="vector"), "b5443ac"), + kept_name("a", replace(config, memory_ranking="fused"), "b5443ac"), kept_name("a", Config(extractor="detail"), "b5443ac"), kept_name("a", Config(extractor="detail", fact_sources=False), "b5443ac"), ): diff --git a/docs/benchmarks.md b/docs/benchmarks.md index 037d061..e6e7674 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -53,7 +53,8 @@ with one question at a time in the tables. File: `ranking="fused"` fuses BM25, the content vector, and the cue vector. `ranking="vector"` ranks by the content vector alone. On this data, the vector ranking -scores higher than the fused ranking at k of 3 and more. The reason was not measured. +scores higher than the fused ranking at k of 3 and more. The section on the ablation below +gives the reason. The same run, by question type: @@ -94,6 +95,49 @@ two distances was 1.0e-6. `MemoryStore` ran in process with a fixed clock. File: `retrieval-memory-in-process-isolated-vector-first20.json`, compared with `bench/compare.py`. +### The ablation of the fused ranking + +Phase 10 ranked each question with the fused query less one arm at a time, with +`--ablate`. The episodes have no cues, so the cue arm is empty. The fused query is thus +BM25 and the content vector. Observed on 2026-10-16, on the local stack, with one question at +a time in the tables, at commit `6a0fb70` with the changes of `--ablate`. The embedder took +every vector from the cache. File: +`retrieval-hotdata-local-isolated-fused-vector-bm25-content-content-cue-bm25.json`. + +| Ranking | Arms | `recall_any@5` | `recall_all@5` | `recall_all@10` | `ndcg_any@10` | Search p50 | +|---|---|---|---|---|---|---| +| `fused` | BM25, content, cue | 0.9690 | 0.9045 | 0.9570 | 0.9311 | 160 ms | +| `bm25-content` | BM25, content | 0.9690 | 0.9045 | 0.9570 | 0.9311 | 111 ms | +| `content-cue` | content, cue | 0.9833 | 0.9475 | 0.9928 | 0.9479 | 126 ms | +| `bm25` | BM25 | 0.9475 | 0.8234 | 0.8926 | 0.8766 | 86 ms | +| `vector` | the content vector, by distance | 0.9833 | 0.9475 | 0.9928 | 0.9479 | 49 ms | + +`bm25-content` ranked the same sessions as `fused` on 419 of 419 questions. +`content-cue` ranked the same top 10 as `vector` on 419 of 419 questions. Thus the BM25 +arm makes the whole gap between the two rankings. + +By question type, at `recall_all@10`: + +| Question type | Questions | `fused` | `vector` | `bm25` | +|---|---|---|---|---| +| single-session-user | 64 | 1.0000 | 1.0000 | 1.0000 | +| single-session-assistant | 5 | 1.0000 | 1.0000 | 1.0000 | +| single-session-preference | 30 | 0.9333 | 1.0000 | 0.7667 | +| multi-session | 121 | 0.9256 | 0.9835 | 0.8182 | +| knowledge-update | 72 | 1.0000 | 1.0000 | 0.9861 | +| temporal-reasoning | 127 | 0.9449 | 0.9921 | 0.8819 | + +On 16 questions, `fused` lost an evidence session from the top 10 that `vector` had: 8 +multi-session, 6 temporal-reasoning, and 2 single-session-preference. BM25 ranked each lost +session from 23 to 40. Fusion with equal weights thus pushed it below sessions that BM25 +ranked high. The run holds about 50 sessions for each question, and BM25 fetches 500 rows, +so the filter after BM25 did not cause the loss. + +BM25 favors long sessions a little. In each haystack, a session has a length percentile +from 0, the shortest, to 1, the longest. The sessions that are not evidence and that +`fused` adds to the top 10 have a mean percentile of 0.645. Those in the top 10 of +`vector` have 0.583, and the evidence sessions have 0.672. + ## Latency Each search time below includes the request to the engine and excludes the embedding of @@ -464,6 +508,27 @@ The time in the table is `duration_s` of the result file. The whole command took seconds the first time, and 14.9 seconds on a later try. The cause of the first wait was not found. +### The ranking end to end + +Phase 10 ran the memory arm on sample A with each ranking of `capture` and `recall`, with +`search` at 64,000 characters, `--extractor detail --session-episodes`, and 4 workers. +Observed on 2026-10-16, with `src/hotmemory` at `b5443ac`. The fused run reused the kept +memory of phase 9. The vector run kept its own memory. Its first `--replay` stopped at a +cache miss after 3 questions, and a `--run --resume` ran the other 51. + +| Ranking | Correct of 54 | Reader tokens, median | Episodes in the block, mean | Facts written | Cost | File | +|---|---|---|---|---|---|---| +| `fused` | 45 | 12,849 | 4.06 | 13,666 | $0 | `e2e-sample-a-p10-reuse-fused-search-64k.json` | +| `vector` | 46 | 13,406 | 4.48 | 14,322 | $2.17 | `e2e-sample-a-p10-keep-vector-search-64k.json` | + +The reader tokens, episodes, and facts of the fused row are those of its `--keep` run, +`e2e-sample-a-p9-keep-search-64k.json`. The two rankings differ on 3 questions. `vector` +answered `66f24dbb`, single-session-user, and `21d02d0d`, multi-session, and missed +`0edc2aef`, single-session-preference. The ranking also changes `capture`, because the +extractor reads the facts that the search of the store returns. Thus the two runs stored +different facts. The vector run took 1,770.6 seconds and 1,722 model calls. In one session, +the extractor reply reached the token limit, and that session gave no facts. + ## Known limits - The end-to-end numbers come from samples of 54 questions, so one question moves an arm diff --git a/docs/contracts.md b/docs/contracts.md index c162a60..456c1e4 100644 --- a/docs/contracts.md +++ b/docs/contracts.md @@ -257,8 +257,16 @@ the last attempt, the driver raises the error. A `search` with query text runs one SQL query over both record tables and `cue_v1`. If the filter sets `kind`, the query reads only the record tables of the kinds that it names. -A filter without `episode` reads only `memory_v1`. With -`ranking="fused"`, the default, the query has three parts. +A filter without `episode` reads only `memory_v1`. + +With `ranking="vector"`, the default, the query ranks by the content distance alone, after +the filters, by a scan. That order is the order of `MemoryStore`, so the two drivers return +the same records in the same order. The default changed from `"fused"` in phase 10. On +LongMemEval-S, the BM25 arm of the fused ranking lowered the retrieval score, and the +vector ranking scored as high end to end. [measured 2026-10-16, +[docs/benchmarks.md](benchmarks.md)] + +With `ranking="fused"`, the query has three parts. 1. Three rankings: BM25 over `content` in each record table, the cosine distance of `content_embedding`, and the cosine distance of `cues_embedding`. BM25 reads only the @@ -276,8 +284,7 @@ A filter without `episode` reads only `memory_v1`. With newer row. The fused order is not the order of the distance, so a later hit can have a smaller -distance than an earlier one. With `ranking="vector"`, the query ranks by the content -distance alone, after the filters, by a scan. That order is the order of `MemoryStore`. +distance than an earlier one. Reads are fast because the table is small. A memory table holds thousands of rows, and a filtered scan of that is fast without an index. Measurement M5 found that the vector @@ -301,7 +308,7 @@ The memory contract is what an agent calls. It is a class named `Memory`, built | `supersede` | scope, key of the record to close, new fact, optional valid_from, actor | Closes the named record and writes the new fact as the next revision under its key, in one `put` with `close_previous`. The new `valid_from` is the argument, else that of the fact, else now. The `valid_until` of the old record becomes the `valid_from` of the new record, and the `expired_at` of the old record becomes now. If the key has no current revision, or the `valid_from` of the old record is later than that of the new record, the call refuses. The library decides nothing by itself. | | `forget` | scopes, and ids or a horizon | Deletes every revision of the key of each named id, or of each key whose current revision has a `forget_after` before the horizon. It finds the keys for a horizon with `list`, up to 10,000 records under each scope. A record already past its `forget_after` is hidden from `list`, so `sweep` deletes it. It checks every id before the first delete. An id that is not a valid record id, or is outside the scopes, makes the call refuse and delete nothing. It returns the deleted ids. | | `profile` | subject, scopes, budget in characters | Returns the current records of kind `fact`, `profile`, and `procedure` for the subject, grouped by `kind`. It returns them as a list and as one rendered block inside the budget. The block ends with the namespaces and record counts that `recall` can reach, so an agent knows what it can search for. This is the block that an agent always loads. | -| `capture` | text, scope, extractor, actor, observed_at, optional episode | Calls the extractor of the caller with the text, `observed_at`, and the current records that `recall` returns for the text under the scope. Then it calls `remember` on the result, and returns the ids. A fact with no `observed_at` gets the one passed to `capture`. With `episode`, a key, it also stores the text as a record of kind `episode` under that key, and adds the id of the episode to the `sources` of each fact. `remember` writes the facts in one flush, and then one `put` writes the episode, as in `load`. If the episode is not written, a retry calls the extractor again. If the episode is current with the same content, `capture` calls no extractor, writes nothing, and returns no id. With `episode`, a blank text, a key that is not valid, or a key that holds another kind makes the call refuse before the extractor runs. The extractor is a plain callable. The library ships no model and names no model. | +| `capture` | text, scope, extractor, actor, observed_at, optional episode, optional chunk_chars | Calls the extractor of the caller with the text, `observed_at`, and the current records that `recall` returns for the text under the scope. Then it calls `remember` on the result, and returns the ids. A fact with no `observed_at` gets the one passed to `capture`. With `episode`, a key, it also stores the text as a record of kind `episode` under that key. It adds the id of the episode to the `sources` of each fact. `remember` writes the facts in one flush, and then one `put` writes the episode, as in `load`. If the episode is not written, a retry calls the extractor again. If the episode is current with the same content, `capture` calls no extractor, writes nothing, and returns no id. With `episode`, three cases make the call refuse before the extractor runs: a blank text, a key that is not valid, and a key that holds another kind. With `episode` and `chunk_chars`, it stores the text as chunks, as the section on chunks below gives. `chunk_chars` without `episode` makes the call refuse. The extractor is a plain callable. The library ships no model and names no model. | | `load` | document, text, scope, extractor, actor, observed_at, chunk_chars | Cuts a Markdown document into chunks. It stores chunk n as a record of kind `episode` under the key `-`, with n from 1 in four digits. For each chunk, it calls the extractor as `capture` does, on the chunk with its heading prefix. Each fact gets the id of the episode of its chunk added to its `sources`. The facts of a chunk are written before its episode. A chunk whose episode is current with the same content is skipped, with no call to the extractor. Thus a second load of the same text writes nothing new. It returns the ids of the episodes and the ids of the facts that the extractor returned. The document must be a valid key. | The key of a fact has two parts joined by `-`. The first part is the subject, with each @@ -327,8 +334,8 @@ the block in this line format, and its default is 2,000. A consumer that renders records in its own format passes its own `budget` and `k`. Then the default does not cut the list first. -With `episodes`, the fact lines come first, and they fit in the budget as above. Then, if -an episode fits in what is left, a line `episodes:` follows, and then each episode in this +With `episodes`, the fact lines come first, and they fit in the budget as above. If an +episode fits in what is left, a line `episodes:` follows, and then each episode in this form: ```text @@ -395,6 +402,34 @@ extractor returns no fact still gets its episode. If the text of a chunk changes second load writes the next revision of its episode and calls the extractor again. If a document gets fewer chunks, the episodes of its old last chunks stay current. +`capture` with `episode` and `chunk_chars` stores the text as chunks, for a thread or a +session that is not Markdown. A blank line separates two turns of such a text. + +1. It cuts the text into blocks at blank lines outside fences. A heading line does not + start a section, and a chunk gets no prefix. Thus a heading inside a turn stays text. +2. It cuts a block that does not fit at line ends, and a line longer than `chunk_chars` + at that size, as `load` does. +3. It packs the blocks in order into chunks of at most `chunk_chars` characters, joined by + a blank line. +4. It stores chunk n as a record of kind `episode` under the key `-`, with n + from 1 in four digits. Thus a text that fits in one chunk is stored under + `-0001`. + +The extractor reads the whole text once, as it does without `chunk_chars`. Its call and +its facts do not change. The extractor does not say which chunk a fact came from. Thus +each fact gets the id of every chunk added to its `sources`, in chunk order. The facts +are written first, and then one flush writes the chunks. If every chunk is current with +the same content, `capture` calls no extractor, writes nothing, and returns no id. If a +chunk changed, `capture` calls the extractor again, and it writes the next revision of +each changed chunk only. If a key `-` is not a valid key or holds another +kind, the call refuses before the extractor runs. If the text gets fewer chunks, the +episodes of its old last chunks stay current. + +`episodes="search"` ranks each chunk on its own, so the block holds the chunks that match +the query and not the whole text. `episodes="cited"` does not change. A fact cites every +chunk of its text, so `"cited"` gives the chunks of each cited text in chunk order, to the +budget. + An episode is the evidence that a fact names in `sources`. `profile` leaves it out, and `recall` gives it only with `episodes`. A consumer also reads an episode with `get`, or joins the two record tables in SQL. A diff --git a/docs/guarantees.md b/docs/guarantees.md index ea71620..f085c66 100644 --- a/docs/guarantees.md +++ b/docs/guarantees.md @@ -59,6 +59,10 @@ A row that no test proves yet names the phase that will prove it. | Do `recall` and `profile` return episodes? | `profile` never does, and its counts leave episodes out. `recall` returns episodes only with `episodes`. Without it, `recall` reads the current records of kind `fact`, `profile`, and `procedure` only. | answered | `test_recall_and_profile_leave_episodes_out`, `test_recall_block_matches_its_pinned_string`, `test_profile_block_matches_its_pinned_string` | | What does `capture` store with an episode key? | The text, as an episode under that key, with the `actor` and `observed_at` of the call. Each fact names the id of the episode in `sources`. `remember` writes the facts in one flush, and one `put` then writes the episode, so each table gets one flush. If the extractor raises, nothing is written. | answered | `test_capture_with_an_episode_stores_it_and_links_its_facts`, `test_capture_writes_one_flush_per_table`, `test_capture_with_a_failing_extractor_writes_no_episode`, `test_capture_refuses_a_bad_episode`, `test_capture_refuses_an_episode_key_that_holds_a_fact` | | Does a retried `capture` with an episode key write anything? | No. If the episode is current with the same content, the extractor is not called and nothing is written, so this holds for an extractor that calls a model. | answered | `test_a_retried_capture_with_an_episode_writes_nothing_new` | +| What does `capture` store with an episode key and `chunk_chars`? | The text, cut at blank lines outside fences into chunks of at most `chunk_chars` characters, with no heading prefix. Chunk n is an episode under the key `-`. The extractor reads the whole text once. Each fact names the id of every chunk in `sources`, in chunk order. The facts take one flush, and the changed chunks take one more. `chunk_chars` without an episode key, or a chunk key that holds another kind, makes the call refuse before the extractor runs. | answered | `test_capture_with_chunks_stores_each_chunk_and_links_every_chunk`, `test_capture_with_chunks_writes_the_chunks_in_one_flush`, `test_capture_refuses_bad_chunks`, `test_capture_refuses_a_chunk_key_that_holds_a_fact`, `test_chunk_text_keeps_a_heading_as_text_with_no_prefix` | +| Does a retried `capture` with chunks write anything? | No. If every chunk is current with the same content, the extractor is not called and nothing is written. If one chunk changed, the extractor runs again, and only that chunk gets its next revision. | answered | `test_a_retried_capture_with_chunks_writes_nothing_new`, `test_capture_of_an_edited_chunk_writes_its_next_revision_only` | +| Do the two drivers recall the same block with chunks? | Yes. After the same `capture` calls with `chunk_chars`, `MemoryStore` and `HotdataStore` with its default ranking return the same records and the same block, with `episodes="search"` and with `episodes="cited"`. A search over episodes ranks each chunk on its own. | answered [measured, local] 2026-10-16 | `test_both_drivers_recall_the_same_block_with_chunks`, `test_recall_search_returns_the_chunks_that_match` | +| Which ranking does the search inside `capture` use? | The ranking of the store, as `recall` does, because `capture` calls `recall`. The default of `HotdataStore` is `ranking="vector"`, the order of `MemoryStore`. | answered [measured, local] 2026-10-16 | `test_capture_searches_with_the_ranking_of_recall`, `test_the_default_ranking_is_the_vector_ranking` | | Which episodes does `recall` add with `episodes`? | With `"cited"`, the episodes that the facts of the block name in `sources`, in order, once each, and only inside the allowed scopes. With `"search"`, the top k episodes for the query. Neither value returns an episode past its `forget_after`. `"cited"` keeps a superseded revision, because the fact cites that revision. `as_of` filters them as it filters facts. They follow the facts, whole, and the block stops before the first episode that does not fit. An episode never takes the place of a fact. | answered | `test_recall_cited_adds_the_episodes_of_the_facts`, `test_recall_cited_names_each_episode_once`, `test_recall_cited_skips_an_episode_outside_the_scopes`, `test_recall_search_adds_the_top_episodes`, `test_recall_episodes_never_take_the_place_of_a_fact`, `test_recall_stops_before_the_first_episode_that_does_not_fit`, `test_recall_as_of_filters_the_episodes`, `test_recall_refuses_an_unknown_episodes_value`, `test_recall_cited_skips_an_episode_past_its_forget_after`, `test_recall_cited_keeps_a_superseded_revision` | | How does the block show a source? | In full, except the id of a record in the namespace of the line, which shows as `@`. The stored source keeps the full id. | answered | `test_recall_shows_a_source_in_its_namespace_as_key_and_revision` | | Can a `put` move a key between `episode` and another kind? | No. A key holds episodes only, or holds no episode at all. A `put` that crosses that line raises an error and writes nothing. A key can change between `fact`, `profile`, and `procedure`. | answered | `test_key_cannot_cross_the_episode_line` | diff --git a/docs/internal/architecture.md b/docs/internal/architecture.md index 848a01a..b5e672d 100644 --- a/docs/internal/architecture.md +++ b/docs/internal/architecture.md @@ -335,7 +335,7 @@ sequenceDiagram end C->>M: capture(text, scope, extractor, actor, observed_at) M->>H: search(text, [scope], kinds fact, profile, procedure, k=10) - H->>API: one fused SELECT on memory_v1 and cue_v1 + H->>API: one vector SELECT on memory_v1 API-->>H: ranked rows H-->>M: hits Note over M: recall keeps the lines that fit in 2,000 characters @@ -365,16 +365,24 @@ flushes, one for each table, in that order. A flush of `HotdataStore` loads its the same time, so with one writer for both, the episode can land without the facts. A retry would then skip the extractor. +With `episode` and `chunk_chars`, `capture` cuts the text at blank lines into chunks, and +reads the current revision of each key `-` with `get`. If every chunk holds the +same content, the call ends there. Else the extractor reads the whole text once, each fact +gets the id of every chunk in its `sources`, and one more flush writes the changed chunks +to `episode_v1`. The search of the extractor runs as above, so it uses the ranking of the +store, as `recall` does. + ## Recall and search `recall` calls `search` with the filter `kind` in (`fact`, `profile`, `procedure`). That -filter names no episode, so the search reads only `memory_v1` and `cue_v1`. The search +filter names no episode, so the search reads only `memory_v1`, and `cue_v1` with +`ranking="fused"`. The search embeds the query once and sends one SQL query. ```mermaid flowchart TD q["recall(query, scopes, as_of, budget, k)"] --> embed["Embed the query
(one embedder call)"] - embed --> sql["Send one SQL query
(fused or vector)"] + embed --> sql["Send one SQL query
(vector or fused)"] sql --> top["The top k current records"] top --> asof{"as_of given?"} asof -- "yes" --> keep["Keep the records valid at as_of"] @@ -392,9 +400,10 @@ flowchart TD With `episodes="cited"`, `recall` sends one `get` for each distinct source that is an episode id in the allowed scopes. With `episodes="search"`, it sends a second search with -the filter `kind` = `episode`, so that search reads `episode_v1`. The episodes that -`capture` and `load` write have no cues, so the fused ranking of that search is BM25 and -the content vector. +the filter `kind` = `episode`, so that search reads `episode_v1`. With chunks, each chunk +is an episode, so that search ranks each chunk on its own. The episodes that `capture` and +`load` write have no cues, so the fused ranking of that search is BM25 and the content +vector. The `as_of` filter runs in the process, after the search. Thus `recall` with `as_of` can return fewer than k records. The search keeps k records before the filter, so a record that @@ -414,7 +423,8 @@ Each ranking applies the same SQL condition, built from these parts: ### The fused ranking -`ranking="fused"` is the default. It runs three rankings and fuses them by reciprocal rank +`ranking="vector"` is the default, and it ranks by the content vector alone. With +`ranking="fused"`, the search runs three rankings and fuses them by reciprocal rank fusion. Reciprocal rank fusion adds `1 / (60 + rank)` from each ranking that holds a row. Each ranking keeps its top rows to a depth of `max(100, 10 * k)`. @@ -592,7 +602,7 @@ sequenceDiagram else a new or changed chunk Note over M: The episode id is known before the write:
scope/key@next revision M->>H: search for the chunk (as capture does) - H->>API: fused SELECT on memory_v1 and cue_v1 + H->>API: vector SELECT on memory_v1 M->>X: extractor(chunk, observed_at, current records) X-->>M: facts Note over M: Add the episode id to the sources of each fact diff --git a/skills/hotmemory/SKILL.md b/skills/hotmemory/SKILL.md index 16c127f..c67a3e8 100644 --- a/skills/hotmemory/SKILL.md +++ b/skills/hotmemory/SKILL.md @@ -104,6 +104,19 @@ python scripts/recall.py --memory-file memory.json --scope team --query "Why did --budget 1000 --episodes cited ``` +Add `--chunk-chars` with `--episode` to store a long text as chunks. The script cuts the +text at blank lines into chunks of at most that many characters, and stores chunk n under +the key `-`. Each fact then names every chunk in its sources. A recall with +`--episodes search` gives the chunks that match the query, and not the whole text. + +```sh +python scripts/capture.py --memory-file memory.json --scope team/alerts --actor sentence-splitter \ + --extractor extractors:one_fact_per_sentence --episode chat-0002 --chunk-chars 40 \ + --text "$(printf 'The disk fills at night.\n\nThe logs rotate at noon.')" +python scripts/recall.py --memory-file memory.json --scope team --query "When do the logs rotate?" \ + --budget 1000 --episodes search +``` + ## Load a document Store a Markdown document as episodes, one for each chunk, and run an extractor on each diff --git a/skills/hotmemory/scripts/capture.py b/skills/hotmemory/scripts/capture.py index 8066990..6f49ed0 100644 --- a/skills/hotmemory/scripts/capture.py +++ b/skills/hotmemory/scripts/capture.py @@ -1,6 +1,7 @@ """Run an extractor over a text, remember the facts it returns, and print their ids. With --episode, the text is also stored as an episode, and each fact names it in its sources. +With --chunk-chars as well, the text is stored as chunks of at most that many characters. """ import importlib @@ -16,11 +17,20 @@ args_parser.add_argument("--actor", default="") args_parser.add_argument("--observed-at", type=parse_time) args_parser.add_argument("--episode", help="a key: also store the text as an episode under it") +args_parser.add_argument( + "--chunk-chars", type=int, help="with --episode: store the text as chunks of this size" +) args = args_parser.parse_args() module, _, name = args.extractor.partition(":") extractor = getattr(importlib.import_module(module), name) with memory(args) as agent_memory: ids = agent_memory.capture( - args.text, args.scope, extractor, args.actor, args.observed_at, args.episode + args.text, + args.scope, + extractor, + args.actor, + args.observed_at, + args.episode, + args.chunk_chars, ) print(*ids, sep="\n") diff --git a/src/hotmemory/_chunks.py b/src/hotmemory/_chunks.py index 525773b..1625970 100644 --- a/src/hotmemory/_chunks.py +++ b/src/hotmemory/_chunks.py @@ -1,4 +1,4 @@ -"""The chunking of a Markdown document for `Memory.load`.""" +"""The chunking of a Markdown document for `Memory.load`, and of a text for `Memory.capture`.""" from __future__ import annotations @@ -48,6 +48,18 @@ def chunk_document(text: str, chunk_chars: int) -> list[str]: return chunks +def chunk_text(text: str, chunk_chars: int) -> list[str]: + """Return the chunks of `text`, cut at blank lines outside fences, with no prefix. + + A heading line does not start a section. A block that does not fit is cut at line + ends, and a line that does not fit is cut at `chunk_chars`. The blocks are packed in + order into bodies of at most `chunk_chars`, joined by a blank line. A text of blank + lines only gives no chunk. + """ + lines = [_Line(line.text, 0, line.fenced) for line in _scan(text)] + return [chunk.text for chunk in pack(lines, chunk_chars)] + + def pack(lines: list[_Line], chunk_chars: int) -> list[Chunk]: """Return the sections of `lines`, cut and packed into bodies of at most `chunk_chars`. diff --git a/src/hotmemory/contract.py b/src/hotmemory/contract.py index 9314bb3..8963121 100644 --- a/src/hotmemory/contract.py +++ b/src/hotmemory/contract.py @@ -9,7 +9,7 @@ from datetime import UTC, datetime, timedelta from typing import Literal, TypeAlias -from hotmemory._chunks import chunk_document +from hotmemory._chunks import chunk_document, chunk_text from hotmemory._rules import ( check_prefix, is_duplicate, @@ -402,6 +402,7 @@ def capture( actor: str = "", observed_at: datetime | None = None, episode: str | None = None, + chunk_chars: int | None = None, ) -> list[str]: """Call `extractor` on `text` and remember the facts that it returns. @@ -410,15 +411,28 @@ def capture( `episode`, a key, `text` is also stored as an episode under that key, written after the facts, and each fact gets the id of the episode added to its `sources`. If that episode is current with the same content, the extractor is not called and nothing - is written. Raises ValueError if `episode` is not a valid key, holds another kind, - or `text` is blank. Returns the ids that `remember` returns. + is written. With `chunk_chars` as well, `text` is cut at blank lines into chunks of + at most `chunk_chars` characters, and chunk n is stored as an episode under the key + `-`, with n from 1 in four digits. The extractor still reads the whole + text, and each fact gets the id of every chunk added to its `sources`, in chunk + order. Only the chunks that changed are written, in one flush after the facts, and + if none changed, the extractor is not called. Raises ValueError if `episode` or a + chunk key is not a valid key or holds another kind, if `text` is blank, or if + `chunk_chars` is given without `episode` or is below 1. Returns the ids that + `remember` returns. """ if episode is None: + if chunk_chars is not None: + raise ValueError("chunk_chars needs episode") return self._extract(text, scope, extractor, actor, observed_at, ()) check_key(episode) if not text.strip(): raise ValueError("text must not be blank when capture stores an episode") namespace = check_namespace(scope) + if chunk_chars is not None: + return self._chunks( + namespace, episode, text, extractor, actor, observed_at, chunk_chars + ) _, facts = self._episode(namespace, episode, text, extractor, actor, observed_at) return facts @@ -487,6 +501,56 @@ def _episode( raise RuntimeError(f"episode {key!r} was written as {written!r}, not {episode!r}") return written, facts + def _chunks( + self, + namespace: tuple[str, ...], + episode: str, + text: str, + extractor: Extractor, + actor: str, + observed_at: datetime | None, + chunk_chars: int, + ) -> list[str]: + """Extract the facts of `text`, then store its changed chunks as episodes. + + Each fact gets the id of every chunk in its `sources`. If every chunk is current + with the same content, nothing is called or written. Returns the ids of the facts. + """ + check_count("chunk_chars", chunk_chars) + if chunk_chars < 1: + raise ValueError("chunk_chars must be 1 or more") + now = self._clock() + ids: list[str] = [] + changed: list[tuple[str, str, str]] = [] + for number, chunk in enumerate(chunk_text(text, chunk_chars), start=1): + key = f"{episode}-{number:04d}" + check_key(key) + current = self._store.get(namespace, key) + if current is not None and current.kind != "episode": + raise ValueError(f"key {key!r} holds kind {current.kind!r}, not an episode") + if current is not None and is_duplicate(current, chunk, now): + ids.append(current.id) + continue + ids.append(record_id(namespace, key, next_revision(current))) + changed.append((key, chunk, ids[-1])) + if not changed: + return [] + facts = self._extract(text, namespace, extractor, actor, observed_at, tuple(ids)) + with self._store.writer(max_rows=len(changed), interval=timedelta.max) as writer: + for key, chunk, _ in changed: + writer.put( + namespace, + key, + kind="episode", + content=chunk, + actor=actor, + observed_at=observed_at, + ) + expected = [chunk_id for _, _, chunk_id in changed] + if writer.flushed != expected: + raise RuntimeError(f"chunks were written as {writer.flushed!r}, not {expected!r}") + return facts + def _extract( self, text: str, diff --git a/src/hotmemory/hotdata.py b/src/hotmemory/hotdata.py index 9a4707e..51bd104 100644 --- a/src/hotmemory/hotdata.py +++ b/src/hotmemory/hotdata.py @@ -133,8 +133,8 @@ class _Index: ) -Ranking = Literal["fused", "vector"] -"""How `search` ranks: by fusion of BM25 and both vector rankings, or by content vector only.""" +Ranking = Literal["vector", "fused"] +"""How `search` ranks: by content vector only, or by fusion of BM25 and both vector rankings.""" class LayoutError(RuntimeError): @@ -149,10 +149,10 @@ class HotdataStore: database when it is provisioned, and every later open must name the same values. `clock` gives `created_at` and the time against which `forget_after` is compared. - `ranking="fused"`, the default, ranks a search by reciprocal rank fusion of BM25 over + `ranking="vector"`, the default, ranks a search by the content cosine distance alone, + as `MemoryStore` does. `ranking="fused"` ranks by reciprocal rank fusion of BM25 over `content`, cosine distance over the content vectors, and cosine distance over the cue - vectors. `ranking="vector"` ranks by the content cosine distance alone, as `MemoryStore` - does. In both, `Hit.distance` is the content cosine distance. + vectors. In both, `Hit.distance` is the content cosine distance. """ def __init__( @@ -164,10 +164,10 @@ def __init__( model: str, dimensions: int, clock: Clock = utc_now, - ranking: Ranking = "fused", + ranking: Ranking = "vector", ) -> None: - if ranking not in ("fused", "vector"): - raise ValueError(f"ranking must be 'fused' or 'vector', got {ranking!r}") + if ranking not in ("vector", "fused"): + raise ValueError(f"ranking must be 'vector' or 'fused', got {ranking!r}") self.client = client self.database = database self._embedder = embedder diff --git a/tests/test_chunks.py b/tests/test_chunks.py index 76fdedf..6bf0ed3 100644 --- a/tests/test_chunks.py +++ b/tests/test_chunks.py @@ -1,8 +1,8 @@ -"""Tests of the chunking that `Memory.load` uses. They need no driver.""" +"""Tests of the chunking that `Memory.load` and `Memory.capture` use. They need no driver.""" import pytest -from hotmemory._chunks import chunk_document +from hotmemory._chunks import chunk_document, chunk_text POST_MORTEM = """# 2026-08-12 Post-mortem: the pool ran out @@ -104,3 +104,36 @@ def test_every_body_fits_the_size() -> None: text = POST_MORTEM * 5 + "\n" + "\n".join(f"| row {n} | value |" for n in range(40)) for size in (40, 80, 200, 1000): assert all(len(body) <= size for body in bodies(chunk_document(text, size))) + + +SESSION = """user: How do I rotate the logs? + +assistant: ### Step 1 + +Stop the writer. + +```sh +# rotate + +logrotate -f app.conf +``` + +user: Thanks, that worked.""" + + +def test_chunk_text_keeps_a_heading_as_text_with_no_prefix() -> None: + assert chunk_text(SESSION, 2000) == [SESSION] + first, *rest = chunk_text(SESSION, 60) + assert first == "user: How do I rotate the logs?\n\nassistant: ### Step 1" + assert all(not chunk.startswith("#") for chunk in rest) + + +def test_chunk_text_never_cuts_at_a_blank_line_inside_a_fence() -> None: + chunks = chunk_text(SESSION, 60) + assert any("```sh\n# rotate\n\nlogrotate -f app.conf\n```" in chunk for chunk in chunks) + assert all(len(chunk) <= 60 for chunk in chunks) + assert "\n\n".join(chunks) == SESSION + + +def test_chunk_text_of_blank_lines_gives_no_chunk() -> None: + assert chunk_text(" \n\n \n", 100) == [] diff --git a/tests/test_contract.py b/tests/test_contract.py index 7859866..39a1f01 100644 --- a/tests/test_contract.py +++ b/tests/test_contract.py @@ -685,6 +685,103 @@ def test_capture_refuses_an_episode_key_that_holds_a_fact(memory: Memory) -> Non assert [record.key for record in memory.store.list(SCOPE)] == ["chat-0001"] +LONG_CHAT = f"{CHAT}\n\n{LATER}" + + +def test_capture_with_chunks_stores_each_chunk_and_links_every_chunk(memory: Memory) -> None: + calls: list[str] = [] + + def extractor(text: str, observed_at: object, current: list[Record]) -> list[Fact]: + calls.append(text) + return first_turn(text, observed_at, current) + + observed = START - timedelta(days=2) + ids = memory.capture( + LONG_CHAT, SCOPE, extractor, observed_at=observed, episode="chat", chunk_chars=60 + ) + + assert calls == [LONG_CHAT] + chunks = [memory.store.get(SCOPE, f"chat-{n:04d}") for n in (1, 2, 3, 4)] + assert [chunk.content if chunk else None for chunk in chunks] == LONG_CHAT.split("\n\n") + assert all(chunk is not None and chunk.kind == "episode" for chunk in chunks) + assert all(chunk is not None and chunk.observed_at == observed for chunk in chunks) + assert memory.store.get(SCOPE, "chat") is None + assert memory.store.get(SCOPE, "chat-0005") is None + [fact] = [memory.store.get(SCOPE, record_id.split("/")[-1].split("@")[0]) for record_id in ids] + assert fact is not None + assert fact.sources == ("chat", *(chunk.id for chunk in chunks if chunk)) + + +def test_capture_with_chunks_writes_the_chunks_in_one_flush(store: Store, clock: FakeClock) -> None: + counting = CountingStore(store) + memory = Memory(cast(Store, counting), clock=clock) + memory.capture(LONG_CHAT, SCOPE, first_turn, episode="chat", chunk_chars=60) + assert counting.calls == ["writer", "writer"] + + +def test_a_text_that_fits_in_one_chunk_gets_the_first_chunk_key(memory: Memory) -> None: + memory.capture(CHAT, SCOPE, first_turn, episode="chat", chunk_chars=2000) + chunk = memory.store.get(SCOPE, "chat-0001") + assert chunk is not None and chunk.content == CHAT + assert memory.store.get(SCOPE, "chat-0002") is None + + +def test_a_retried_capture_with_chunks_writes_nothing_new(memory: Memory, clock: FakeClock) -> None: + calls: list[str] = [] + + def extractor(text: str, observed_at: object, current: list[Record]) -> list[Fact]: + calls.append(text) + return first_turn(text, observed_at, current) + + memory.capture(LONG_CHAT, SCOPE, extractor, episode="chat", chunk_chars=60) + before = memory.store.list(SCOPE, limit=1000) + clock.advance() + assert memory.capture(LONG_CHAT, SCOPE, extractor, episode="chat", chunk_chars=60) == [] + assert calls == [LONG_CHAT] + assert memory.store.list(SCOPE, limit=1000) == before + + +def test_capture_of_an_edited_chunk_writes_its_next_revision_only( + memory: Memory, clock: FakeClock +) -> None: + memory.capture(LONG_CHAT, SCOPE, first_turn, episode="chat", chunk_chars=60) + clock.advance() + edited = LONG_CHAT.replace("before noon", "at noon") + [fact_id] = memory.capture(edited, SCOPE, first_turn, episode="chat", chunk_chars=60) + + revisions = [len(memory.store.history(SCOPE, f"chat-{n:04d}")) for n in (1, 2, 3, 4)] + assert revisions == [1, 2, 1, 1] + fact = memory.store.get(SCOPE, fact_id.split("/")[-1].split("@")[0]) + assert fact is not None + first = tuple(f"team/alerts/chat-{n:04d}@1" for n in (1, 2, 3, 4)) + assert fact.sources == ("chat", *first, "team/alerts/chat-0002@2") + + +@pytest.mark.parametrize( + ("episode", "chunk_chars", "match"), + [(None, 60, "needs episode"), ("chat", 0, "1 or more"), ("a@b", 60, "must not contain")], +) +def test_capture_refuses_bad_chunks( + memory: Memory, episode: str | None, chunk_chars: int, match: str +) -> None: + with pytest.raises(ValueError, match=match): + memory.capture(LONG_CHAT, SCOPE, first_turn, episode=episode, chunk_chars=chunk_chars) + assert memory.store.list(SCOPE) == [] + + +def test_capture_refuses_a_chunk_key_that_holds_a_fact(memory: Memory) -> None: + memory.store.put(SCOPE, "chat-0002", kind="fact", content="Not an episode.") + with pytest.raises(ValueError, match="episode"): + memory.capture(LONG_CHAT, SCOPE, first_turn, episode="chat", chunk_chars=60) + assert [record.key for record in memory.store.list(SCOPE)] == ["chat-0002"] + + +def test_recall_search_returns_the_chunks_that_match(memory: Memory) -> None: + memory.capture(LONG_CHAT, SCOPE, first_turn, episode="chat", chunk_chars=60) + records, _ = memory.recall("node group ceiling", [SCOPE], k=1, budget=4000, episodes="search") + assert [record.key for record in records if record.kind == "episode"] == ["chat-0004"] + + def test_recall_shows_a_source_in_its_namespace_as_key_and_revision(memory: Memory) -> None: sources = ("team/alerts/chat-0001@2", "team/alerts/night/chat-0002@1", "team/x@1", "wiki") memory.remember([Fact(kind="fact", content="The disk fills.", sources=sources)], SCOPE) diff --git a/tests/test_oracle.py b/tests/test_oracle.py index 90f3ddd..d5ce019 100644 --- a/tests/test_oracle.py +++ b/tests/test_oracle.py @@ -7,7 +7,7 @@ import pytest from hotdata_framework import HotdataClient, ManagedDatabase -from hotmemory import Filter, MemoryStore, Store +from hotmemory import Fact, Filter, Memory, MemoryStore, Record, Store from hotmemory.hotdata import HotdataStore, Ranking from .conftest import DIMENSIONS, MODEL, START, FakeClock, empty, fake_embedder @@ -142,3 +142,81 @@ def test_fused_ranking_returns_the_oracle_set( found = {hit.record.id for hit in drivers["fused"].search(query, prefixes, where, k=k)} assert found == expected + + +def test_the_default_ranking_is_the_vector_ranking( + hotdata_database: tuple[HotdataClient, ManagedDatabase], +) -> None: + client, database = hotdata_database + store = HotdataStore( + client, database, embedder=fake_embedder, model=MODEL, dimensions=DIMENSIONS + ) + assert store._ranking == "vector" + + +SESSIONS = [ + "user: The disk on db-1 fills at night.\n\nassistant: Rotate the logs before noon.", + "user: The pool ran out of nodes.\n\nassistant: Raise the node group ceiling.\n\n" + "user: The ceiling is now 8.", +] + + +def every_turn(text: str, observed_at: object, current: list[Record]) -> list[Fact]: + """Return one fact for each user turn of a chat.""" + return [ + Fact(kind="fact", subject="chat", content=turn.removeprefix("user: ")) + for turn in text.split("\n\n") + if turn.startswith("user: ") + ] + + +@pytest.mark.parametrize("episodes", ["search", "cited"]) +def test_both_drivers_recall_the_same_block_with_chunks( + hotdata_database: tuple[HotdataClient, ManagedDatabase], episodes: str +) -> None: + client, database = hotdata_database + clock = FakeClock() + oracle = Memory(MemoryStore(embedder=fake_embedder, clock=clock), clock=clock) + store = HotdataStore( + client, database, embedder=fake_embedder, model=MODEL, dimensions=DIMENSIONS, clock=clock + ) + driver = Memory(store, clock=clock) + try: + for memory in (oracle, driver): + clock.now = START + for number, text in enumerate(SESSIONS, start=1): + memory.capture(text, PAY, every_turn, episode=f"chat{number}", chunk_chars=50) + clock.advance() + blocks = [ + memory.recall("node ceiling", [PAY], k=3, budget=4000, episodes=episodes) # type: ignore[arg-type] + for memory in (oracle, driver) + ] + assert blocks[1][1] == blocks[0][1] + assert [record.id for record in blocks[1][0]] == [record.id for record in blocks[0][0]] + finally: + empty(store) + + +@pytest.mark.parametrize("ranking", ["vector", "fused"]) +def test_capture_searches_with_the_ranking_of_recall( + hotdata_database: tuple[HotdataClient, ManagedDatabase], ranking: Ranking +) -> None: + client, database = hotdata_database + clock = FakeClock() + store = hotdata_store(client, database, clock, ranking) + memory = Memory(store, clock=clock) + seen: list[list[str]] = [] + + def extractor(text: str, observed_at: object, current: list[Record]) -> list[Fact]: + seen.append([record.id for record in current]) + return [] + + try: + fill(store, clock) + text = "The disk on db-1 fills, and the queue backed up." + expected, _ = memory.recall(text, [("team",)]) + memory.capture(text, ("team",), extractor) + assert seen == [[record.id for record in expected]] + assert expected + finally: + empty(store) diff --git a/tests/test_skill.py b/tests/test_skill.py index 847449b..3c2ba77 100644 --- a/tests/test_skill.py +++ b/tests/test_skill.py @@ -35,7 +35,7 @@ def test_every_command_in_the_skill_file_exits_zero(tmp_path: Path) -> None: shutil.copytree(SKILL, folder, ignore=shutil.ignore_patterns("__pycache__")) path = os.pathsep.join([str(Path(sys.executable).parent), os.environ.get("PATH", "")]) found = commands() - assert len(found) == 10 + assert len(found) == 12 printed = {} for command in found: result = subprocess.run( @@ -58,3 +58,7 @@ def test_every_command_in_the_skill_file_exits_zero(tmp_path: Path) -> None: assert '"key": "chat-0001"' in records [cited] = [out for command, out in printed.items() if "--episodes cited" in command] assert "\nepisodes:\n## chat-0001@1 [observed: " in cited + assert '"key": "chat-0002-0001"' in records + assert '"key": "chat-0002-0002"' in records + [searched] = [out for command, out in printed.items() if "--episodes search" in command] + assert "\n## chat-0002-0002@1 [observed: " in searched From 89ee7fc756ca062d0c1c94e9728cc8f947e07237 Mon Sep 17 00:00:00 2001 From: Rohan Dsouza Date: Sun, 18 Oct 2026 13:03:53 +0530 Subject: [PATCH 3/7] Phase 10 task 5: the bench runner takes --chunk-chars bench.e2e takes --chunk-chars with --session-episodes, and capture stores each session as chunks of that size. The chunk size is part of the kept name. bench.diagnose maps a chunk key to its session. --- bench/README.md | 1 + bench/diagnose.py | 14 +++++++++---- bench/e2e.py | 40 ++++++++++++++++++++++++++++++------ bench/tests/test_diagnose.py | 3 +++ bench/tests/test_e2e.py | 38 ++++++++++++++++++++++++++++++++++ 5 files changed, 86 insertions(+), 10 deletions(-) diff --git a/bench/README.md b/bench/README.md index fa091f7..3aafbed 100644 --- a/bench/README.md +++ b/bench/README.md @@ -89,6 +89,7 @@ The options are these: | `--extractor` | a variant name in `VARIANTS` of `bench/prompts.py` | `baseline` | | `--no-fact-sources` | write the facts with no session source, as phases 5 and 6 did | off | | `--session-episodes` | capture each session as an episode, with the facts citing it | off | +| `--chunk-chars` | store each session as chunks of at most this many characters, with each fact citing every chunk; needs `--session-episodes` | off | | `--episodes` | `none`, `cited`, or `search`, the `episodes` of the memory recall; a value other than `none` needs `--session-episodes` | `none` | | `--replay` | call no model, and stop at the first reply that the cache does not hold | off | | `--workers` | the number of processes that run the questions | `1` | diff --git a/bench/diagnose.py b/bench/diagnose.py index 45b1da1..199a06a 100644 --- a/bench/diagnose.py +++ b/bench/diagnose.py @@ -38,6 +38,7 @@ existing_databases, fact_source, local_store, + session_key, ) from bench.embed import CACHE_DIR from bench.models import JUDGE, CachedModels, Offline, Remote @@ -132,12 +133,17 @@ def memory_config(result: dict[str, Any]) -> Config: fact_sources=found.get("fact_sources", False), session_episodes=found.get("session_episodes", False), episodes=found.get("episodes", "none"), + chunk_chars=found.get("chunk_chars"), ) def evidence_in_block(question: Question, shown: Sequence[str]) -> bool: - """Return True if an episode key in `shown` is that of an evidence session of `question`.""" - return any(episode_key(session_id) in shown for session_id in question.answer_session_ids) + """Return True if an episode key in `shown` is that of an evidence session of `question`. + + A chunk key counts as the key of its session. + """ + keys = {session_key(key) for key in shown} + return any(episode_key(session_id) in keys for session_id in question.answer_session_ids) def table(rows: Sequence[dict[str, Any]]) -> str: @@ -305,10 +311,10 @@ def session_of_fact(fact: Record, sessions: dict[str, str]) -> str | None: """Return the session id of `fact` from its sources, or None if no source names one. `sessions` maps the results of `fact_source` and of `episode_key` to the session id. A - source that is an episode id matches by its key. + source that is an episode id matches by its key, and a chunk by the key of its session. """ for source in fact.sources: - key = source.rpartition("/")[2].rpartition("@")[0] + key = session_key(source.rpartition("/")[2].rpartition("@")[0]) for name in (source, key): if name in sessions: return sessions[name] diff --git a/bench/e2e.py b/bench/e2e.py index 04bc05b..3ceedcb 100644 --- a/bench/e2e.py +++ b/bench/e2e.py @@ -24,6 +24,7 @@ import json import multiprocessing import random +import re import subprocess import time import uuid @@ -86,8 +87,9 @@ class Config: `budget` characters, and extracts with the variant `extractor` of `prompts.VARIANTS`. With `fact_sources`, each fact carries the source of its session from `fact_source`. With `session_episodes`, `capture` stores each session as an episode under the key - from `episode_key`, and the facts carry no session source. `episodes` is the value of - `recall`, and needs `session_episodes` unless it is `none`. + from `episode_key`, and the facts carry no session source. With `chunk_chars` as well, + `capture` stores each session as chunks of at most that many characters. `episodes` is + the value of `recall`, and needs `session_episodes` unless it is `none`. """ retrieval_ranking: str = "vector" @@ -98,6 +100,7 @@ class Config: fact_sources: bool = True session_episodes: bool = False episodes: str = "none" + chunk_chars: int | None = None def __post_init__(self) -> None: for ranking in (self.retrieval_ranking, self.memory_ranking): @@ -111,6 +114,8 @@ def __post_init__(self) -> None: raise ValueError(f"episodes must be one of {', '.join(EPISODES)}") if self.episodes != "none" and not self.session_episodes: raise ValueError("episodes needs session_episodes") + if self.chunk_chars is not None and (self.chunk_chars < 1 or not self.session_episodes): + raise ValueError("chunk_chars must be at least 1, and needs session_episodes") @property def variant(self) -> prompts.Variant: @@ -202,11 +207,19 @@ def fact_source(session_id: str) -> str: return "session:" + _session_hash(session_id) +CHUNK_SUFFIX = re.compile(r"(?<=^session-[0-9a-f]{12})-[0-9]{4}$") + + def episode_key(session_id: str) -> str: """Return the key of the episode of a session: `session-` and the hash of `fact_source`.""" return "session-" + _session_hash(session_id) +def session_key(key: str) -> str: + """Return the episode key of the session of `key`: `key` without a chunk suffix `-`.""" + return CHUNK_SUFFIX.sub("", key) + + def _session_hash(session_id: str) -> str: """Return the first 12 hex digits of the SHA-256 of `session_id`.""" return hashlib.sha256(session_id.encode()).hexdigest()[:12] @@ -358,7 +371,8 @@ def capture_session( """Run `capture` on one session as the memory arm does, and return the fact ids. With `config.session_episodes`, the session is stored as an episode under - `episode_key`. Else, with `config.fact_sources`, each fact carries `fact_source`. + `episode_key`, or as chunks of `config.chunk_chars` under keys that start with it. + Else, with `config.fact_sources`, each fact carries `fact_source`. """ if config.session_episodes: source, episode = "", episode_key(session.id) @@ -371,6 +385,7 @@ def capture_session( actor=ACTOR, observed_at=session.date, episode=episode, + chunk_chars=config.chunk_chars if episode is not None else None, ) @@ -527,13 +542,14 @@ def kept_name(sample: str, config: Config, commit: str) -> str: The name holds what decides the memory that `capture` writes: the sample, the extractor variant with 6 hex digits of the SHA-256 of its model and prompt, how the facts cite - their session (`episodes`, `sources`, or `plain`), the memory ranking, and the commit. + their session (`episodes`, `chunks`, `sources`, or `plain`), the memory ranking, + and the commit. The database of one question appends `-`. """ variant = config.variant digest = hashlib.sha256(f"{variant.model}\n{variant.prompt}".encode()).hexdigest()[:6] if config.session_episodes: - cites = "episodes" + cites = "episodes" if config.chunk_chars is None else f"chunks{config.chunk_chars}" elif config.fact_sources: cites = "sources" else: @@ -806,6 +822,12 @@ def main(argv: Sequence[str] | None = None) -> Path | None: action="store_true", help="capture each session as an episode, and write the facts with no session source", ) + parser.add_argument( + "--chunk-chars", + type=int, + default=None, + help="with --session-episodes: store each session as chunks of this many characters", + ) parser.add_argument( "--episodes", choices=EPISODES, default="none", help="the episodes value of recall" ) @@ -857,6 +879,7 @@ def main(argv: Sequence[str] | None = None) -> Path | None: fact_sources=not args.no_fact_sources, session_episodes=args.session_episodes, episodes=args.episodes, + chunk_chars=args.chunk_chars, ) except ValueError as error: parser.error(str(error)) @@ -1042,7 +1065,12 @@ def main(argv: Sequence[str] | None = None) -> Path | None: ), "memory": ( f"capture on each session in date order with the {config.extractor} " - f"extractor, then recall with the {config.memory_ranking} ranking, k " + + ( + f"extractor, in chunks of {config.chunk_chars} characters, then recall" + if config.chunk_chars is not None + else "extractor, then recall" + ) + + f" with the {config.memory_ranking} ranking, k " f"{config.k}, budget {config.budget} characters, episodes " f"{config.episodes}, as_of the end of the question day" ), diff --git a/bench/tests/test_diagnose.py b/bench/tests/test_diagnose.py index 3efdcbf..c1e0b58 100644 --- a/bench/tests/test_diagnose.py +++ b/bench/tests/test_diagnose.py @@ -87,6 +87,8 @@ def test_session_of_fact_reads_the_source() -> None: keys = {episode_key("answer_a_1"): "answer_a_1"} cited = replace(fact("k"), sources=(f"bench/q/{episode_key('answer_a_1')}@1",)) assert session_of_fact(cited, keys) == "answer_a_1" + chunk = replace(fact("k"), sources=(f"bench/q/{episode_key('answer_a_1')}-0002@1",)) + assert session_of_fact(chunk, keys) == "answer_a_1" def test_evidence_in_block_matches_the_episode_keys() -> None: @@ -101,6 +103,7 @@ def test_evidence_in_block_matches_the_episode_keys() -> None: ) assert evidence_in_block(question, [episode_key("answer_a_1")]) assert not evidence_in_block(question, [episode_key("other")]) + assert evidence_in_block(question, [f"{episode_key('answer_a_1')}-0003"]) assert not evidence_in_block(question, []) diff --git a/bench/tests/test_e2e.py b/bench/tests/test_e2e.py index 3d3872b..eabb801 100644 --- a/bench/tests/test_e2e.py +++ b/bench/tests/test_e2e.py @@ -33,6 +33,7 @@ run_question, run_shard, sample_ids, + session_key, shards_of, smoke_ids, summarize, @@ -393,9 +394,45 @@ def test_capture_session_replays_the_same_extractor_prompts( assert seen[0] == seen[1] +def test_capture_session_with_chunks_stores_chunks_and_the_same_prompts( + questions: dict[str, Question], +) -> None: + question = questions["q_one"] + seen: dict[int | None, list[str]] = {} + for chunk_chars in (None, 40): + models = FakeModels() + memory = Memory(MemoryStore(embedder=fake_embedder, clock=FakeClock())) + config = Config(session_episodes=True, chunk_chars=chunk_chars) + namespace = ("bench", "r", question.id) + for session in question.sessions: + capture_session(memory, namespace, session, models, config, []) + seen[chunk_chars] = [p for m, p in models.prompts if m == EXTRACTOR] + keys = {record.key for record in memory.store.list(namespace, limit=1000)} + first = episode_key(question.sessions[0].id) + assert (f"{first}-0001" in keys) == (chunk_chars is not None) + assert (first in keys) == (chunk_chars is None) + assert seen[40] == seen[None] + + +def test_session_key_drops_a_chunk_suffix() -> None: + key = episode_key("answer_a_1") + assert session_key(f"{key}-0002") == key + assert session_key(key) == key + assert session_key("doc-0002") == "doc-0002" + + +@pytest.mark.parametrize("chunk_chars", [0, 2000]) +def test_chunk_chars_needs_session_episodes(chunk_chars: int) -> None: + with pytest.raises(ValueError, match="chunk_chars"): + Config(chunk_chars=chunk_chars, session_episodes=chunk_chars == 0) + + def test_kept_name_holds_what_decides_the_memory() -> None: config = Config(extractor="detail", session_episodes=True, episodes="search", budget=64000) name = kept_name("a", config, "b5443ac") + assert kept_name("a", replace(config, chunk_chars=2000), "b5443ac").endswith( + "-chunks2000-vector-b5443ac" + ) assert name.startswith("hotmemory-kept-a-detail") assert name.endswith("-episodes-vector-b5443ac") after_capture = replace(config, episodes="cited", budget=32000, k=10) @@ -407,6 +444,7 @@ def test_kept_name_holds_what_decides_the_memory() -> None: kept_name("a", replace(config, memory_ranking="fused"), "b5443ac"), kept_name("a", Config(extractor="detail"), "b5443ac"), kept_name("a", Config(extractor="detail", fact_sources=False), "b5443ac"), + kept_name("a", replace(config, chunk_chars=2000), "b5443ac"), ): assert other != name From 12339f880e876d89b73d1dd0327f4138b73acae7 Mon Sep 17 00:00:00 2001 From: Rohan Dsouza Date: Sun, 18 Oct 2026 14:01:37 +0530 Subject: [PATCH 4/7] Show a run of chunk sources as one range in the block MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A fact captured in chunks cites every chunk, so its line in the block grew by about 300 characters at 1,000-character chunks. The block now shows a run of sources -@, with n counting up by 1 and one revision, as -…@. The stored sources do not change. --- CHANGELOG.md | 3 +++ docs/contracts.md | 13 +++++++---- docs/guarantees.md | 1 + src/hotmemory/contract.py | 37 +++++++++++++++++++++++++++++-- tests/test_contract.py | 46 ++++++++++++++++++++++++++++++++++++++- 5 files changed, 93 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6af657b..eab91b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -72,3 +72,6 @@ each one, and its failure message points to this file. - `Memory.capture` takes `chunk_chars`, with `episode`. It cuts the text at blank lines into chunks and stores chunk n as an episode under `-`. The extractor still reads the whole text, and each fact names every chunk in `sources`. +- The block of `recall` and `profile` shows a run of sources `-@`, + with n counting up by 1 and one revision, as `-…@`. The + stored sources do not change. diff --git a/docs/contracts.md b/docs/contracts.md index 456c1e4..8e51ffd 100644 --- a/docs/contracts.md +++ b/docs/contracts.md @@ -327,7 +327,10 @@ Times are in ISO 8601 in UTC, to the second. A null `valid_from` shows as `unkno null `valid_until` as `now`, and a record with no sources as `none`. A source that is the id of a record in the same namespace shows as `@`, without the namespace. For example, a fact under `team/alerts` with the source `team/alerts/notes-0001@1` shows -`[sources: notes-0001@1]`. Any other source shows in full. The block holds whole +`[sources: notes-0001@1]`. A run of such sources next to each other shows as one range +if their keys are `-` with one prefix, n in four digits counts up by 1, and +the revision is the same. For example, `notes-0001@1, notes-0002@1, notes-0003@1` shows as +`notes-0001…0003@1`. Any other source shows in full. The block holds whole lines only, joined by newlines. It stops before the first line that does not fit in the budget. The list holds the records of the block. The budget counts the characters of the block in this line format, and its default is 2,000. A consumer that renders the @@ -415,9 +418,11 @@ session that is not Markdown. A blank line separates two turns of such a text. from 1 in four digits. Thus a text that fits in one chunk is stored under `-0001`. -The extractor reads the whole text once, as it does without `chunk_chars`. Its call and -its facts do not change. The extractor does not say which chunk a fact came from. Thus -each fact gets the id of every chunk added to its `sources`, in chunk order. The facts +The extractor reads the whole text once, as it does without `chunk_chars`. The extractor +does not say which chunk a fact came from. Thus each fact gets the id of every chunk added +to its `sources`, in chunk order. The block shows those ids as one range, so a fact line +stays short. The extractor also gets the records that `recall` returns for the text, and a +longer line leaves room for fewer of them. The facts are written first, and then one flush writes the chunks. If every chunk is current with the same content, `capture` calls no extractor, writes nothing, and returns no id. If a chunk changed, `capture` calls the extractor again, and it writes the next revision of diff --git a/docs/guarantees.md b/docs/guarantees.md index f085c66..b2bc49d 100644 --- a/docs/guarantees.md +++ b/docs/guarantees.md @@ -61,6 +61,7 @@ A row that no test proves yet names the phase that will prove it. | Does a retried `capture` with an episode key write anything? | No. If the episode is current with the same content, the extractor is not called and nothing is written, so this holds for an extractor that calls a model. | answered | `test_a_retried_capture_with_an_episode_writes_nothing_new` | | What does `capture` store with an episode key and `chunk_chars`? | The text, cut at blank lines outside fences into chunks of at most `chunk_chars` characters, with no heading prefix. Chunk n is an episode under the key `-`. The extractor reads the whole text once. Each fact names the id of every chunk in `sources`, in chunk order. The facts take one flush, and the changed chunks take one more. `chunk_chars` without an episode key, or a chunk key that holds another kind, makes the call refuse before the extractor runs. | answered | `test_capture_with_chunks_stores_each_chunk_and_links_every_chunk`, `test_capture_with_chunks_writes_the_chunks_in_one_flush`, `test_capture_refuses_bad_chunks`, `test_capture_refuses_a_chunk_key_that_holds_a_fact`, `test_chunk_text_keeps_a_heading_as_text_with_no_prefix` | | Does a retried `capture` with chunks write anything? | No. If every chunk is current with the same content, the extractor is not called and nothing is written. If one chunk changed, the extractor runs again, and only that chunk gets its next revision. | answered | `test_a_retried_capture_with_chunks_writes_nothing_new`, `test_capture_of_an_edited_chunk_writes_its_next_revision_only` | +| How does the block show the chunks that a fact cites? | As one range. A run of sources in the namespace of the line, next to each other, with keys `-` of one prefix, n counting up by 1, and one revision, shows as `-…@`. Every other source shows as before, so a range is exact also for keys that a caller chose. The stored sources do not change. | answered | `test_recall_shows_a_run_of_chunks_as_a_range`, `test_a_fact_captured_in_chunks_shows_its_chunks_as_one_range` | | Do the two drivers recall the same block with chunks? | Yes. After the same `capture` calls with `chunk_chars`, `MemoryStore` and `HotdataStore` with its default ranking return the same records and the same block, with `episodes="search"` and with `episodes="cited"`. A search over episodes ranks each chunk on its own. | answered [measured, local] 2026-10-16 | `test_both_drivers_recall_the_same_block_with_chunks`, `test_recall_search_returns_the_chunks_that_match` | | Which ranking does the search inside `capture` use? | The ranking of the store, as `recall` does, because `capture` calls `recall`. The default of `HotdataStore` is `ranking="vector"`, the order of `MemoryStore`. | answered [measured, local] 2026-10-16 | `test_capture_searches_with_the_ranking_of_recall`, `test_the_default_ranking_is_the_vector_ranking` | | Which episodes does `recall` add with `episodes`? | With `"cited"`, the episodes that the facts of the block name in `sources`, in order, once each, and only inside the allowed scopes. With `"search"`, the top k episodes for the query. Neither value returns an episode past its `forget_after`. `"cited"` keeps a superseded revision, because the fact cites that revision. `as_of` filters them as it filters facts. They follow the facts, whole, and the block stops before the first episode that does not fit. An episode never takes the place of a fact. | answered | `test_recall_cited_adds_the_episodes_of_the_facts`, `test_recall_cited_names_each_episode_once`, `test_recall_cited_skips_an_episode_outside_the_scopes`, `test_recall_search_adds_the_top_episodes`, `test_recall_episodes_never_take_the_place_of_a_fact`, `test_recall_stops_before_the_first_episode_that_does_not_fit`, `test_recall_as_of_filters_the_episodes`, `test_recall_refuses_an_unknown_episodes_value`, `test_recall_cited_skips_an_episode_past_its_forget_after`, `test_recall_cited_keeps_a_superseded_revision` | diff --git a/src/hotmemory/contract.py b/src/hotmemory/contract.py index 8963121..0f9e1ea 100644 --- a/src/hotmemory/contract.py +++ b/src/hotmemory/contract.py @@ -41,6 +41,8 @@ PROFILE_LIMIT = 100 COUNT_LIMIT = 1000 _NOT_IN_KEY = re.compile(r"[^A-Za-z0-9_-]") +_CHUNK_ID = re.compile(r"^(.+)-([0-9]{4})@([0-9]+)$") +RANGE_MARK = "\u2026" RECALLED_KINDS: tuple[Kind, ...] = ("fact", "profile", "procedure") EPISODES_LINE = "episodes:" Episodes: TypeAlias = Literal["cited", "search"] @@ -88,10 +90,12 @@ def render_line(record: Record) -> str: The line is `- [sources: a, b] [valid: to ]`. Each run of whitespace in the content becomes one space. A null `valid_from` shows as `unknown`, a null `valid_until` as `now`, and no sources as `none`. A source that is the id of a - record in the namespace of `record` shows as `@`. + record in the namespace of `record` shows as `@`. A run of two or more + such sources, next to each other, with keys `-` of one prefix, n in four + digits counting up by 1, and one revision, shows as `-…@`. """ content = " ".join(record.content.split()) - sources = ", ".join(_shown_source(source, record.namespace) for source in record.sources) + sources = ", ".join(_shown_sources(record.sources, record.namespace)) sources = sources or "none" start = _time(record.valid_from, "unknown") end = _time(record.valid_until, "now") @@ -595,6 +599,35 @@ def _slot(record_id: str) -> tuple[tuple[str, ...], str]: return namespace, key +def _shown_sources(sources: Sequence[str], namespace: tuple[str, ...]) -> list[str]: + """Return `sources` as the block shows them, with each run of chunks shown as a range.""" + result: list[str] = [] + run: list[tuple[str, int, str]] = [] + + def close() -> None: + if len(run) > 1: + prefix, first, revision = run[0] + result.append(f"{prefix}-{first:04d}{RANGE_MARK}{run[-1][1]:04d}@{revision}") + elif run: + prefix, number, revision = run[0] + result.append(f"{prefix}-{number:04d}@{revision}") + run.clear() + + for source in sources: + text = _shown_source(source, namespace) + found = _CHUNK_ID.match(text) if text != source else None + if found is None: + close() + result.append(text) + continue + prefix, number, revision = found.group(1), int(found.group(2)), found.group(3) + if run and (run[-1][0], run[-1][1] + 1, run[-1][2]) != (prefix, number, revision): + close() + run.append((prefix, number, revision)) + close() + return result + + def _shown_source(source: str, namespace: tuple[str, ...]) -> str: """Return `source` as the block shows it: `@` for a record in `namespace`.""" prefix = "/".join(namespace) + "/" diff --git a/tests/test_contract.py b/tests/test_contract.py index 39a1f01..26897fc 100644 --- a/tests/test_contract.py +++ b/tests/test_contract.py @@ -782,6 +782,50 @@ def test_recall_search_returns_the_chunks_that_match(memory: Memory) -> None: assert [record.key for record in records if record.kind == "episode"] == ["chat-0004"] +@pytest.mark.parametrize( + ("sources", "shown"), + [ + ( + ("chat", "team/alerts/s1-0001@1", "team/alerts/s1-0002@1", "team/alerts/s1-0003@1"), + "chat, s1-0001\u20260003@1", + ), + ( + ("team/alerts/s1-0001@1", "team/alerts/s1-0002@1", "team/alerts/s1-0002@2"), + "s1-0001\u20260002@1, s1-0002@2", + ), + ( + ("team/alerts/s1-0001@1", "team/alerts/s1-0003@1", "team/alerts/s1-0004@1"), + "s1-0001@1, s1-0003\u20260004@1", + ), + ( + ("team/alerts/s1-0002@1", "team/alerts/s1-0001@1"), + "s1-0002@1, s1-0001@1", + ), + ( + ("team/alerts/s1-0001@1", "team/alerts/s1-0002@1", "team/alerts/s2-0001@1"), + "s1-0001\u20260002@1, s2-0001@1", + ), + (("team/alerts/doc-0003@1",), "doc-0003@1"), + (("team/alerts/s1-0001@1", "team/night/s1-0002@1"), "s1-0001@1, team/night/s1-0002@1"), + (("team/alerts/s1-01@1", "team/alerts/s1-02@1"), "s1-01@1, s1-02@1"), + ], +) +def test_recall_shows_a_run_of_chunks_as_a_range( + memory: Memory, sources: tuple[str, ...], shown: str +) -> None: + memory.remember([Fact(kind="fact", content="The disk fills.", sources=sources)], SCOPE) + _, block = memory.recall("disk", [SCOPE]) + assert block == f"- The disk fills. [sources: {shown}] [valid: unknown to now]" + + +def test_a_fact_captured_in_chunks_shows_its_chunks_as_one_range(memory: Memory) -> None: + memory.capture(LONG_CHAT, SCOPE, first_turn, episode="talk", chunk_chars=60) + _, block = memory.recall("disk fills at night", [SCOPE]) + assert block == ( + "- The disk fills at night. [sources: chat, talk-0001\u20260004@1] [valid: unknown to now]" + ) + + def test_recall_shows_a_source_in_its_namespace_as_key_and_revision(memory: Memory) -> None: sources = ("team/alerts/chat-0001@2", "team/alerts/night/chat-0002@1", "team/x@1", "wiki") memory.remember([Fact(kind="fact", content="The disk fills.", sources=sources)], SCOPE) @@ -830,7 +874,7 @@ def test_recall_cited_names_each_episode_once(memory: Memory, clock: FakeClock) "chat-0002", "chat-0003", ] - assert "[sources: chat, chat-0002@1, chat-0003@1]" in block + assert "[sources: chat, chat-0002\u20260003@1]" in block def test_recall_cited_skips_an_episode_outside_the_scopes(memory: Memory) -> None: From 6f1166b1abbec76fb5c18cd1567e10b129eff553 Mon Sep 17 00:00:00 2001 From: Rohan Dsouza Date: Sun, 18 Oct 2026 15:33:12 +0530 Subject: [PATCH 5/7] Phase 10 task 5: report the chunk sizes and budgets on sample A --- docs/benchmarks.md | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/docs/benchmarks.md b/docs/benchmarks.md index e6e7674..35dff39 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -529,6 +529,48 @@ extractor reads the facts that the search of the store returns. Thus the two run different facts. The vector run took 1,770.6 seconds and 1,722 model calls. In one session, the extractor reply reached the token limit, and that session gave no facts. +### Episode chunks + +Phase 10 stored each session as chunks with `capture`, `chunk_chars`, and ran `search` +with each chunk size at three budgets, on sample A, with the vector ranking, +`--extractor detail`, and 4 workers. Each chunk size was one run with `--keep` at 8,000 +characters, and each larger budget was a run with `--reuse`. Observed on 2026-10-18, +with `src/hotmemory` at `12339f8`. The reader tokens are the median over the 54 +questions, and the chunks are the mean count of chunks in the block. + +| Chunk size | Budget | Correct of 54 | Reader tokens | Chunks in the block | Cost | File | +|---|---|---|---|---|---|---| +| 1,000 | 8,000 | 41 | 2,471 | 3.1 | $0.55 | `e2e-sample-a-p10-keep-chunks1000-8k.json` | +| 1,000 | 16,000 | 48 | 4,336 | 12.4 | $0.81 | `e2e-sample-a-p10-reuse-chunks1000-16k.json` | +| 1,000 | 32,000 | 49 | 7,832 | 29.9 | $1.27 | `e2e-sample-a-p10-reuse-chunks1000-32k.json` | +| 2,000 | 8,000 | 36 | 2,327 | 1.4 | $0.56 | `e2e-sample-a-p10-keep-chunks2000-8k.json` | +| 2,000 | 16,000 | 46 | 4,171 | 6.2 | $0.78 | `e2e-sample-a-p10-reuse-chunks2000-16k.json` | +| 2,000 | 32,000 | 48 | 7,717 | 15.7 | $1.26 | `e2e-sample-a-p10-reuse-chunks2000-32k.json` | +| 4,000 | 8,000 | 31 | 1,878 | 0.2 | $0.49 | `e2e-sample-a-p10-keep-chunks4000-8k.json` | +| 4,000 | 16,000 | 44 | 3,780 | 2.6 | $0.73 | `e2e-sample-a-p10-reuse-chunks4000-16k.json` | +| 4,000 | 32,000 | 46 | 7,509 | 7.4 | $1.21 | `e2e-sample-a-p10-reuse-chunks4000-32k.json` | + +The runs of phase 8 with whole sessions as episodes, for comparison: + +| Run | Correct of 54 | Reader tokens | File | +|---|---|---|---| +| `search`, 16,000 | 34 | 1,739 | `e2e-sample-a-p8-search-16k.json` | +| `search`, 32,000 | 43 | 5,716 | `e2e-sample-a-p8-search-32k.json` | +| `search`, 64,000 | 45 | 12,849 | `e2e-sample-a-p8-search-64k.json` | +| `search`, 64,000, vector ranking, phase 10 | 46 | 13,406 | `e2e-sample-a-p10-keep-vector-search-64k.json` | + +Chunks of 1,000 characters at 16,000 scored 48 with 4,336 reader tokens. The best whole-session +run scored 45 with 12,849 tokens. At 8,000 characters, the facts take most of the budget, +because they fill it before the episodes. At 32,000 characters with chunks of 1,000, the +block held 29.9 chunks on average, so `k`, 30, and not the budget, limited the episodes. + +The block of `recall` shows a run of chunk sources as one range, as +[docs/contracts.md](contracts.md) gives. A fact line is still a little longer than with one +episode, so the list of known facts that the extractor reads can hold one fact less. Thus +part of each keep run missed the cache of the earlier vector run and paid for extraction. +The smoke run with chunks of 2,000 at 64,000 scored 16 of 18, as in phase 9, for $0.71, +`e2e-smoke-p10-smoke-chunks2000.json`. + ## Known limits - The end-to-end numbers come from samples of 54 questions, so one question moves an arm From d241eae1373790fc80df895ae9a12c0bdc8bd180 Mon Sep 17 00:00:00 2001 From: Rohan Dsouza Date: Sun, 18 Oct 2026 16:46:17 +0530 Subject: [PATCH 6/7] Phase 10 task 6: the best chunk configuration on both samples, and its diagnosis --- docs/benchmarks.md | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/docs/benchmarks.md b/docs/benchmarks.md index 35dff39..31b6b26 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -571,6 +571,38 @@ part of each keep run missed the cache of the earlier vector run and paid for ex The smoke run with chunks of 2,000 at 64,000 scored 16 of 18, as in phase 9, for $0.71, `e2e-smoke-p10-smoke-chunks2000.json`. +### The best chunk configuration on both samples + +Phase 10 ran chunks of 1,000 characters at 16,000 on sample B, with `--keep`. It also +kept the memory of sample A again at 16,000 for the diagnosis. That run called no model, +and its 54 blocks had the same SHA-256 and verdict as the `--reuse` run of the table above. +Observed on 2026-10-18, with `src/hotmemory` at `12339f8`. + +| Sample | Run | Correct of 54 | Reader tokens | Cost | File | +|---|---|---|---|---|---| +| A | chunks of 1,000, 16,000 | 48 | 4,336 | $0 | `e2e-sample-a-p10-keep-chunks1000-16k.json` | +| A | phase 8, `search`, 64,000 | 45 | 12,849 | | `e2e-sample-a-p8-search-64k.json` | +| B | chunks of 1,000, 16,000 | 46 | 4,360 | $2.05 | `e2e-sample-b-p10-keep-chunks1000-16k.json` | +| B | phase 8, `search`, 64,000 | 45 | 12,677 | | `e2e-sample-b-p8-search-64k.json` | + +The diagnosis read the kept memory of each run. The phase 8 rows are those of the +diagnoses of `search` at 64,000. + +| Run | Missed | Not extracted | Extracted, ranked below k | Recalled, all present | Recalled, some present | Evidence chunk or episode in block | +|---|---|---|---|---|---|---| +| A, chunks of 1,000, 16,000 | 6 | 2 | 0 | 2 | 2 | 6 | +| A, phase 8, `search`, 64,000 | 9 | 5 | 0 | 2 | 2 | 8 | +| B, chunks of 1,000, 16,000 | 8 | 2 | 1 | 2 | 3 | 7 | +| B, phase 8, `search`, 64,000 | 9 | 5 | 0 | 1 | 3 | 8 | + +Files: the result files above with `-diagnosis` before `.json`. With chunks, a chunk of an +evidence session counts as that session in the last column. The diagnoses cost $0.08 on +sample A and $0.10 on sample B. + +The retrieval score with the vector ranking, rerun on the 419 questions at the same commit, +gave the same sessions in the same order as the vector ranking of the ablation, on 419 of +419 questions: `recall_all@10` 0.9928. File: `retrieval-hotdata-local-isolated-vector.json`. + ## Known limits - The end-to-end numbers come from samples of 54 questions, so one question moves an arm From d9b719c409ac7b452d0bde6bbf4f831eb8a819a6 Mon Sep 17 00:00:00 2001 From: Rohan Dsouza Date: Sun, 18 Oct 2026 20:06:17 +0530 Subject: [PATCH 7/7] Audit the docs of phase 10 The bench README names chunks in the kept name and limits the BM25 warnings to the fused ranking. The benchmarks report which keep runs paid for extraction, and round each median down. The guarantees, contracts, and changelog state the range rule in full. The plan gives task 3 as it was built. --- CHANGELOG.md | 5 +++-- CONTRIBUTING.md | 2 +- README.md | 2 +- bench/README.md | 26 +++++++++++++++----------- docs/benchmarks.md | 28 +++++++++++++++------------- docs/contracts.md | 7 ++++--- docs/guarantees.md | 4 ++-- docs/internal/architecture.md | 4 ++-- docs/internal/plan.md | 9 ++++++--- 9 files changed, 49 insertions(+), 38 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eab91b6..6ef831a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -72,6 +72,7 @@ each one, and its failure message points to this file. - `Memory.capture` takes `chunk_chars`, with `episode`. It cuts the text at blank lines into chunks and stores chunk n as an episode under `-`. The extractor still reads the whole text, and each fact names every chunk in `sources`. -- The block of `recall` and `profile` shows a run of sources `-@`, - with n counting up by 1 and one revision, as `-…@`. The +- The block of `recall` and `profile` shows a run of sources in the namespace of its line, + with keys `-`, n in four digits counting up by 1, and one revision, as + `-…@`. The stored sources do not change. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ed303a8..dff8b80 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -53,7 +53,7 @@ model. The tests pass a fixed clock and a fake embedder to the store, from `tests/conftest.py`. - `tests/test_contract.py` runs `Memory` over every driver through the same `store` fixture. It pins the rendered blocks of `recall` and `profile` as exact strings. - `tests/test_chunks.py` tests the chunking of `load` without a driver. + `tests/test_chunks.py` tests the chunking of `load` and `capture` without a driver. - The in-memory driver is the reference for the Hotdata driver. `tests/test_oracle.py` builds the same records in both drivers and compares the results of `search`. The ranking by content vector alone must return the same records in the same order, with diff --git a/README.md b/README.md index b580a7d..60d39a8 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ The library has two layers: text is optional, and it takes an extractor callable that the caller supplies. `load` cuts a Markdown document into chunks, stores each chunk as an episode, and extracts facts that name their episode as a source. `capture` with an `episode` key stores its - text as an episode in the same way. `profile` returns facts, profiles, and procedures, + text as an episode in the same way, or as chunks with `chunk_chars`. `profile` returns facts, profiles, and procedures, never episodes. `recall` returns them too, and adds episodes after the facts only when the caller asks for them. diff --git a/bench/README.md b/bench/README.md index 3aafbed..037b863 100644 --- a/bench/README.md +++ b/bench/README.md @@ -102,9 +102,9 @@ The options are these: Each worker gets the questions in turn. Without `--keep` or `--reuse`, it runs them over a new local database of its own, and empties the namespaces of each question after it. Each -table then holds one question at a time, as in a run with one worker. BM25 ranks -over the whole table before the scope filter, so the rows of a second question in a table -can change the ranking. An extractor variant names a prompt and a model. Each fact carries +table then holds one question at a time, as in a run with one worker. With +`--memory-ranking fused`, BM25 ranks over the whole table before the scope filter, so the +rows of a second question in a table can change the ranking. An extractor variant names a prompt and a model. Each fact carries the source `session:`, where the hash is the first 12 hex digits of the SHA-256 of the session id. The block shows the sources to the reader, and the id of an evidence session holds `answer`, so the source does not show the id. If an extractor reply reaches @@ -114,7 +114,9 @@ so a replay skips the same session. The result file counts these sessions in With `--session-episodes`, `capture` stores each session as an episode under the key `session-`, with the same hash. Each fact then cites that episode, and carries no -`session:` source. The block shows the source as `session-@1`. The source +`session:` source. The block shows the source as `session-@1`. With `--chunk-chars` as well, `capture` +stores the session as chunks under `session--`, each fact cites every chunk, and +the block shows the sources as one range, `session--0001…@1`. The source holds no run id, so a second run with another `--episodes` or `--recall-budget` replays `capture` from the cache. The result file lists the episode keys of each block in `episodes_in_block`, and the SHA-256 of each block in `block_sha256`. The result file is @@ -134,14 +136,15 @@ uv run --group bench --env-file .env python -m bench.e2e --run --arms memory --e ``` With `--keep`, each question runs over a new local database of its own, and the database -stays after the run. One database holds one question, because BM25 ranks the whole table -before the namespace filter, and the memory of a second question changes the block. The +stays after the run. One database holds one question. With `--memory-ranking fused`, BM25 +ranks the whole table before the namespace filter, so the memory of a second question +would change the block. The name of the database is `hotmemory-kept------`: | Part | What it holds | |---|---| | `` | The variant name, and 6 hex digits of the SHA-256 of its model and prompt. | -| `` | `episodes` with `--session-episodes`, `sources` with fact sources, or `plain`. | +| `` | `episodes` with `--session-episodes`, `chunks` with `--chunk-chars` as well, `sources` with fact sources, or `plain`. | | `` | The `--memory-ranking`. | | `` | The last commit that changed `src/hotmemory`. | @@ -163,10 +166,11 @@ uv run --group bench --env-file .env python -m bench.e2e --run --smoke --arms me It runs the 18 questions of `bench/sample-smoke.json`, 3 of each type drawn from sample A with the seed `20261016`. Each worker runs its questions over a `MemoryStore` of its own, in its own process, and empties the memory of each question after it. It decides no gain: 18 -questions can be off by about 23 points, and `MemoryStore` ranks by vector only. Because -the ranking differs, `capture` shows the extractor another list of known facts than a run -on the local engine. So the first smoke run of a configuration pays for its extraction, -and a later one replays it from the cache. Pass `--workers 4` to make the first run +questions can be off by about 23 points. `MemoryStore` ranks by vector only, as +`HotdataStore` does by default. With `--memory-ranking fused`, the ranking differs, so +`capture` shows the extractor another list of known facts than a run on the local engine. +The first smoke run of a configuration pays for the extraction that the cache does not +hold, and a later one replays it from the cache. Pass `--workers 4` to make the first run faster. `--smoke` refuses `--keep`, `--reuse`, and `--sample`. Each run writes the file `bench/results/.rows.jsonl` as it goes, where `` is diff --git a/docs/benchmarks.md b/docs/benchmarks.md index 31b6b26..2726d3e 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -189,7 +189,7 @@ sample. | Reader | `gpt-4o-2024-08-06`, temperature 0, up to 800 output tokens, the chain-of-thought prompt of the reference code | | Judge | `gpt-4o-2024-08-06`, temperature 0, the prompt of the reference code for each question type | | Extractor of the memory arm | `gpt-4o-mini-2024-07-18`, with the `baseline` prompt in `bench/prompts.py`, except in the section on extractor variants | -| Driver | `HotdataStore` with the fused ranking, on the local stack | +| Driver | `HotdataStore` on the local stack, with the fused ranking up to phase 9. Each phase 10 section names its ranking. | The arms: @@ -519,7 +519,7 @@ cache miss after 3 questions, and a `--run --resume` ran the other 51. | Ranking | Correct of 54 | Reader tokens, median | Episodes in the block, mean | Facts written | Cost | File | |---|---|---|---|---|---|---| | `fused` | 45 | 12,849 | 4.06 | 13,666 | $0 | `e2e-sample-a-p10-reuse-fused-search-64k.json` | -| `vector` | 46 | 13,406 | 4.48 | 14,322 | $2.17 | `e2e-sample-a-p10-keep-vector-search-64k.json` | +| `vector` | 46 | 13,405 | 4.48 | 14,322 | $2.17 | `e2e-sample-a-p10-keep-vector-search-64k.json` | The reader tokens, episodes, and facts of the fused row are those of its `--keep` run, `e2e-sample-a-p9-keep-search-64k.json`. The two rankings differ on 3 questions. `vector` @@ -536,28 +536,28 @@ with each chunk size at three budgets, on sample A, with the vector ranking, `--extractor detail`, and 4 workers. Each chunk size was one run with `--keep` at 8,000 characters, and each larger budget was a run with `--reuse`. Observed on 2026-10-18, with `src/hotmemory` at `12339f8`. The reader tokens are the median over the 54 -questions, and the chunks are the mean count of chunks in the block. +questions, rounded down, and the chunks are the mean count of chunks in the block. | Chunk size | Budget | Correct of 54 | Reader tokens | Chunks in the block | Cost | File | |---|---|---|---|---|---|---| -| 1,000 | 8,000 | 41 | 2,471 | 3.1 | $0.55 | `e2e-sample-a-p10-keep-chunks1000-8k.json` | +| 1,000 | 8,000 | 41 | 2,470 | 3.1 | $0.55 | `e2e-sample-a-p10-keep-chunks1000-8k.json` | | 1,000 | 16,000 | 48 | 4,336 | 12.4 | $0.81 | `e2e-sample-a-p10-reuse-chunks1000-16k.json` | -| 1,000 | 32,000 | 49 | 7,832 | 29.9 | $1.27 | `e2e-sample-a-p10-reuse-chunks1000-32k.json` | +| 1,000 | 32,000 | 49 | 7,831 | 29.9 | $1.27 | `e2e-sample-a-p10-reuse-chunks1000-32k.json` | | 2,000 | 8,000 | 36 | 2,327 | 1.4 | $0.56 | `e2e-sample-a-p10-keep-chunks2000-8k.json` | | 2,000 | 16,000 | 46 | 4,171 | 6.2 | $0.78 | `e2e-sample-a-p10-reuse-chunks2000-16k.json` | -| 2,000 | 32,000 | 48 | 7,717 | 15.7 | $1.26 | `e2e-sample-a-p10-reuse-chunks2000-32k.json` | +| 2,000 | 32,000 | 48 | 7,716 | 15.7 | $1.26 | `e2e-sample-a-p10-reuse-chunks2000-32k.json` | | 4,000 | 8,000 | 31 | 1,878 | 0.2 | $0.49 | `e2e-sample-a-p10-keep-chunks4000-8k.json` | -| 4,000 | 16,000 | 44 | 3,780 | 2.6 | $0.73 | `e2e-sample-a-p10-reuse-chunks4000-16k.json` | +| 4,000 | 16,000 | 44 | 3,779 | 2.6 | $0.73 | `e2e-sample-a-p10-reuse-chunks4000-16k.json` | | 4,000 | 32,000 | 46 | 7,509 | 7.4 | $1.21 | `e2e-sample-a-p10-reuse-chunks4000-32k.json` | The runs of phase 8 with whole sessions as episodes, for comparison: | Run | Correct of 54 | Reader tokens | File | |---|---|---|---| -| `search`, 16,000 | 34 | 1,739 | `e2e-sample-a-p8-search-16k.json` | -| `search`, 32,000 | 43 | 5,716 | `e2e-sample-a-p8-search-32k.json` | +| `search`, 16,000 | 34 | 1,738 | `e2e-sample-a-p8-search-16k.json` | +| `search`, 32,000 | 43 | 5,715 | `e2e-sample-a-p8-search-32k.json` | | `search`, 64,000 | 45 | 12,849 | `e2e-sample-a-p8-search-64k.json` | -| `search`, 64,000, vector ranking, phase 10 | 46 | 13,406 | `e2e-sample-a-p10-keep-vector-search-64k.json` | +| `search`, 64,000, vector ranking, phase 10 | 46 | 13,405 | `e2e-sample-a-p10-keep-vector-search-64k.json` | Chunks of 1,000 characters at 16,000 scored 48 with 4,336 reader tokens. The best whole-session run scored 45 with 12,849 tokens. At 8,000 characters, the facts take most of the budget, @@ -566,8 +566,10 @@ block held 29.9 chunks on average, so `k`, 30, and not the budget, limited the e The block of `recall` shows a run of chunk sources as one range, as [docs/contracts.md](contracts.md) gives. A fact line is still a little longer than with one -episode, so the list of known facts that the extractor reads can hold one fact less. Thus -part of each keep run missed the cache of the earlier vector run and paid for extraction. +episode, so the list of known facts that the extractor reads can hold one fact less. The +keep runs of chunks of 1,000 and 4,000 made 108 model calls each, the 54 reader calls and +the 54 judge calls, so every extractor reply came from the cache. The keep run of chunks of +2,000 made 168 calls, so 60 extractor prompts missed the cache. The smoke run with chunks of 2,000 at 64,000 scored 16 of 18, as in phase 9, for $0.71, `e2e-smoke-p10-smoke-chunks2000.json`. @@ -583,7 +585,7 @@ Observed on 2026-10-18, with `src/hotmemory` at `12339f8`. | A | chunks of 1,000, 16,000 | 48 | 4,336 | $0 | `e2e-sample-a-p10-keep-chunks1000-16k.json` | | A | phase 8, `search`, 64,000 | 45 | 12,849 | | `e2e-sample-a-p8-search-64k.json` | | B | chunks of 1,000, 16,000 | 46 | 4,360 | $2.05 | `e2e-sample-b-p10-keep-chunks1000-16k.json` | -| B | phase 8, `search`, 64,000 | 45 | 12,677 | | `e2e-sample-b-p8-search-64k.json` | +| B | phase 8, `search`, 64,000 | 45 | 12,676 | | `e2e-sample-b-p8-search-64k.json` | The diagnosis read the kept memory of each run. The phase 8 rows are those of the diagnoses of `search` at 64,000. diff --git a/docs/contracts.md b/docs/contracts.md index 8e51ffd..fe7b985 100644 --- a/docs/contracts.md +++ b/docs/contracts.md @@ -255,7 +255,8 @@ If the engine refuses a load with `409 RESOURCE_LOCKED`, the driver tries it aga 8 attempts in all. The wait starts at 0.25 seconds and doubles to at most 4 seconds. After the last attempt, the driver raises the error. -A `search` with query text runs one SQL query over both record tables and `cue_v1`. If +A `search` with query text runs one SQL query over both record tables, and over `cue_v1` +with `ranking="fused"`. If the filter sets `kind`, the query reads only the record tables of the kinds that it names. A filter without `episode` reads only `memory_v1`. @@ -420,8 +421,8 @@ session that is not Markdown. A blank line separates two turns of such a text. The extractor reads the whole text once, as it does without `chunk_chars`. The extractor does not say which chunk a fact came from. Thus each fact gets the id of every chunk added -to its `sources`, in chunk order. The block shows those ids as one range, so a fact line -stays short. The extractor also gets the records that `recall` returns for the text, and a +to its `sources`, in chunk order. While the chunks share a revision, the block shows those +ids as one range, so a fact line stays short. The extractor also gets the records that `recall` returns for the text, and a longer line leaves room for fewer of them. The facts are written first, and then one flush writes the chunks. If every chunk is current with the same content, `capture` calls no extractor, writes nothing, and returns no id. If a diff --git a/docs/guarantees.md b/docs/guarantees.md index b2bc49d..624af0f 100644 --- a/docs/guarantees.md +++ b/docs/guarantees.md @@ -17,7 +17,7 @@ marked [measured, local] was observed against the local stack in [local.md](loca The Test column names the tests that prove the guarantee. A test in `tests/test_conformance.py` proves a row of the storage contract against every driver. A test in `tests/test_contract.py` proves a row of the memory contract against every driver. -A test in `tests/test_chunks.py` proves how `load` cuts a document, with no driver. +A test in `tests/test_chunks.py` proves how `load` cuts a document and how `capture` cuts a text into chunks, with no driver. A test in `tests/test_hotdata.py`, `tests/test_hotdata_retry.py`, or `tests/test_oracle.py` proves a row that only the Hotdata driver can show. `tests/test_skill.py` runs the commands of the skill file. If a named test does not exist, `tests/test_ledger.py` fails. @@ -65,7 +65,7 @@ A row that no test proves yet names the phase that will prove it. | Do the two drivers recall the same block with chunks? | Yes. After the same `capture` calls with `chunk_chars`, `MemoryStore` and `HotdataStore` with its default ranking return the same records and the same block, with `episodes="search"` and with `episodes="cited"`. A search over episodes ranks each chunk on its own. | answered [measured, local] 2026-10-16 | `test_both_drivers_recall_the_same_block_with_chunks`, `test_recall_search_returns_the_chunks_that_match` | | Which ranking does the search inside `capture` use? | The ranking of the store, as `recall` does, because `capture` calls `recall`. The default of `HotdataStore` is `ranking="vector"`, the order of `MemoryStore`. | answered [measured, local] 2026-10-16 | `test_capture_searches_with_the_ranking_of_recall`, `test_the_default_ranking_is_the_vector_ranking` | | Which episodes does `recall` add with `episodes`? | With `"cited"`, the episodes that the facts of the block name in `sources`, in order, once each, and only inside the allowed scopes. With `"search"`, the top k episodes for the query. Neither value returns an episode past its `forget_after`. `"cited"` keeps a superseded revision, because the fact cites that revision. `as_of` filters them as it filters facts. They follow the facts, whole, and the block stops before the first episode that does not fit. An episode never takes the place of a fact. | answered | `test_recall_cited_adds_the_episodes_of_the_facts`, `test_recall_cited_names_each_episode_once`, `test_recall_cited_skips_an_episode_outside_the_scopes`, `test_recall_search_adds_the_top_episodes`, `test_recall_episodes_never_take_the_place_of_a_fact`, `test_recall_stops_before_the_first_episode_that_does_not_fit`, `test_recall_as_of_filters_the_episodes`, `test_recall_refuses_an_unknown_episodes_value`, `test_recall_cited_skips_an_episode_past_its_forget_after`, `test_recall_cited_keeps_a_superseded_revision` | -| How does the block show a source? | In full, except the id of a record in the namespace of the line, which shows as `@`. The stored source keeps the full id. | answered | `test_recall_shows_a_source_in_its_namespace_as_key_and_revision` | +| How does the block show a source? | In full, except the id of a record in the namespace of the line, which shows as `@`. A run of chunk ids shows as one range, as the row on chunks gives. The stored source keeps the full id. | answered | `test_recall_shows_a_source_in_its_namespace_as_key_and_revision` | | Can a `put` move a key between `episode` and another kind? | No. A key holds episodes only, or holds no episode at all. A `put` that crosses that line raises an error and writes nothing. A key can change between `fact`, `profile`, and `procedure`. | answered | `test_key_cannot_cross_the_episode_line` | | What does a `put` with `close_previous` write? | The new revision, and the previous row with `superseded_by`, `valid_until` set to the new `valid_from`, and `expired_at` set to the clock's time. A missing `valid_from`, or one earlier than the previous `valid_from`, raises an error, and the put or the whole flush writes nothing. Without `close_previous`, the previous span stays open. | answered | `test_close_previous_closes_the_current_revision`, `test_close_previous_refuses_a_span_it_cannot_close`, `test_close_previous_on_a_new_key_or_same_content`, `test_put_without_close_previous_leaves_the_span_open` | | Can a record have empty content? | No. `put` and a buffered `put` refuse content that is empty after normalization, and write nothing. | answered | `test_put_refuses_empty_content` | diff --git a/docs/internal/architecture.md b/docs/internal/architecture.md index b5e672d..5622bb1 100644 --- a/docs/internal/architecture.md +++ b/docs/internal/architecture.md @@ -423,8 +423,8 @@ Each ranking applies the same SQL condition, built from these parts: ### The fused ranking -`ranking="vector"` is the default, and it ranks by the content vector alone. With -`ranking="fused"`, the search runs three rankings and fuses them by reciprocal rank +`ranking="fused"` is an option. The default, `ranking="vector"`, ranks by the content +vector alone. With `ranking="fused"`, the search runs three rankings and fuses them by reciprocal rank fusion. Reciprocal rank fusion adds `1 / (60 + rank)` from each ranking that holds a row. Each ranking keeps its top rows to a depth of `max(100, 10 * k)`. diff --git a/docs/internal/plan.md b/docs/internal/plan.md index 24df566..847f4fb 100644 --- a/docs/internal/plan.md +++ b/docs/internal/plan.md @@ -148,9 +148,12 @@ Tasks: and stop for the owner's agreement. The ranking: the vector ranking, or a rebuilt fusion that ranks BM25 inside the scope or fuses two arms with no weights. The chunks, as a starting point: `capture` with `episode` takes `chunk_chars`, cuts the text at - blank lines as `load` does, and stores chunk n under `-`. The extractor - still reads the whole text, so its prompt and its cache do not change. The proposal - names which chunks a fact cites, and how `"cited"` uses them. + blank lines outside fences, without the headings and prefixes of `load`, and stores + chunk n under `-`. The extractor still reads the whole text once. Each fact + cites every chunk, and the block shows a run of chunk sources as one range. A fact line + is still a little longer, so the known facts that the extractor reads can change, and + part of the extraction can miss the cache. The proposal names which chunks a fact + cites, and how `"cited"` uses them. 4. Build the agreed change for both drivers, with `CHANGELOG.md`, `docs/contracts.md`, `docs/guarantees.md`, and the skill commands. If the default ranking changes, the oracle test and the benchmark runner follow.