From e8f4ff55e0929c2d8f75580373f4a5677a5e3c8e Mon Sep 17 00:00:00 2001 From: Rohan Dsouza Date: Tue, 13 Oct 2026 12:12:34 +0530 Subject: [PATCH 1/7] Set up releases to PyPI Add the release procedure from hotdata-langchain, adapted to uv and make verify. - scripts/release.sh prepares a release pull request and pushes the tag. - scripts/changelog.py moves the Unreleased notes into a version section, prints the notes of a version, and checks that a version change has a section. - publish.yml builds on a vX.Y.Z tag, publishes through PyPI trusted publishing in the pypi environment, then creates the GitHub Release. - check-release.yml runs the changelog check on a pull request. - verify.yml runs make verify on Python 3.11 to 3.14, and make floor runs the tests at the lowest allowed dependency versions. The sdist now holds src/hotmemory, README.md, CHANGELOG.md, and LICENSE only. pyproject.toml gains classifiers, project URLs, and an SPDX license. The hotdata extra needs pyarrow 16.0. At the floor, pyarrow 14.0.0 installs next to numpy 2 and fails to import, and pyarrow 16.0.0 passes the offline tests. The README links to repository files with absolute links, so they work on PyPI, and check_links.py checks those links too. --- .github/workflows/check-release.yml | 21 ++++ .github/workflows/publish.yml | 76 +++++++++++++++ .github/workflows/verify.yml | 14 ++- AGENTS.md | 3 + CHANGELOG.md | 8 +- CONTRIBUTING.md | 8 +- Makefile | 10 +- README.md | 59 ++++++++---- RELEASING.md | 92 ++++++++++++++++++ pyproject.toml | 32 ++++++- scripts/changelog.py | 105 ++++++++++++++++++++ scripts/check_links.py | 25 +++-- scripts/release.sh | 142 ++++++++++++++++++++++++++++ tests/test_changelog.py | 69 ++++++++++++++ uv.lock | 2 +- 15 files changed, 634 insertions(+), 32 deletions(-) create mode 100644 .github/workflows/check-release.yml create mode 100644 .github/workflows/publish.yml create mode 100644 RELEASING.md create mode 100644 scripts/changelog.py create mode 100755 scripts/release.sh create mode 100644 tests/test_changelog.py diff --git a/.github/workflows/check-release.yml b/.github/workflows/check-release.yml new file mode 100644 index 0000000..5efeae5 --- /dev/null +++ b/.github/workflows/check-release.yml @@ -0,0 +1,21 @@ +name: check-release + +on: + pull_request: + paths: + - pyproject.toml + - CHANGELOG.md + +permissions: + contents: read + +jobs: + changelog: + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + fetch-depth: 0 + - name: Check that a version change has a changelog section + run: python3 scripts/changelog.py check "origin/$GITHUB_BASE_REF" diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..42a67b4 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,76 @@ +name: publish + +on: + push: + tags: + - "v[0-9]*" + +concurrency: + group: publish-${{ github.ref_name }} + cancel-in-progress: false + +permissions: + contents: read + +jobs: + build: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + - uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v6 + with: + python-version: "3.12" + - name: Check that the tag matches the project version + run: | + version=$(python3 -c "import tomllib; print(tomllib.load(open('pyproject.toml', 'rb'))['project']['version'])") + if [[ "$GITHUB_REF_NAME" != "v$version" ]]; then + echo "The tag $GITHUB_REF_NAME does not match version $version in pyproject.toml" >&2 + exit 1 + fi + - name: Check that CHANGELOG.md has a section for the version + run: python3 scripts/changelog.py notes "${GITHUB_REF_NAME#v}" > /dev/null + - run: uv build + - run: uvx twine check --strict dist/* + - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5 + with: + name: dist + path: dist/ + + pypi: + needs: build + runs-on: ubuntu-latest + timeout-minutes: 10 + environment: + name: pypi + url: https://pypi.org/p/hotmemory + permissions: + id-token: write + steps: + - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5 + with: + name: dist + path: dist/ + - uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 + + github-release: + needs: pypi + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: write + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5 + with: + name: dist + path: dist/ + - name: Create the GitHub Release with the changelog notes + env: + GH_TOKEN: ${{ github.token }} + run: | + python3 scripts/changelog.py notes "${GITHUB_REF_NAME#v}" > notes.md + gh release create "$GITHUB_REF_NAME" dist/* \ + --title "hotmemory ${GITHUB_REF_NAME#v}" \ + --notes-file notes.md \ + --verify-tag diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 4d9d062..93d1b95 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -12,13 +12,25 @@ jobs: verify: runs-on: ubuntu-latest timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + python-version: ["3.11", "3.12", "3.13", "3.14"] steps: - uses: actions/checkout@v5 - uses: astral-sh/setup-uv@v6 with: - python-version: "3.11" + python-version: ${{ matrix.python-version }} - run: make verify + floor: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v5 + - uses: astral-sh/setup-uv@v6 + - run: make floor + integration: runs-on: ubuntu-latest timeout-minutes: 20 diff --git a/AGENTS.md b/AGENTS.md index 8fd7062..c0db6a5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -8,6 +8,7 @@ rules, and points to the files that hold everything else. ```sh make verify # the full check; run it before each commit make integration # the hotdata tests and the conformance suite; needs make local-up +make floor # the offline tests at the lowest allowed dependency versions make local-up # start the local RuntimeDB stack (Postgres, RustFS, engine) make local-down # stop it and delete its data cp .env.template .env # then fill in .env; it is ignored by git @@ -28,6 +29,8 @@ uv run --env-file .env scripts/measure_local.py --cloud # M4 to M6 in the clou - [docs/internal/brief.md](docs/internal/brief.md): the design and its reasons. - [CONTRIBUTING.md](CONTRIBUTING.md): the checks inside `make verify` and the test rules. - [CHANGELOG.md](CHANGELOG.md): each change to a public surface. +- [RELEASING.md](RELEASING.md): how to publish a version to PyPI. Only the repository owner + publishes. - `src/hotmemory/`: the library. `tests/test_conformance.py` is the conformance suite. - [docs/benchmarks.md](docs/benchmarks.md): the benchmark numbers and their configuration. - `bench/`: the LongMemEval-S runner, outside the package. [bench/README.md](bench/README.md) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1299d4e..a508c89 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,11 @@ names in `hotmemory.__all__`, the method set of `Store`, the fields and field ty `Record` for each schema version, and the filter keys of `list` and `search`. A test guards each one, and its failure message points to this file. -## Unreleased +New entries go under `## [Unreleased]`. A release moves them into a section named +`## [X.Y.Z] - YYYY-MM-DD`, and that section becomes the notes of the GitHub Release. +[RELEASING.md](RELEASING.md) gives the procedure. + +## [Unreleased] ### Added @@ -37,6 +41,8 @@ each one, and its failure message points to this file. ### Changed +- The `hotdata` extra needs `pyarrow` 16.0 or later. With `pyarrow` 14 and `numpy` 2, + `import pyarrow` fails. - `Store` gains `sweep`, which deletes every revision of each key whose current revision is past `forget_after`, and returns the deleted ids. `MemoryStore` and `HotdataStore` implement it. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index dff8b80..7bc2db9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -35,6 +35,7 @@ passes in CI. It has no tiers, because the full check takes less than five secon | Target | What it does | |---|---| | `make verify` | Runs every check above. | +| `make floor` | Runs the offline tests on Python 3.11 with the lowest version of each dependency that `pyproject.toml` allows. It needs the network to install them. | | `make integration` | Runs the tests marked `hotdata` against the local stack. Start the stack first with `make local-up`. To use another engine, set `HOTMEMORY_TEST_URL`. | | `make local-up` | Starts the local RuntimeDB stack: Postgres, RustFS, and the engine. [docs/local.md](docs/local.md) tells you how to point the library at it. | | `make local-down` | Stops the stack and deletes its data. | @@ -76,8 +77,11 @@ model. The tests pass a fixed clock and a fake embedder to the store, from offline. `make integration` sets it to the local stack. A run provisions one database with a new name, empties its tables after each test, and deletes it at the end. They are the only tests that use the network. -- CI runs two jobs on each pull request. `verify` runs `make verify`. `integration` starts - the local stack with `make local-up` and runs `make integration`. +- CI runs three jobs on each pull request. `verify` runs `make verify` on Python 3.11, + 3.12, 3.13, and 3.14. `floor` runs `make floor`. `integration` starts the local stack + with `make local-up` and runs `make integration`. If a pull request changes + `pyproject.toml` or `CHANGELOG.md`, `check-release` also runs. [RELEASING.md](RELEASING.md) + describes it. - No test calls a model. The tests pass a fake embedder. `capture` and `load` take an extractor callable, and the tests pass a fake extractor that returns fixed facts. - There is no coverage gate, no mutation-testing gate, and no report generator. The output diff --git a/Makefile b/Makefile index c1ef29e..db49845 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ export LOCAL_PORT ?= 3000 STORAGE_URL := http://127.0.0.1:9000 -.PHONY: verify integration local-up local-down local-pull +.PHONY: verify floor integration local-up local-down local-pull verify: uv run --group dev ruff check . @@ -12,6 +12,14 @@ verify: uv run --group dev pytest --disable-socket -q uv run --no-project python scripts/check_links.py +floor: + @requirements="$$(mktemp)"; \ + uv pip compile pyproject.toml --all-extras --resolution lowest-direct \ + --python-version 3.11 --quiet --output-file "$$requirements" && \ + uv run --isolated --no-project --python 3.11 --with-requirements "$$requirements" \ + --with-editable . --with pytest --with pytest-socket --with tiktoken \ + pytest --disable-socket -q -p no:cacheprovider + integration: HOTMEMORY_TEST_URL=$${HOTMEMORY_TEST_URL:-http://localhost:$(LOCAL_PORT)} \ uv run --group dev pytest -q -m hotdata diff --git a/README.md b/README.md index 1fb8aec..d13c1ce 100644 --- a/README.md +++ b/README.md @@ -8,10 +8,11 @@ Hotdata table, so an agent can join its memory to its own data in one SQL query. the memory systems that we surveyed stores memory as typed columns in the same engine as the data of the consumer. -Status: version 0.0.0, not published. The storage contract exists in Python with two -drivers. `MemoryStore` runs in process memory. `HotdataStore` keeps records in one Hotdata -managed database. The memory contract, `Memory`, runs over either driver. -[docs/internal/roadmap.md](docs/internal/roadmap.md) lists the phases. +Status: alpha. A minor version before 1.0 can change a public surface, and [CHANGELOG.md] +lists each change. The storage contract exists in Python with two drivers. `MemoryStore` +runs in process memory. `HotdataStore` keeps records in one Hotdata managed database. The +memory contract, `Memory`, runs over either driver. [docs/internal/roadmap.md] lists the +phases. The library has two layers: @@ -31,6 +32,21 @@ of the consumer. There is no hotmemory server. The first consumer is an incident investigator built on Hotdata. The library is not specific to it. A plain Python agent, a LangGraph agent, or any process with a Hotdata API key can use it. +## Install + +hotmemory needs Python 3.11 or later. The core package has no dependencies: + +```sh +pip install hotmemory +``` + +Each driver and embedder that calls a service is in an extra. The `hotdata` extra adds +`HotdataStore`. The `openai` extra adds `OpenAIEmbedder`. + +```sh +pip install "hotmemory[hotdata,openai]" +``` + ## Try the storage contract `MemoryStore` needs an embedder for a search with query text. An embedder is a callable @@ -103,7 +119,7 @@ episode = memory.store.get(("team", "alerts"), "disk-incident-0001") print(episode.content.splitlines()[0]) # # Disk incident ``` -`skills/hotmemory/SKILL.md` gives an agent the same operations as commands. +[skills/hotmemory/SKILL.md] gives an agent the same operations as commands. ## Store memory in Hotdata @@ -125,14 +141,14 @@ store.put(("team", "alerts"), "disk", kind="fact", content="The disk fills at ni print([hit.record.id for hit in store.search("disk", [("team",)])]) ``` -One process writes to a database. [docs/contracts.md](docs/contracts.md) gives the tables, -the retrieval query, and the rules of the driver. [docs/local.md](docs/local.md) tells you +One process writes to a database. [docs/contracts.md] gives the tables, +the retrieval query, and the rules of the driver. [docs/local.md] tells you how to run it against a local engine. ## Benchmark hotmemory ran LongMemEval-S, a public benchmark of 500 questions over long chat histories, -on 2026-10-10. [docs/benchmarks.md](docs/benchmarks.md) gives every number with its +on 2026-10-10. [docs/benchmarks.md] gives every number with its configuration. On all 500 questions, on 2026-10-22, the memory arm scored 439 of 500 (87.8%). It ran @@ -172,17 +188,28 @@ p50 with the vector ranking, and 998 ms with the fused ranking. ## Documents -- [docs/contracts.md](docs/contracts.md): the record, the storage operations, the memory +- [docs/contracts.md]: the record, the storage operations, the memory operations, and the platform facts behind them. -- [docs/guarantees.md](docs/guarantees.md): each behavior that a consumer can rely on, its +- [docs/guarantees.md]: each behavior that a consumer can rely on, its state, and its proof. -- [docs/local.md](docs/local.md): how to run the library against a local RuntimeDB +- [docs/local.md]: how to run the library against a local RuntimeDB container. -- [docs/benchmarks.md](docs/benchmarks.md): the numbers of hotmemory on LongMemEval-S, - each with its configuration. [bench/README.md](bench/README.md) tells you how to run it. -- [CONTRIBUTING.md](CONTRIBUTING.md): the one command that checks a change, and the rules +- [docs/benchmarks.md]: the numbers of hotmemory on LongMemEval-S, + each with its configuration. [bench/README.md] tells you how to run it. +- [CONTRIBUTING.md]: the one command that checks a change, and the rules of the test suite. -- [CHANGELOG.md](CHANGELOG.md): each change to a public surface. -- [docs/internal/](docs/internal/brief.md): the design brief, the survey, the roadmap, the +- [CHANGELOG.md]: each change to a public surface. +- [docs/internal/][docs/internal/brief.md]: the design brief, the survey, the roadmap, the plan for the open workstream, the research that chose the benchmarks, and the architecture: how each operation runs on the Hotdata platform. + +[CHANGELOG.md]: https://github.com/hotdata-dev/hotmemory/blob/main/CHANGELOG.md +[docs/internal/roadmap.md]: https://github.com/hotdata-dev/hotmemory/blob/main/docs/internal/roadmap.md +[docs/contracts.md]: https://github.com/hotdata-dev/hotmemory/blob/main/docs/contracts.md +[docs/local.md]: https://github.com/hotdata-dev/hotmemory/blob/main/docs/local.md +[docs/benchmarks.md]: https://github.com/hotdata-dev/hotmemory/blob/main/docs/benchmarks.md +[docs/guarantees.md]: https://github.com/hotdata-dev/hotmemory/blob/main/docs/guarantees.md +[bench/README.md]: https://github.com/hotdata-dev/hotmemory/blob/main/bench/README.md +[CONTRIBUTING.md]: https://github.com/hotdata-dev/hotmemory/blob/main/CONTRIBUTING.md +[docs/internal/brief.md]: https://github.com/hotdata-dev/hotmemory/blob/main/docs/internal/brief.md +[skills/hotmemory/SKILL.md]: https://github.com/hotdata-dev/hotmemory/blob/main/skills/hotmemory/SKILL.md diff --git a/RELEASING.md b/RELEASING.md new file mode 100644 index 0000000..955d385 --- /dev/null +++ b/RELEASING.md @@ -0,0 +1,92 @@ +# Releasing + +A release publishes one version of hotmemory to PyPI and creates a GitHub Release. Use +`./scripts/release.sh` for each release. Do not change the version, push a tag, or create a +GitHub Release by hand. + +## Prerequisites + +- [uv](https://docs.astral.sh/uv/) runs the scripts and updates `uv.lock`. +- The [GitHub CLI](https://cli.github.com/), `gh`, opens the release pull request. Log in + with `gh auth login`. +- You can push to `main` and push tags to this repository. + +The publish uses PyPI trusted publishing. PyPI accepts the upload from the workflow +`publish.yml` in the GitHub environment `pypi`, and no API token exists. The section +[One-time setup](#one-time-setup) tells you how to configure it. + +## Release a version + +1. Make sure that `CHANGELOG.md` has the notes of the release under `## [Unreleased]`. +2. Prepare the release pull request. Give a bump kind, or the version itself: + + ```sh + ./scripts/release.sh prepare minor # or patch, major, or 0.2.0 + ``` + + The script starts from the latest `main`. It sets the version in `pyproject.toml`, + moves the Unreleased notes into a section for the version, updates `uv.lock`, and runs + `make verify`. Then it pushes the branch `release/vX.Y.Z` and opens the pull request. +3. Merge the pull request after CI passes. +4. Publish from a clean checkout: + + ```sh + ./scripts/release.sh publish + ``` + + The script tags the version on the latest `main` and pushes the tag `vX.Y.Z`. + +## What the tag starts + +The tag starts the workflow `publish.yml`. It runs three jobs in order: + +1. `build` makes sure that the tag matches the version in `pyproject.toml` and that + `CHANGELOG.md` has a section for the version. Then it builds the wheel and the sdist with + `uv build` and checks them with `twine check --strict`. +2. `pypi` uploads the wheel and the sdist to PyPI. It runs in the environment `pypi`. +3. `github-release` creates the GitHub Release. The release notes are the changelog section + of the version, and the wheel and the sdist are attached. + +If `build` or `pypi` fails, no GitHub Release is created. A version that PyPI accepted +cannot be uploaded again. To fix a bad release, release a new version. + +## The checks + +- On a pull request that changes `pyproject.toml` or `CHANGELOG.md`, the workflow + `check-release.yml` runs `scripts/changelog.py check`. If the version changed and + `CHANGELOG.md` has no section for the new version, it fails. +- If the Unreleased section is empty, `release.sh prepare` stops. +- If the tag exists or the changelog section is missing, `release.sh publish` stops. +- If the tag and the version differ, the `build` job stops. + +## What the package contains + +The wheel contains `src/hotmemory` only. The sdist contains `src/hotmemory`, `README.md`, +`CHANGELOG.md`, and `LICENSE`. The benchmark runner in `bench/`, the skill in +`skills/hotmemory/`, the tests, and the documents stay in the repository. To see the +contents, run `uv build` and list the archives in `dist/`. + +The README is the project description on PyPI. A relative link does not work on PyPI, so +the README links to repository files with absolute GitHub links. `make verify` makes sure +that each of these links names a file that exists. + +## One-time setup + +An owner of the GitHub repository and the PyPI project does these steps once, before the +first release. + +1. On GitHub, open Settings, then Environments, and create the environment `pypi`. To + require an approval before each upload, add required reviewers. +2. On PyPI, open Your account, then Publishing, and add a pending trusted publisher with + these values: + + | Field | Value | + |---|---| + | PyPI project name | `hotmemory` | + | Owner | `hotdata-dev` | + | Repository name | `hotmemory` | + | Workflow name | `publish.yml` | + | Environment name | `pypi` | + + The first upload creates the project and makes the publisher permanent. +3. After the first release, add the other maintainers to the PyPI project. diff --git a/pyproject.toml b/pyproject.toml index 1e51bbb..ed09539 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,20 +4,46 @@ version = "0.0.0" description = "Agent memory as tables on Hotdata" readme = "README.md" requires-python = ">=3.11" -license = { text = "MIT" } +license = "MIT" +license-files = ["LICENSE"] +authors = [{ name = "Hotdata" }] +keywords = ["agent", "memory", "llm", "hotdata", "vector-search"] +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Topic :: Scientific/Engineering :: Artificial Intelligence", + "Topic :: Software Development :: Libraries :: Python Modules", + "Typing :: Typed", +] dependencies = [] +[project.urls] +Homepage = "https://github.com/hotdata-dev/hotmemory" +Documentation = "https://github.com/hotdata-dev/hotmemory/tree/main/docs" +Changelog = "https://github.com/hotdata-dev/hotmemory/blob/main/CHANGELOG.md" +Issues = "https://github.com/hotdata-dev/hotmemory/issues" + [project.optional-dependencies] -hotdata = ["hotdata-framework>=0.14.1", "pyarrow>=14.0"] +hotdata = ["hotdata-framework>=0.14.1", "pyarrow>=16.0"] openai = ["openai>=1.0"] [build-system] -requires = ["hatchling"] +requires = ["hatchling>=1.27"] build-backend = "hatchling.build" [tool.hatch.build.targets.wheel] packages = ["src/hotmemory"] +[tool.hatch.build.targets.sdist] +only-include = ["src/hotmemory", "CHANGELOG.md", "LICENSE", "README.md"] + [dependency-groups] dev = [ "hotmemory[hotdata,openai]", diff --git a/scripts/changelog.py b/scripts/changelog.py new file mode 100644 index 0000000..90567d9 --- /dev/null +++ b/scripts/changelog.py @@ -0,0 +1,105 @@ +"""Read and update CHANGELOG.md for a release. + +Commands: + + release VERSION DATE Move the notes under "## [Unreleased]" into a new + "## [VERSION] - DATE" section. Exits 1 when Unreleased + is empty. Does nothing when the section exists. + notes VERSION Print the body of the "## [VERSION]" section. + check [BASE] Compare the version in pyproject.toml with the one at + the git ref BASE (default origin/main). Exits 1 when + the version changed and CHANGELOG.md has no section + for the new version. +""" + +from __future__ import annotations + +import re +import subprocess +import sys +import tomllib +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +CHANGELOG = ROOT / "CHANGELOG.md" +PYPROJECT = ROOT / "pyproject.toml" + +UNRELEASED = re.compile(r"^## \[Unreleased\][ \t]*\n(.*?)(?=^## \[|\Z)", re.M | re.S) + + +def section_heading(version: str) -> re.Pattern[str]: + return re.compile(rf"^## \[{re.escape(version)}\].*$", re.M) + + +def release(text: str, version: str, date: str) -> str: + """Return `text` with the Unreleased notes moved under a new version section.""" + if section_heading(version).search(text): + return text + match = UNRELEASED.search(text) + if match is None: + raise SystemExit("CHANGELOG.md has no '## [Unreleased]' section") + body = match.group(1).strip() + if not body: + raise SystemExit("CHANGELOG.md has no notes under '## [Unreleased]'") + section = f"## [Unreleased]\n\n## [{version}] - {date}\n\n{body}\n\n" + return text[: match.start()] + section + text[match.end() :].lstrip("\n") + + +def notes(text: str, version: str) -> str: + """Return the body of the section for `version`, without its heading.""" + match = section_heading(version).search(text) + if match is None: + raise SystemExit(f"CHANGELOG.md has no '## [{version}]' section") + rest = text[match.end() :] + following = re.search(r"^## \[", rest, re.M) + body = rest[: following.start()] if following else rest + return body.strip() or f"Release {version}." + + +def project_version(pyproject: str) -> str: + version: str = tomllib.loads(pyproject)["project"]["version"] + return version + + +def check(base: str) -> str: + """Return a status line, or exit 1 when a version change has no changelog section.""" + try: + previous = subprocess.run( + ["git", "show", f"{base}:pyproject.toml"], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ).stdout + except subprocess.CalledProcessError: + return f"skip: no pyproject.toml at {base}" + old = project_version(previous) + new = project_version(PYPROJECT.read_text(encoding="utf-8")) + if old == new: + return f"version unchanged ({new})" + if not section_heading(new).search(CHANGELOG.read_text(encoding="utf-8")): + raise SystemExit( + f"pyproject.toml version changed to {new}, but CHANGELOG.md has no '## [{new}]' section" + ) + return f"release metadata ok for {new}" + + +def main(argv: list[str]) -> int: + match argv: + case ["release", version, date]: + text = CHANGELOG.read_text(encoding="utf-8") + CHANGELOG.write_text(release(text, version, date), encoding="utf-8") + case ["notes", version]: + print(notes(CHANGELOG.read_text(encoding="utf-8"), version)) + case ["check"]: + print(check("origin/main")) + case ["check", base]: + print(check(base)) + case _: + print(__doc__, file=sys.stderr) + return 2 + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/scripts/check_links.py b/scripts/check_links.py index 0f355c3..d81bf73 100644 --- a/scripts/check_links.py +++ b/scripts/check_links.py @@ -1,9 +1,11 @@ -"""Fail when a relative Markdown link points at a file that does not exist. +"""Fail when a Markdown link points at a repository file that does not exist. -Checks every Markdown file at the repository root and under docs/. A link -with a scheme (https:, mailto:) or a bare anchor is skipped. Links inside fenced -code blocks and inline code spans are ignored. Prints one line per broken link -and exits 1 when any is found. +Checks every Markdown file at the repository root and under docs/, in inline +links and in reference definitions. A relative link is resolved from the file +that holds it. A link under REPO_URL is resolved from the repository root. Any +other link with a scheme (https:, mailto:) or a bare anchor is skipped. Links +inside fenced code blocks and inline code spans are ignored. Prints one line per +broken link and exits 1 when any is found. """ from __future__ import annotations @@ -17,6 +19,8 @@ FENCE = re.compile(r"^\s*(```|~~~)") CODE_SPAN = re.compile(r"`[^`]*`") LINK = re.compile(r"!?\[[^\]]*\]\(\s*]+)>?(?:\s+\"[^\"]*\")?\s*\)") +REFERENCE = re.compile(r"^\s{0,3}\[[^\]]+\]:\s*]+)>?") +REPO_URL = "https://github.com/hotdata-dev/hotmemory/blob/main/" SCHEME = re.compile(r"^[a-zA-Z][a-zA-Z0-9+.-]*:") @@ -37,16 +41,23 @@ def links(path: Path) -> list[tuple[int, str]]: continue for match in LINK.finditer(CODE_SPAN.sub("", line)): found.append((number, match.group(1))) + reference = REFERENCE.match(line) + if reference: + found.append((number, reference.group(1))) return found def broken(path: Path) -> list[str]: problems: list[str] = [] for number, target in links(path): - if SCHEME.match(target) or target.startswith("#"): + if target.startswith(REPO_URL): + base, target = ROOT, target.removeprefix(REPO_URL) + elif SCHEME.match(target) or target.startswith("#"): continue + else: + base = path.parent file_part = target.split("#", 1)[0] - if not (path.parent / file_part).exists(): + if not (base / file_part).exists(): problems.append(f"{path.relative_to(ROOT)}:{number}: broken link {target}") return problems diff --git a/scripts/release.sh b/scripts/release.sh new file mode 100755 index 0000000..62ea207 --- /dev/null +++ b/scripts/release.sh @@ -0,0 +1,142 @@ +#!/usr/bin/env bash +# Prepare and publish a release. RELEASING.md describes the procedure. +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT" + +die() { echo "error: $*" >&2; exit 1; } +need() { command -v "$1" >/dev/null 2>&1 || die "$1 is required"; } + +# A Python with tomllib (3.11 or later), from uv, outside the project environment. +py() { uv run --quiet --no-project --python '>=3.11' python "$@"; } + +usage() { + cat <<'EOF' +Usage: + ./scripts/release.sh prepare [patch|minor|major|X.Y.Z] + ./scripts/release.sh publish + + prepare Bump the version, move the Unreleased notes in CHANGELOG.md into a + section for the new version, run make verify, and open a release PR. + publish Tag the version on main and push the tag. The tag starts the PyPI + publish and the GitHub Release. +EOF +} + +get_field() { + py - "$1" <<'PY' +import sys, tomllib +from pathlib import Path +print(tomllib.loads(Path("pyproject.toml").read_text())["project"][sys.argv[1]]) +PY +} + +set_version() { + py - "$1" <<'PY' +import re, sys +from pathlib import Path +path = Path("pyproject.toml") +text, n = re.subn(r'(?m)^version = "[^"]+"', f'version = "{sys.argv[1]}"', path.read_text(), count=1) +if n != 1: + raise SystemExit("could not update the version in pyproject.toml") +path.write_text(text) +PY +} + +bump_version() { + py - "$1" "$2" <<'PY' +import re, sys +kind, current = sys.argv[1], sys.argv[2] +match = re.fullmatch(r"(\d+)\.(\d+)\.(\d+)", current) +if not match: + raise SystemExit(f"cannot bump {current}; pass the new version as X.Y.Z") +major, minor, patch = (int(part) for part in match.groups()) +if kind == "patch": + patch += 1 +elif kind == "minor": + minor, patch = minor + 1, 0 +elif kind == "major": + major, minor, patch = major + 1, 0, 0 +else: + raise SystemExit(f"unknown bump kind: {kind}") +print(f"{major}.{minor}.{patch}") +PY +} + +ensure_clean() { + [[ -z "$(git status --porcelain)" ]] || die "working tree is not clean" +} + +update_main() { + git fetch origin main + git checkout main + git pull --ff-only origin main + ensure_clean +} + +cmd_prepare() { + local bump="${1:-}" + [[ -n "$bump" ]] || { usage; die "missing bump kind or version"; } + need gh + need uv + ensure_clean + update_main + + local current new branch + current="$(get_field version)" + if [[ "$bump" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + new="$bump" + else + new="$(bump_version "$bump" "$current")" + fi + [[ "$new" != "$current" ]] || die "new version ($new) equals current ($current)" + git rev-parse --verify --quiet "refs/tags/v${new}" >/dev/null && die "tag v${new} exists" + + py scripts/changelog.py release "$new" "$(date +%Y-%m-%d)" + set_version "$new" + # uv.lock records the version of this project. + uv lock + branch="release/v${new}" + git checkout -b "$branch" + make verify + + git add pyproject.toml CHANGELOG.md uv.lock + git commit -m "Release v${new}" + git push -u origin "$branch" + gh pr create --base main --head "$branch" \ + --title "Release hotmemory v${new}" \ + --body "Release hotmemory v${new}. + +- [x] Version set in \`pyproject.toml\` +- [x] \`CHANGELOG.md\` has a \`## [${new}]\` section +- [ ] CI passes + +After the merge, run \`./scripts/release.sh publish\` from a clean \`main\`." + + echo "Prepared hotmemory v${new}. Merge the PR, then run: ./scripts/release.sh publish" +} + +cmd_publish() { + need git + need uv + ensure_clean + update_main + + local version tag + version="$(get_field version)" + tag="v${version}" + git rev-parse --verify --quiet "refs/tags/${tag}" >/dev/null && die "tag ${tag} exists" + py scripts/changelog.py notes "$version" >/dev/null + + git tag -a "$tag" -m "hotmemory ${version}" + git push origin "$tag" + echo "Pushed ${tag}. CI publishes to PyPI and creates the GitHub Release." +} + +case "${1:-}" in + prepare) shift; cmd_prepare "${1:-}" ;; + publish) cmd_publish ;; + -h|--help|help|"") usage ;; + *) usage; die "unknown command: $1" ;; +esac diff --git a/tests/test_changelog.py b/tests/test_changelog.py new file mode 100644 index 0000000..c687a2e --- /dev/null +++ b/tests/test_changelog.py @@ -0,0 +1,69 @@ +import importlib.util +from pathlib import Path +from types import ModuleType + +import pytest + +ROOT = Path(__file__).resolve().parent.parent + + +def _load() -> ModuleType: + spec = importlib.util.spec_from_file_location("changelog", ROOT / "scripts" / "changelog.py") + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +changelog = _load() + +TEXT = """# Changelog + +Each change to a public surface has an entry here. + +## [Unreleased] + +### Added + +- `Store`. + +## [0.1.0] - 2026-10-01 + +### Added + +- `Record`. +""" + + +def test_release_moves_the_unreleased_notes_into_the_new_section() -> None: + result = changelog.release(TEXT, "0.2.0", "2026-10-13") + assert result == TEXT.replace( + "## [Unreleased]\n\n", "## [Unreleased]\n\n## [0.2.0] - 2026-10-13\n\n" + ) + + +def test_release_refuses_an_empty_unreleased_section() -> None: + first = changelog.release(TEXT, "0.2.0", "2026-10-13") + with pytest.raises(SystemExit, match="no notes"): + changelog.release(first, "0.3.0", "2026-10-14") + + +def test_release_leaves_an_existing_section_alone() -> None: + first = changelog.release(TEXT, "0.2.0", "2026-10-13") + assert changelog.release(first, "0.2.0", "2026-10-14") == first + + +def test_notes_return_the_body_of_one_section() -> None: + assert changelog.notes(TEXT, "0.1.0") == "### Added\n\n- `Record`." + released = changelog.release(TEXT, "0.2.0", "2026-10-13") + assert changelog.notes(released, "0.2.0") == "### Added\n\n- `Store`." + + +def test_notes_refuse_a_missing_section() -> None: + with pytest.raises(SystemExit, match=r"no '## \[9.9.9\]' section"): + changelog.notes(TEXT, "9.9.9") + + +def test_the_repository_changelog_has_an_unreleased_section() -> None: + text = (ROOT / "CHANGELOG.md").read_text(encoding="utf-8") + assert changelog.UNRELEASED.search(text) diff --git a/uv.lock b/uv.lock index 763b34f..ee59986 100644 --- a/uv.lock +++ b/uv.lock @@ -341,7 +341,7 @@ dev = [ requires-dist = [ { name = "hotdata-framework", marker = "extra == 'hotdata'", specifier = ">=0.14.1" }, { name = "openai", marker = "extra == 'openai'", specifier = ">=1.0" }, - { name = "pyarrow", marker = "extra == 'hotdata'", specifier = ">=14.0" }, + { name = "pyarrow", marker = "extra == 'hotdata'", specifier = ">=16.0" }, ] provides-extras = ["hotdata", "openai"] From 4de87a1731122f890a73601f585525d801762933 Mon Sep 17 00:00:00 2001 From: Rohan Dsouza Date: Fri, 23 Oct 2026 09:22:18 +0530 Subject: [PATCH 2/7] Prepare the first release: the changelog, the CI pins, and the privacy audit - CHANGELOG.md: the Unreleased section describes the first release as one Added list. The pre-release history stays in git. - verify.yml pins each action by commit SHA, as publish.yml does. Dependabot updates the pins and hotdata-framework each week. - The internal docs no longer name a person, the private engine branch and its flag, the engine team, the engine work in progress, or the machine that runs the benchmarks. --- .github/dependabot.yml | 12 ++++ .github/workflows/verify.yml | 14 ++-- CHANGELOG.md | 120 +++++++++++------------------------ CONTRIBUTING.md | 2 + docs/internal/benchmarks.md | 5 +- docs/internal/brief.md | 13 ++-- docs/internal/plan.md | 6 +- docs/internal/roadmap.md | 8 +-- docs/internal/survey.md | 4 +- 9 files changed, 75 insertions(+), 109 deletions(-) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..8600be4 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,12 @@ +version: 2 +updates: + - package-ecosystem: github-actions + directory: "/" + schedule: + interval: weekly + - package-ecosystem: uv + directory: "/" + schedule: + interval: weekly + allow: + - dependency-name: hotdata-framework diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 93d1b95..6606a4a 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -17,8 +17,8 @@ jobs: matrix: python-version: ["3.11", "3.12", "3.13", "3.14"] steps: - - uses: actions/checkout@v5 - - uses: astral-sh/setup-uv@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + - uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v6 with: python-version: ${{ matrix.python-version }} - run: make verify @@ -27,8 +27,8 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/checkout@v5 - - uses: astral-sh/setup-uv@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + - uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v6 - run: make floor integration: @@ -38,11 +38,11 @@ jobs: contents: read packages: read steps: - - uses: actions/checkout@v5 - - uses: astral-sh/setup-uv@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + - uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v6 with: python-version: "3.11" - - uses: docker/login-action@v3 + - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 with: registry: ghcr.io username: ${{ github.actor }} diff --git a/CHANGELOG.md b/CHANGELOG.md index a508c89..1ba9687 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,92 +11,48 @@ New entries go under `## [Unreleased]`. A release moves them into a section name ## [Unreleased] +The first release. It needs Python 3.11 or later. The core package has no dependencies. + ### Added - `Record`, the frozen dataclass for schema version 1, and `normalize`, the form that deduplication compares. - `Store`, the protocol of the storage contract, with `put`, `get`, `history`, `list`, - `search`, `delete`, `list_namespaces`, and `writer`. `Writer` is the protocol of the - buffer that `writer` returns. + `search`, `delete`, `sweep`, `list_namespaces`, and `writer`. `Writer` is the protocol of + the buffer that `writer` returns. + - Records are immutable. A put under an existing key writes a new revision. A put whose + content duplicates the current revision but brings a new source writes a new revision + with the merged sources. + - `put` takes `close_previous`, which closes the validity span of the current revision + in the same write. + - A put that moves a key between `episode` and another kind raises `ValueError`. + - `sweep` deletes every revision of each key whose current revision is past + `forget_after`, and returns the deleted ids. - `Filter` and `TimeRange`, the exact filters of `list` and `search`, and `Hit`, one search - result with its distance. -- `MemoryStore` and `MemoryWriter`, the in-process driver. + result with its distance. `Filter.kind` takes one kind or a tuple of kinds. +- `MemoryStore` and `MemoryWriter`, the in-process driver. `MemoryStore` takes `records`, + the revisions that it starts with, and `records()` returns every revision that it holds. + A writer flush in which one put raises writes nothing. - `hotmemory.hotdata`, in the `hotdata` extra, with `HotdataStore`, the driver over one - Hotdata managed database, and `HotdataWriter`. `HotdataStore.provision` creates or opens - the database by name, and `HotdataStore.open` opens it by id. `LayoutError` reports a - database whose layout differs. `ranking="vector"` ranks a search by content distance - alone. -- `Memory`, the memory contract over one `Store`, with `remember`, `recall`, - `candidates`, `supersede`, `forget`, `profile`, and `capture`. `Fact` is the frozen - dataclass of one structured fact that `remember` takes. -- `MemoryStore` takes `records`, the revisions that it starts with, and `records()` - returns every revision that it holds. -- `Memory.load`, which cuts a Markdown document into chunks by headings and fences, - stores each chunk as an episode, and runs an extractor on each chunk. Each fact names - the episode of its chunk in `sources`. -- The skill file `skills/hotmemory/SKILL.md`, with one script in - `skills/hotmemory/scripts/` for each memory operation, including `load.py`. -- `hotmemory.openai`, in the `openai` extra, with `OpenAIEmbedder`, an `Embedder` over - the OpenAI embeddings API. - -### Changed - -- The `hotdata` extra needs `pyarrow` 16.0 or later. With `pyarrow` 14 and `numpy` 2, - `import pyarrow` fails. -- `Store` gains `sweep`, which deletes every revision of each key whose current - revision is past `forget_after`, and returns the deleted ids. `MemoryStore` and - `HotdataStore` implement it. -- A `put` that moves a key between `episode` and another kind raises `ValueError` in - every driver. -- `Store.put` and `Writer.put` take `close_previous`, which closes the validity span of - the current revision in the same write. `MemoryStore` and `HotdataStore` implement it. -- A put whose content duplicates the current revision but brings a new source writes a - new revision with the merged sources. Before, it wrote nothing. -- A `MemoryStore` writer flush in which one put raises writes nothing, as in - `HotdataStore`. -- The vector rankings of a fused `HotdataStore.search` filter first and rank by a scan, so - a search in a narrow scope returns the top k rows of that scope. -- `Filter.kind` takes one kind or a tuple of kinds, and matches a record of any kind named. - The filter key stays `kind`. In `HotdataStore`, a filter without `episode` reads only - `memory_v1`. -- `Memory.recall` and `Memory.profile` read the records of kind `fact`, `profile`, and - `procedure` only. They never return an episode, and the counts of `profile` leave - episodes out. -- `HotdataStore.provision` builds no vector index by default, so each write skips the cost - of the index. `provision` and `open` take `vector_index`, an experimental option: with - it, `provision` builds the three vector indexes, and the vector rankings search through - them. A filtered search through a vector index can miss rows loaded after its build. -- `HotdataStore.provision` builds no sorted index on `created_at` by default, because a - load into a table with it takes longer as the table grows. `provision` and `open` take - `sorted_index`: with it, `provision` builds the sorted index in each record table. - Databases provisioned before this change keep the index and open as before. -- `HotdataStore` writes a table by SQL `INSERT` when its rows in the write are all new and - number at most 5, as in a `put` of a new fact. The `INSERT` writes only the ids that the - table does not hold. Other writes stay keyed loads. If the engine refuses the `INSERT`, - the store writes by load from then on. -- `HotdataStore` reads the current revisions before a write with lists of at most 500 - keys. Before, one condition for each key could overflow the stack of the engine on a - flush of about 2,000 rows. -- `HotdataStore.provision` reads every page of the database list. Before, it read the - first 100 databases, and in a larger workspace it could create a second database with - the name. -- `Memory.capture` takes `episode`, a key. With it, `capture` also stores the text as an - episode under that key, after the facts, and each fact names the episode in `sources`. - A capture of the same text under the same key writes nothing and calls no extractor. -- `Memory.recall` takes `episodes`, `"cited"` or `"search"`. With it, the block adds the - episodes that its facts cite, or the top episodes for the query, after the facts and - inside the budget. Without it, `recall` returns no episode, as before. Neither value - returns an episode past its `forget_after`. -- The block of `recall` and `profile` shows a source that is the id of a record in the - namespace of its line as `@`. Before, it showed the full id. The stored - sources do not change. -- `HotdataStore` ranks by content vector alone by default, as `MemoryStore` does. Before, - the default was `ranking="fused"`, which stays as an option. On LongMemEval-S, the BM25 - arm of the fused ranking lowered the retrieval score. -- `Memory.capture` takes `chunk_chars`, with `episode`. It cuts the text at blank lines - into chunks and stores chunk n as an episode under `-`. The extractor still - reads the whole text, and each fact names every chunk in `sources`. -- The block of `recall` and `profile` shows a run of sources in the namespace of its line, - with keys `-`, n in four digits counting up by 1, and one revision, as - `-…@`. The - stored sources do not change. + Hotdata managed database, and `HotdataWriter`. The extra needs `hotdata-framework` 0.14.1 + and `pyarrow` 16.0 or later. + - `HotdataStore.provision` opens the database with the name, or creates it. + `HotdataStore.open` opens a database by id. `LayoutError` reports a database whose + layout differs. + - A search ranks by content vector by default, as in `MemoryStore`. `ranking="fused"` + ranks by reciprocal rank fusion of BM25 and the vector rankings. + - `provision` and `open` take `vector_index`, an experimental option that builds and + uses the vector indexes, and `sorted_index`, which builds a sorted index on + `created_at`. Both are off by default. +- `Memory`, the memory contract over one `Store`, with `remember`, `recall`, `candidates`, + `supersede`, `forget`, `profile`, `capture`, and `load`. `Fact` is the frozen dataclass + of one structured fact that `remember` takes. + - `recall` and `profile` read the records of kind `fact`, `profile`, and `procedure`. + `recall` takes `episodes`, `"cited"` or `"search"`, to add episodes after the facts + inside the budget. + - `capture` takes an extractor callable. With `episode`, it also stores the text as an + episode, and with `chunk_chars`, as chunks of that size. + - `load` cuts a Markdown document into chunks by headings and fences, stores each chunk + as an episode, and runs an extractor on each chunk. +- `hotmemory.openai`, in the `openai` extra, with `OpenAIEmbedder`, an `Embedder` over the + OpenAI embeddings API. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7bc2db9..957cc25 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -82,6 +82,8 @@ model. The tests pass a fixed clock and a fake embedder to the store, from with `make local-up` and runs `make integration`. If a pull request changes `pyproject.toml` or `CHANGELOG.md`, `check-release` also runs. [RELEASING.md](RELEASING.md) describes it. +- Each workflow pins its actions by commit SHA. Dependabot opens a pull request each week + to update the pins and `hotdata-framework`. - No test calls a model. The tests pass a fake embedder. `capture` and `load` take an extractor callable, and the tests pass a fake extractor that returns fixed facts. - There is no coverage gate, no mutation-testing gate, and no report generator. The output diff --git a/docs/internal/benchmarks.md b/docs/internal/benchmarks.md index 57d84d5..4b6ca6d 100644 --- a/docs/internal/benchmarks.md +++ b/docs/internal/benchmarks.md @@ -122,8 +122,8 @@ GPT-4o is about 60%. A memory layer that scores below that adds nothing. ## 3. LoCoMo LoCoMo is ten long conversations between two speakers, with questions about them. The -data is CC BY-NC 4.0, which forbids commercial use. An internal evaluation is likely -inside that license. Nobody here asked a lawyer. +data is CC BY-NC 4.0, which forbids commercial use. A run needs a +check of the license first. | Item | Value | |---|---| @@ -442,4 +442,3 @@ the sample works. paper PDF was not read. - The LongMemEval M split was not downloaded. - Whether Hotdata accepts reads in parallel during a load was not measured. -- The license position of LoCoMo for internal use was not reviewed by a lawyer. diff --git a/docs/internal/brief.md b/docs/internal/brief.md index 9c9bf72..efe2fab 100644 --- a/docs/internal/brief.md +++ b/docs/internal/brief.md @@ -65,10 +65,8 @@ These come from measurements against Hotdata between 2026-08-27 and 2026-09-18. - A write is a file load in every released engine and in production. The caller uploads parquet and calls a keyed load in replace, append, upsert, update, or delete mode. - [measured] On RuntimeDB main since 2026-09-25, a SQL write path exists. It ran behind - the `[engine] sql_writes` flag until 2026-10-18, and runs for every instance after. - The cloud accepted INSERT on 2026-10-21. [measured, M4] INSERT and CREATE TABLE, CREATE - TABLE AS, and DROP TABLE are implemented. UPDATE and DELETE are refused. An INSERT is an append + [measured] The cloud accepted SQL INSERT on 2026-10-21. [measured, M4] INSERT and CREATE + TABLE, CREATE TABLE AS, and DROP TABLE are implemented. UPDATE and DELETE are refused. An INSERT is an append load whose source is a query: the planned rows are staged as parquet and appended through the same locked load path, so it holds the same per-table lock and maintains indexes the same way. A SQL write needs the read and write claims and can target only the database's @@ -77,10 +75,9 @@ These come from measurements against Hotdata between 2026-08-27 and 2026-09-18. seconds, ten thousand rows cost 2.5 seconds. Batching is free. The only lever is call count. [measured] - There is no conditional write. No mode takes a predicate on the existing value and none - returns the affected row. Two writers on one key is last writer wins. [measured] This is - the constraint that will change: UPDATE and DELETE over row ids are open on RuntimeDB, - and an UPDATE with a WHERE clause that returns `rows_affected` is a conditional write. - The design below assumes today's engine. + returns the affected row. Two writers on one key is last writer wins. [measured] If the + engine adds UPDATE with a WHERE clause that returns `rows_affected`, that is a + conditional write. The design below assumes today's engine. - A load holds an exclusive lock on the whole table. A second concurrent load is refused with `409 RESOURCE_LOCKED`. Distinct keys buy no concurrency. [measured] - Read after write was never stale across 30 loads. A read costs 0.3 to 0.6 seconds. A cold diff --git a/docs/internal/plan.md b/docs/internal/plan.md index d0276b7..92336f8 100644 --- a/docs/internal/plan.md +++ b/docs/internal/plan.md @@ -153,7 +153,7 @@ now that the engine accepts SQL writes. The second: cues help retrieval on facts - `provision` builds no vector index by default. `vector_index=True` is an experimental option that builds the three vector indexes and searches through them. It stays, so the index can come back when the engine updates an index as data arrives and fixes the - filtered search through it. Work on both is under way in the engine, by report. + filtered search through it. - The local stack runs the latest engine image, `sha256:75403e84eb47`, built 2026-10-19. - Task 6 changes from "capture every question into one database" to the prefilled store below, because a namespace that recall never reads cannot change an answer. @@ -167,7 +167,7 @@ now that the engine accepts SQL writes. The second: cues help retrieval on facts - LongMemEval-S runs in full, on new databases, as the comparable score. - The large store test moves to LongMemEval-M, where each question has one consistent user. BEAM comes later. -- The runs move to a Kubernetes namespace later, for a known and stable machine. +- The runs move to a stable machine later. ### What the phase found @@ -176,7 +176,7 @@ Each item was observed on 2026-10-21 unless marked. - The cloud accepts SQL `INSERT`. One row with a vector took 531 ms by `INSERT` and 1,682 ms by load. At 10 rows they were equal. At 100 rows the load was five times faster. At 1,000 rows the `INSERT` was refused with `Payload Too Large`, at 32 MB of SQL. - The engine refuses `UPDATE` and `DELETE`, and no work on them has started. + The engine refuses `UPDATE` and `DELETE`. - The engine applies the scope filter inside `bm25_search`, before its depth, locally and in the cloud. So BM25 loses no rows of a narrow scope in a shared table. The plan of phase 9 to measure that loss is void. diff --git a/docs/internal/roadmap.md b/docs/internal/roadmap.md index dc6a752..85c05a4 100644 --- a/docs/internal/roadmap.md +++ b/docs/internal/roadmap.md @@ -74,8 +74,8 @@ taken up now. `HotdataStore.provision` uses the first. It is filed as hotdata-dev/sdk-python-framework#89. Phase 11 works around it in `provision`. - The vector rankings on the vector index, from phase 2. A filtered search through the - vector index misses rows loaded after the build. The defect is reported to the engine - team. Until it is fixed, the vector rankings scan, and phase 11 measures that cost. + vector index misses rows loaded after the build. Until it is fixed, the vector rankings + scan, and phase 11 measures that cost. - SQL `UPDATE` and `DELETE`, from phase 11. The engine refuses them, so a closed revision, `forget`, and `sweep` stay on keyed loads. - The comparison with other memory layers in one harness, such as MemoryBench, from @@ -110,8 +110,8 @@ A closed phase keeps its row above with the date it closed and the pull request. later phases in four ways. The engine allows one vector index per table, so the cue vectors have their own table. The engine refuses an index on an empty table, so `provision` builds over a seed row. A filtered search through a vector index misses rows - loaded after the build, so the vector rankings filter after their fetch. The defect is - reported to the engine team, and it reproduces in production. One process writes to a + loaded after the build, so the vector rankings filter after their fetch. The defect + also occurs in the cloud. One process writes to a database until the engine has a conditional write. - Phase 3 closed on 2026-10-08 with PR #9, which closed issue #8. It changed the plan for later phases in four ways. The vector rankings of `HotdataStore` filter first and rank diff --git a/docs/internal/survey.md b/docs/internal/survey.md index 4e1cbc6..ff9cbb8 100644 --- a/docs/internal/survey.md +++ b/docs/internal/survey.md @@ -142,7 +142,7 @@ run is a later proof that the layer is a general memory and not only one consume ## 3. The fork question -Rohan asked whether forking mem0 onto a Hotdata backend gets the orchestration with fewer +The owner asked whether forking mem0 onto a Hotdata backend gets the orchestration with fewer steps. The code answers no, and a provider is not worth it either. The vector store is pluggable without a fork. `VectorStoreBase` has eleven abstract methods @@ -165,7 +165,7 @@ for the extractor contract, and nothing more. ## 4. Changes the survey proposes to the brief -Eight changes, each tied to what was read. They are proposals until Rohan accepts them. +Eight changes, each tied to what was read. They are proposals until the owner accepts them. 1. Add `expired_at` to the record beside `valid_until`. `valid_until` is when the world changed and `expired_at` is when the store learned it. `supersede` sets both. (Graphiti) From 9ffaacdcf420812b3694bd9f1735368a972c2c07 Mon Sep 17 00:00:00 2001 From: Rohan Dsouza Date: Fri, 23 Oct 2026 09:28:21 +0530 Subject: [PATCH 3/7] Keep the notes of the owner out of git, and make the architecture public - docs/internal/ is now ignored: the plan, the roadmap, the brief, the survey, and the benchmark research stay only in the owner's checkout. - docs/local.md moves there too, because the engine image is private and only the Hotdata team can start the local stack. - docs/architecture.md is public. It no longer names the engine, the storage paths of the local catalog, or what the framework source says about the network between the client and the engine. - The public docs say "the Hotdata engine" or "the local engine stack" in place of the engine's name, and no longer link to the moved files. --- .gitignore | 4 + AGENTS.md | 13 +- CONTRIBUTING.md | 8 +- README.md | 16 +- docs/{internal => }/architecture.md | 43 +-- docs/benchmarks.md | 6 +- docs/contracts.md | 10 +- docs/guarantees.md | 12 +- docs/internal/benchmarks.md | 444 ---------------------------- docs/internal/brief.md | 438 --------------------------- docs/internal/plan.md | 437 --------------------------- docs/internal/roadmap.md | 207 ------------- docs/internal/survey.md | 209 ------------- docs/local.md | 103 ------- pyproject.toml | 2 +- 15 files changed, 45 insertions(+), 1907 deletions(-) rename docs/{internal => }/architecture.md (93%) delete mode 100644 docs/internal/benchmarks.md delete mode 100644 docs/internal/brief.md delete mode 100644 docs/internal/plan.md delete mode 100644 docs/internal/roadmap.md delete mode 100644 docs/internal/survey.md delete mode 100644 docs/local.md diff --git a/.gitignore b/.gitignore index 6191e93..0ab03b2 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,7 @@ build/ bench/data/ bench/cache/ bench/results/ + +# Notes of the owner, kept out of git: the plan, the roadmap, the design brief, the +# research, and the local stack. +docs/internal/ diff --git a/AGENTS.md b/AGENTS.md index c0db6a5..74303dd 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -9,7 +9,7 @@ rules, and points to the files that hold everything else. make verify # the full check; run it before each commit make integration # the hotdata tests and the conformance suite; needs make local-up make floor # the offline tests at the lowest allowed dependency versions -make local-up # start the local RuntimeDB stack (Postgres, RustFS, engine) +make local-up # start the local engine stack (Postgres, RustFS, engine) make local-down # stop it and delete its data cp .env.template .env # then fill in .env; it is ignored by git uv run --env-file .env scripts/measure_cloud.py # M1 to M3 @@ -19,14 +19,13 @@ uv run --env-file .env scripts/measure_local.py --cloud # M4 to M6 in the clou ## Where things are -- [docs/internal/plan.md](docs/internal/plan.md): the open workstream, the tasks of each phase, - and its stop conditions. -- [docs/internal/roadmap.md](docs/internal/roadmap.md): every phase and its status. +- `docs/internal/`: the notes of the owner. Git ignores this folder, so it exists only in + the owner's checkout. `plan.md` holds the open workstream, the tasks of each phase, and + its stop conditions. `roadmap.md` holds every phase and its status. `brief.md` holds the + design and its reasons. `local.md` tells you how to run against the local stack. - [docs/contracts.md](docs/contracts.md): the record and the operations. - [docs/guarantees.md](docs/guarantees.md): what a consumer can rely on, and the proof. -- [docs/internal/architecture.md](docs/internal/architecture.md): how each operation runs on - the platform. -- [docs/internal/brief.md](docs/internal/brief.md): the design and its reasons. +- [docs/architecture.md](docs/architecture.md): how each operation runs on the platform. - [CONTRIBUTING.md](CONTRIBUTING.md): the checks inside `make verify` and the test rules. - [CHANGELOG.md](CHANGELOG.md): each change to a public surface. - [RELEASING.md](RELEASING.md): how to publish a version to PyPI. Only the repository owner diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 957cc25..b887aef 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -4,8 +4,9 @@ - [uv](https://docs.astral.sh/uv/) installs the development tools and runs the scripts. - GNU Make runs the targets below. -- Docker Desktop runs the local RuntimeDB stack. You need it only for `make local-up` and - `make integration`. +- Docker Desktop runs the local engine stack. You need it only for `make local-up` and + `make integration`. The engine image is private, so only the Hotdata team can start the + stack. ## The one command @@ -37,7 +38,7 @@ passes in CI. It has no tiers, because the full check takes less than five secon | `make verify` | Runs every check above. | | `make floor` | Runs the offline tests on Python 3.11 with the lowest version of each dependency that `pyproject.toml` allows. It needs the network to install them. | | `make integration` | Runs the tests marked `hotdata` against the local stack. Start the stack first with `make local-up`. To use another engine, set `HOTMEMORY_TEST_URL`. | -| `make local-up` | Starts the local RuntimeDB stack: Postgres, RustFS, and the engine. [docs/local.md](docs/local.md) tells you how to point the library at it. | +| `make local-up` | Starts the local engine stack: Postgres, RustFS, and the engine, on port 3000. | | `make local-down` | Stops the stack and deletes its data. | | `make local-pull` | Downloads newer images for the stack. | @@ -97,4 +98,3 @@ same pull request. If you change a public surface, add an entry to [CHANGELOG.md](CHANGELOG.md). No file in this repository names a private repository, a customer, or a deployment detail. -This rule includes `docs/internal/`. diff --git a/README.md b/README.md index d13c1ce..5e73f1d 100644 --- a/README.md +++ b/README.md @@ -11,8 +11,7 @@ the data of the consumer. Status: alpha. A minor version before 1.0 can change a public surface, and [CHANGELOG.md] lists each change. The storage contract exists in Python with two drivers. `MemoryStore` runs in process memory. `HotdataStore` keeps records in one Hotdata managed database. The -memory contract, `Memory`, runs over either driver. [docs/internal/roadmap.md] lists the -phases. +memory contract, `Memory`, runs over either driver. The library has two layers: @@ -142,8 +141,7 @@ print([hit.record.id for hit in store.search("disk", [("team",)])]) ``` One process writes to a database. [docs/contracts.md] gives the tables, -the retrieval query, and the rules of the driver. [docs/local.md] tells you -how to run it against a local engine. +the retrieval query, and the rules of the driver. ## Benchmark @@ -192,24 +190,18 @@ p50 with the vector ranking, and 998 ms with the fused ranking. operations, and the platform facts behind them. - [docs/guarantees.md]: each behavior that a consumer can rely on, its state, and its proof. -- [docs/local.md]: how to run the library against a local RuntimeDB - container. - [docs/benchmarks.md]: the numbers of hotmemory on LongMemEval-S, each with its configuration. [bench/README.md] tells you how to run it. - [CONTRIBUTING.md]: the one command that checks a change, and the rules of the test suite. - [CHANGELOG.md]: each change to a public surface. -- [docs/internal/][docs/internal/brief.md]: the design brief, the survey, the roadmap, the - plan for the open workstream, the research that chose the benchmarks, and the - architecture: how each operation runs on the Hotdata platform. +- [docs/architecture.md]: how each operation runs on the Hotdata platform. [CHANGELOG.md]: https://github.com/hotdata-dev/hotmemory/blob/main/CHANGELOG.md -[docs/internal/roadmap.md]: https://github.com/hotdata-dev/hotmemory/blob/main/docs/internal/roadmap.md [docs/contracts.md]: https://github.com/hotdata-dev/hotmemory/blob/main/docs/contracts.md -[docs/local.md]: https://github.com/hotdata-dev/hotmemory/blob/main/docs/local.md [docs/benchmarks.md]: https://github.com/hotdata-dev/hotmemory/blob/main/docs/benchmarks.md [docs/guarantees.md]: https://github.com/hotdata-dev/hotmemory/blob/main/docs/guarantees.md [bench/README.md]: https://github.com/hotdata-dev/hotmemory/blob/main/bench/README.md [CONTRIBUTING.md]: https://github.com/hotdata-dev/hotmemory/blob/main/CONTRIBUTING.md -[docs/internal/brief.md]: https://github.com/hotdata-dev/hotmemory/blob/main/docs/internal/brief.md +[docs/architecture.md]: https://github.com/hotdata-dev/hotmemory/blob/main/docs/architecture.md [skills/hotmemory/SKILL.md]: https://github.com/hotdata-dev/hotmemory/blob/main/skills/hotmemory/SKILL.md diff --git a/docs/internal/architecture.md b/docs/architecture.md similarity index 93% rename from docs/internal/architecture.md rename to docs/architecture.md index 17ab08c..71bb7d1 100644 --- a/docs/internal/architecture.md +++ b/docs/architecture.md @@ -4,14 +4,14 @@ This page tells you how hotmemory stores memory on Hotdata. It follows each oper the call of the consumer down to the engine, the catalog, and object storage. It also draws the line between the work that hotmemory does and the work that the platform does. -The rules of each operation are in [contracts.md](../contracts.md), and their proofs are in -[guarantees.md](../guarantees.md). This page does not repeat the rules. It shows the calls, +The rules of each operation are in [contracts.md](contracts.md), and their proofs are in +[guarantees.md](guarantees.md). This page does not repeat the rules. It shows the calls, the SQL, and the loads that carry them out. Each claim has one of these sources: - The code of hotmemory at commit `a0145ee`, the merge of phase 10. Such a claim has no mark. -- A measurement in [contracts.md](../contracts.md) or [guarantees.md](../guarantees.md), marked +- A measurement in [contracts.md](contracts.md) or [guarantees.md](guarantees.md), marked [measured] with its date. - A look at the local stack on 2026-10-12, marked [observed, local]. - The source of `hotdata-framework` 0.14.1 and `hotdata` 0.11.0, the client libraries that @@ -40,7 +40,7 @@ flowchart LR end subgraph platform["Hotdata platform"] api["Hotdata API"] - engine["RuntimeDB
(the query engine)"] + engine["The Hotdata engine"] catalog[("Postgres catalog
and DuckLake metadata")] objects[("Object storage
parquet and index files")] api --> engine @@ -51,10 +51,7 @@ flowchart LR framework -- "upload to a presigned URL" --> objects ``` -In the cloud, the API is at `api.hotdata.dev`. The source of `hotdata-framework` names a -CDN and a gateway between the client and the engine. [read from source, not observed] -Locally, the engine serves the API itself on port 3000 and accepts every request with no -token. [local.md](../local.md) describes the local stack. +In the cloud, the API is at `api.hotdata.dev`. ### Who does what @@ -130,22 +127,6 @@ Every table has the key column `id`. A keyed load replaces or deletes rows by th A plain vector index covers a vector column that the caller fills. hotmemory fills each vector itself with the embedder of the caller. -On the local stack, the platform keeps the data in two places. [observed, local] - -- The Postgres catalog holds the tables of RuntimeDB, for example `databases`, `uploads`, - `jobs`, and `indexes`. It also holds the DuckLake tables, for example - `ducklake_snapshot`, `ducklake_data_file`, and `ducklake_delete_file`. DuckLake metadata - records which parquet files make up each version of a table. -- Object storage holds the rows of each table as parquet files under - `ducklake_data//public//`. It holds the index files under - `cache//public/
/indexes/`: a `.tar.gz` file for BM25, a `.usearch` and - a `.feather` file for a vector index, and a parquet file for a sorted index. It also - holds query results as parquet files under `cache/_runtimedb_internal/runtimedb_results/`. - -The local `memory_v1` folder held hundreds of parquet files, one table with many loads -behind it. Whether each load writes exactly one new file was not observed. Where the cloud -keeps these files was not observed. - ## Provision `HotdataStore.provision(name, embedder=..., model=..., dimensions=...)` opens the database @@ -185,7 +166,7 @@ sequenceDiagram participant F as hotdata-framework end box Hotdata platform - participant API as Hotdata API and RuntimeDB + participant API as Hotdata API and engine participant S as Object storage end P->>F: list_managed_databases() @@ -245,7 +226,7 @@ sequenceDiagram participant E as Embedder end box Hotdata platform - participant API as Hotdata API and RuntimeDB + participant API as Hotdata API and engine participant S as Object storage end C->>W: put(...) for each record @@ -343,7 +324,7 @@ sequenceDiagram participant H as HotdataStore end box Hotdata platform - participant API as Hotdata API and RuntimeDB + participant API as Hotdata API and engine end C->>M: capture(text, scope, extractor, actor, observed_at) M->>H: search(text, [scope], kinds fact, profile, procedure, k=10) @@ -516,7 +497,7 @@ over one database, one for each ranking. | Local stack | 130 ms | 42 ms | about 50 | | Cloud | 998 ms | 361 ms | about 50 | -These are the LongMemEval-S numbers of [benchmarks.md](../benchmarks.md), measured on +These are the LongMemEval-S numbers of [benchmarks.md](benchmarks.md), measured on 2026-10-10. In the cloud, one request costs about 400 ms at every table size up to 100,000 rows. [measured 2026-10-05] @@ -536,7 +517,7 @@ sequenceDiagram participant H as HotdataStore end box Hotdata platform - participant API as Hotdata API and RuntimeDB + participant API as Hotdata API and engine end C->>M: supersede(scope, key, fact, valid_from) M->>H: get(scope, key) @@ -603,7 +584,7 @@ measurement, and the vector rankings do not read it. ## Load `load` turns one Markdown document into episodes and facts. It cuts the document into -chunks first, with no call to the platform. [contracts.md](../contracts.md) gives the four +chunks first, with no call to the platform. [contracts.md](contracts.md) gives the four steps of the cut. ```mermaid @@ -616,7 +597,7 @@ sequenceDiagram participant H as HotdataStore end box Hotdata platform - participant API as Hotdata API and RuntimeDB + participant API as Hotdata API and engine end C->>M: load(document, text, scope, extractor) Note over M: Cut the text into chunks.
Chunk n gets the key document-000n. diff --git a/docs/benchmarks.md b/docs/benchmarks.md index bbcdd12..a735051 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -16,7 +16,7 @@ gives the commands that repeat each run and write the files again. | SHA-256 of the data file | `d6f21ea9d60a0d56f34a05b609c79c88a451d2ae03597821ea3d5a9678c3a442` | | Embedder | OpenAI `text-embedding-3-small`, 1,536 dimensions. Each text is cut to 8,191 tokens. | | Driver | `HotdataStore`, except in the driver check | -| Engine, local | The local stack of [docs/local.md](local.md), RuntimeDB image `sha256:48046ae3bf22`, hotdata-framework 0.14.1 | +| Engine, local | The local stack of `make local-up`, engine image `sha256:48046ae3bf22`, hotdata-framework 0.14.1 | Each question has its own history of 38 to 62 sessions. The runner sorts the sessions of a question by date, because 211 of the 500 histories are out of order in the file. Each @@ -625,7 +625,7 @@ phase 10. File: `e2e-sample-all-p11-full-chunks1000-16k.json`. | Budget | 16,000 characters. The block had 15,561 characters at the median. | | Reader input | 4,346 tokens at the median and 4,588 at the 95th percentile, with the prompt | | Reader and judge | `gpt-4o-2024-08-06`, temperature 0, the prompts of the reference code | -| Engine | The local stack, RuntimeDB image `sha256:75403e84eb47` | +| Engine | The local stack, engine image `sha256:75403e84eb47` | The budget is in characters, not tokens. Compare it with another system by the reader input in tokens. @@ -659,7 +659,7 @@ model and gave 48 and 46 again, with the same blocks. Files: ## A large memory store Phase 11 measured recall from one store that holds many memories. Each number below was -observed on 2026-10-21. The local runs used the RuntimeDB image `sha256:75403e84eb47`, +observed on 2026-10-21. The local runs used the engine image `sha256:75403e84eb47`, built on 2026-10-19, unless a paragraph names another. The cloud run used `api.hotdata.dev`. diff --git a/docs/contracts.md b/docs/contracts.md index 95b0fa5..b54cf4c 100644 --- a/docs/contracts.md +++ b/docs/contracts.md @@ -47,9 +47,9 @@ API owns and loads from files. The facts below shape the contracts. A fact marke - A provider-backed vector index cannot share its table with any other index. The engine refuses the second index. A plain vector index and a BM25 index can share a table. [measured 2026-08-18] -- RuntimeDB, the Hotdata query engine, runs on a laptop with a Postgres container and an - S3-compatible storage container. The bare engine image alone refuses managed tables. - [measured 2026-10-05] [local.md](local.md) tells you how to start the stack. +- The Hotdata engine runs on a laptop with a Postgres container and an S3-compatible + storage container. The bare engine image alone refuses managed tables. + [measured 2026-10-05] `make local-up` starts the stack. - The engine refuses a BM25 index or a plain vector index on an empty table. It allows one vector index per table. [measured, local, 2026-10-08] - A filtered search through a plain vector index returns only the rows that were in the @@ -174,7 +174,7 @@ Version 1 ships two drivers. and `content` only. It does not rank by BM25 or by `cues`. - `HotdataStore` uses one managed database, keyed loads and small SQL inserts, a lock for its writes, and the retrieval query below. It needs the `hotdata` extra. With the - local RuntimeDB stack, it is also the development driver. The section on `HotdataStore` below gives the details. + local engine stack, it is also the development driver. The section on `HotdataStore` below gives the details. ### Tables and retrieval @@ -496,5 +496,5 @@ These three rules apply to every operation: language, for extraction that must run centrally, or for scope enforcement above the API key. - Extraction on the server. `capture` and `load` take a callable and do nothing more. -- A third driver. The local RuntimeDB stack covers offline development. +- A third driver. The local engine stack covers offline development. - Access control beyond scope filtering. diff --git a/docs/guarantees.md b/docs/guarantees.md index 08d264c..98ffe30 100644 --- a/docs/guarantees.md +++ b/docs/guarantees.md @@ -12,7 +12,7 @@ Each guarantee has one of two states. - To measure: a measurement must give the answer before the library relies on it. A guarantee marked [measured] was observed against a real Hotdata workspace. A guarantee -marked [measured, local] was observed against the local stack in [local.md](local.md). +marked [measured, local] was observed against the local stack that `make local-up` starts. The Test column names the tests that prove the guarantee. A test in `tests/test_conformance.py` proves a row of the storage contract against every driver. A @@ -79,7 +79,7 @@ A row that no test proves yet names the phase that will prove it. ## Measurements Phase 0 runs six measurements. M1, M2, and M3 run against a throwaway cloud database that -the run creates and deletes. M4, M5, and M6 were planned for a local RuntimeDB container. +the run creates and deletes. M4, M5, and M6 were planned for a local engine container. The bare container refuses managed tables, so they also run against throwaway cloud databases, with `--cloud`. The scripts are `scripts/measure_cloud.py` and `scripts/measure_local.py`. @@ -169,7 +169,7 @@ Cloud result, 2026-10-05, `api.hotdata.dev`: the engine refused the first INSERT hundred single-row upload-and-append loads took a median of 2,136 ms, a p95 of 2,614 ms, and a mean of 2,208 ms. This agrees with the 2.1 seconds per call measured earlier. -Local result, 2026-10-05, the local stack in [local.md](local.md) with SQL writes turned +Local result, 2026-10-05, the local stack with SQL writes turned on: one hundred single-row INSERT statements took a median of 17 ms, a p95 of 22 ms, and a mean of 18 ms. One hundred single-row upload-and-append loads took a median of 24 ms, a p95 of 29 ms, and a mean of 24 ms. Both tables held 101 rows after the calls. @@ -232,7 +232,7 @@ refused a provider-backed vector index beside them, with this error: Embedding-backed vector indexes cannot coexist with other indexes on the same table. ``` -Local result, 2026-10-05, the local stack in [local.md](local.md): a local request costs +Local result, 2026-10-05, the local stack: a local request costs about 6 to 10 ms, so the cost of the engine shows. The vector index starts to pay between one thousand and ten thousand rows, and it pays clearly at one hundred thousand rows. The settings are the same as in the cloud run. @@ -274,7 +274,7 @@ in CI, in place of a throwaway cloud database. Bare container result, 2026-10-05, with the `latest` image pulled on that day (digest `sha256:302371bb1923`): the first call fails. The container refuses to create a managed database, with `managed catalogs require ducklake.metadata_pg_url to be configured`. -[local.md](local.md) explains the cause. +The image keeps its catalog in SQLite, and a managed table needs a Postgres catalog. Cloud result, 2026-10-05, `api.hotdata.dev`: every call works. The calls create a database with two keyed tables and load it in replace, upsert, and delete mode. They build @@ -283,7 +283,7 @@ a BM25 index and run `bm25_search`. They build a provider-backed vector index The row count after the loads was 3, as expected. This is the reference for the local result. -Local result, 2026-10-05, the local stack in [local.md](local.md), with Postgres and +Local result, 2026-10-05, the local stack, with Postgres and RustFS beside the engine: every call works except the provider-backed vector index. It fails with `Embedding provider 'sys_emb_openai' not found`, because the stack configures no embedding provider. The row count after the loads was 3. So the integration tests can run diff --git a/docs/internal/benchmarks.md b/docs/internal/benchmarks.md deleted file mode 100644 index 4b6ca6d..0000000 --- a/docs/internal/benchmarks.md +++ /dev/null @@ -1,444 +0,0 @@ -# Benchmarks: which ones to run, and what they ask of the contracts - -Date: 2026-10-08. Status: research, no code. This file picks the public benchmarks that -hotmemory runs. It says how each one connects to the two contracts. It lists the contract -questions that the benchmarks expose. Phase 5 in `roadmap.md` builds the runner. This file -exists now so that phases 2 and 3 build a contract that a benchmark can drive. - -This file is a record of the research and is not updated. Phase 5 ran the benchmark on -2026-10-10, and `docs/benchmarks.md` holds the observed numbers. Where the two disagree, -`docs/benchmarks.md` is right. For example, the loader counted 43 late sessions that are -evidence, where section 2.1 says 75. - -Four benchmarks were read: DolphinBench, LongMemEval, LoCoMo, and supermemory's MemoryBench. -Each one was read from its code, its data, and its paper, in a shallow clone. The -per-benchmark notes with file and line references are not committed here. - -Evidence labels. [counted] means that the number was counted from the released data file -on 2026-10-08. [read] means that the number comes from a paper, a blog, or a README, and -nobody here reproduced it. [estimate] means arithmetic on stated assumptions. Nothing in -this file was run against hotmemory, so no number here is [measured]. - -## 1. The answer - -Run LongMemEval-S first, with our own Python runner. Run LoCoMo second, as a fast and -cheap smoke test. Use MemoryBench later, only to compare hotmemory with other memory -layers under one reader and one judge. Do not run DolphinBench before an agent exists -around hotmemory, because it grades a whole agent and not a memory layer. - -| Benchmark | What it grades | Retrieval recall with gold labels | Ingest size | Fit now | -|---|---|---|---|---| -| LongMemEval-S | Answers to 500 questions over chat histories | Yes, session and turn ids | 23,867 sessions [counted] | First choice | -| LoCoMo | Answers to 1,540 questions over 10 long conversations | Yes, turn ids, with label errors | 272 sessions [counted] | Smoke test | -| MemoryBench | A TypeScript runner around LongMemEval, LoCoMo, and ConvoMem | No, a judge decides relevance | Same as the benchmark it wraps | Comparison later | -| DolphinBench | An agent that takes tool actions that depend on memory | No | 13,539 messages [counted] | After phase 4 | - -The reasons follow in sections 2 to 5. Section 6 is the contract review, which is the -main purpose of this file. Section 7 gives the code layout, and section 8 the models and -costs. - -## 2. LongMemEval-S - -LongMemEval tests five memory abilities: extraction from one session, reasoning across -sessions, temporal reasoning, knowledge updates, and abstention (the reader says that the -history does not hold the answer). Code and data are MIT. The current data file is -`longmemeval_s_cleaned.json`, released after a cleanup in 2025-09. - -### 2.1 The data - -| Item | Value | -|---|---| -| Questions | 500 [counted] | -| Question types | multi-session 133, temporal-reasoning 133, knowledge-update 78, single-session-user 70, single-session-assistant 56, single-session-preference 30 [counted] | -| Abstention questions | 30, the ids that end in `_abs`, spread over four types [counted] | -| Sessions per question | 38 to 62, mean 47.7 [counted] | -| Turns per question | 396 to 616, mean 494 [counted] | -| Tokens per question | about 101,600 [counted] | -| All sessions | 23,867, of which 19,195 have a unique id [counted] | -| All turns | 246,750, of which 122,416 are user turns [counted] | -| Evidence labels | `answer_session_ids` per question, and `has_answer` on the turns inside them | -| Session timestamp | One per session, for example `2023/05/30 (Tue) 23:40`, no time zone | -| Question time | `question_date`, in the same format | - -Each question has its own haystack (the set of sessions that the system ingests before it -answers). The reference runner builds a new index for each question. So one question maps -to one namespace. - -Two facts in the data contradict the README, and a runner must handle both. - -1. In 211 of the 500 haystacks, the sessions are not in date order [counted]. The runner - sorts by date before it ingests, because supersession depends on order. -2. In 76 questions, a session is dated after `question_date`, always on the same day, by - up to 23.7 hours [counted]. In 75 of them, that session is evidence. A - `recall(as_of=question_date)` drops it. The runner sets `as_of` to the end of the day - of `question_date`. - -### 2.2 The two scores - -Retrieval recall needs no model. The reference code ranks corpus ids for each question and -compares them with the gold ids. It reports `recall_any@k`, `recall_all@k`, and -`ndcg_any@k` for k in 1, 3, 5, 10, 30, and 50. Session-level retrieval indexes user turns -only. Turn-level ids are `{session_id}_{turn_index + 1}`, where the index counts every -turn. The scored set skips abstention questions. It also skips questions with no user-side -evidence turn. That leaves 419 questions [counted from the data, the script was not run]. The paper -reports retrieval only on the M split, so no official S baseline exists [read]. - -End-to-end accuracy needs two models. A reader model gets the retrieved sessions, sorted by -date, and the line `Current Date: {question_date}`, and answers with a chain-of-thought -prompt. A judge model compares the answer with the gold answer. The reference judge is -`gpt-4o-2024-08-06` at temperature 0, with one prompt per question type. The temporal -prompt accepts an answer that is one day off. The knowledge-update prompt accepts an answer -that gives the old value and the new value. The paper reports 0.97 to 0.98 agreement -between this judge and human experts [read]. - -### 2.3 Published numbers - -All numbers are accuracy on S, from vendor pages [read]. The vendors do not say whether they -used the cleaned data. Make sure that a figure matches its page before you quote it. - -| System | Accuracy | Reader | Note | -|---|---|---|---| -| Full context, paper | 60.6% | GPT-4o | Oracle sessions only: 87.0% | -| Zep | 71.2% | GPT-4o | Its full-context baseline is 60.2% | -| supermemory | 81.6%, 84.6%, 85.2% | GPT-4o, GPT-5, Gemini 3 Pro | A later page says 97% as "Recall@20 with aggregation", which is a different metric | -| Mastra | 84.23%, 94.87% | GPT-4o, GPT-5-mini | | -| Emergence | 79% to 86% | not stated | Three variants | -| Hindsight | 83.6% to 91.4% | | Judged by GPT-OSS-120B, not GPT-4o | -| mem0 | 94.4% | not stated | Split and judge not stated | - -The only comparable column is GPT-4o as reader and GPT-4o as judge. Full context with -GPT-4o is about 60%. A memory layer that scores below that adds nothing. - -### 2.4 Why it is first - -- It scores retrieval with gold labels and no model. That isolates the store and the - retrieval query, which is what phases 2 and 3 build. -- Its question types map to the record's fields. Knowledge-update tests `supersede`. - Temporal-reasoning tests `observed_at`, `valid_from`, and `as_of`. Abstention tests - whether `recall` returns too much. -- It is the benchmark that the field reports, and `roadmap.md` already names it. -- Its license is MIT. - -## 3. LoCoMo - -LoCoMo is ten long conversations between two speakers, with questions about them. The -data is CC BY-NC 4.0, which forbids commercial use. A run needs a -check of the license first. - -| Item | Value | -|---|---| -| Conversations | 10, two speakers each [counted] | -| Sessions | 272, 19 to 32 per conversation [counted] | -| Turns | 5,882 [counted] | -| Tokens | 184,631 text-only, about 18,500 per conversation [counted] | -| Scored questions | 1,540: single-hop 841, temporal 321, multi-hop 282, open-domain 96 [counted] | -| Excluded | 446 adversarial questions. 444 of them have no answer field [counted] | -| Evidence | Turn ids such as `D3:7` per question. 9 are malformed [counted] | -| Session timestamp | One per session, for example `1:56 pm on 8 May, 2023` | - -The labels are weak. An independent audit found 99 wrong gold answers in 1,540, which caps -the best score at 93.57% [read]. The same audit found that a judge accepted 62.81% of -answers that were wrong but on topic [read]. The mem0 judge prompt accepts an answer that -"touches on the same topic". Thus a LoCoMo score says little about accuracy. - -The published numbers disagree with each other. The mem0 paper gives mem0 66.88 J (the -score from the judge model) and full context 72.90 [read]. Zep reran its own system with a -corrected setup and reported 75.14 [read]. Letta reported 74.0% for an agent with plain -file tools [read]. mem0 later claimed 92.5 with GPT-5 and a more lenient judge, but its -committed result file says 91.56% [read]. Full context beats every memory layer in the -2025 numbers. - -LoCoMo is useful for two things. First, it is small: a full run is ten namespaces and 272 -sessions, so it finishes in minutes on the local stack. Second, the mem0 paper reports -search latency at p50 and p95 for five systems, and that is the only public latency table -[read]. mem0 reports 0.148 s p50 and 0.200 s p95 for search. Zep reports 0.513 s and -0.778 s. The original LoCoMo code also computes retrieval recall against the evidence turn -ids, which the mem0 runner does not. - -## 4. MemoryBench - -MemoryBench is a runner, not a dataset. It is TypeScript on Bun, under MIT, and it wraps -LongMemEval-S, LoCoMo, and ConvoMem. It ships providers for supermemory, mem0, Zep, a -filesystem, and plain RAG. Its value is comparison: one command runs several memory layers -with the same reader and the same judge. - -The provider interface is small (`src/types/provider.ts`): - -```ts -initialize(config) -ingest(sessions: UnifiedSession[], { containerTag }) -> { documentIds, taskIds? } -awaitIndexing(result, containerTag, onProgress?) -search(query, { containerTag, limit?, threshold? }) -> unknown[] -clear(containerTag) -``` - -A session reaches `ingest` with its messages and `metadata.date` as ISO. The runner calls -`ingest` once per session, in order, and `awaitIndexing` once per question. `search` always -gets the raw question, `limit: 10`, and `threshold: 0.3`. It does not get the question date. -The container tag is `{questionId}-{runId}`, one per question. - -Use it later and not first, for these reasons. - -- The retrieval metrics are not the LongMemEval metrics. A judge model decides whether each - result is relevant, and no gold label is used. The code sets `recall@k` to 1 for any - relevant result. Thus it equals hit@k (issue #67). -- A Python library joins through an HTTP shim. There is no plugin mechanism. A new provider - edits three files in the TypeScript tree. -- The repository has no tests and no CI. A batch of correctness issues from 2026-08-13 is - still open, with fixes unmerged (issues #60 to #78). -- The search call carries no question date, so `recall` cannot use `as_of`. Temporal - questions run without it. - -It reports what we want to report: accuracy per question type, latency per phase at p50, -p95, and p99, and context tokens per question. Serial ingest is one flag: -`--concurrency-ingest 1`. - -## 5. DolphinBench - -DolphinBench comes from mem0, released 2026-09-21, under Apache-2.0. It does not ask -questions. It gives an agent a task in simulated apps, for example "send the email", and -checks the arguments of the tool call. The right arguments depend on facts from the history. - -| Item | Value | -|---|---| -| Personas | 3, with 5,011, 3,400, and 5,128 sessions [counted] | -| Session shape | One user message with a dated narrative time. No assistant turns | -| History | 2023-01 to 2027-12, about 500,000 user tokens per persona [read] | -| Tests | 600, 200 per persona. 264 depend on one fact, the rest on 2 to 9 [counted] | -| Checks | 1,194 exact field checks and 610 judge checks. Every check must pass for the test to pass | -| Judge | Fixed to an Azure OpenAI model. The runner refuses a paid run with another judge | - -The adapter does not see a memory interface. It sees `run_interaction`, which sends one -message to the agent, and `freeze`, after which memory is read-only. hotmemory needs an -agent loop, tools that call `recall` and `capture`, and Azure OpenAI credentials for the -judge. The published runs cost $96 for the cheapest configuration and $1,133 to $1,831 for -Claude Code with Sonnet 5, without the judge [read]. The repository had 10 commits at the -time of reading, and its own driver document says that a paid end-to-end run is -unverified. - -DolphinBench measures what the first consumer does: an agent that must act on a fact from -memory. When phase 4 gives an agent around hotmemory, run it. The MCP server in phase 6 is the -other entry point. It does not test the contract in isolation. - -## 6. What the benchmarks ask of the contracts - -This section is the main result. Each item states what a benchmark needs, whether the -contract gives it today, and which phase decides the open ones. - -### 6.1 What the contract already gives - -1. Gold ids fit in `sources`. LongMemEval scores session ids and turn ids. LoCoMo scores - turn ids such as `D3:7`. A record from `capture` carries - `sources=("session:", "turn:_")`, and the runner maps a ranked list of - records to a ranked list of unique session ids. No new field is needed. -2. The session date fits in `observed_at`. Every benchmark gives one timestamp per session - and none per turn. `capture(text, scope, actor, observed_at, extractor)` takes it. The - record refuses a timestamp without a time zone, so the runner attaches UTC. -3. One question or one conversation fits in one namespace. For example, the namespace for - question `q123` in run `r7` is `("bench", "lme_s", "r7", "q123")`. - `recall(scopes=[that namespace])` matches the isolation that every runner expects. -4. Knowledge updates fit `supersede`, and temporal questions fit the two clocks. -5. A raw-turn baseline needs no model. Turns go into the `episode` table, and retrieval - runs over them. This is the same shape as the reference retrieval runner. It measures - the store before extraction exists. - -### 6.2 Questions for phase 2, the Hotdata driver - -1. The cost of a write decides the ingest plan. A cloud load costs about 2.1 s, and a local - load costs about 24 ms (measured in phase 0). The estimates below assume one table and - one writer. - - | Write pattern | LongMemEval-S, cloud | LongMemEval-S, local | LoCoMo, cloud | - |---|---|---|---| - | One load per question or conversation | 500 loads, about 18 min | under 1 min | 10 loads, about 21 s | - | One load per session | 23,867 loads, about 14 h | about 10 min | 272 loads, about 10 min | - | One load per turn | about 247,000 loads, about 144 h | about 1.6 h | 5,882 loads, about 3.4 h | - - All values are [estimate]. Proposal: run the full ingest on the local stack, and run the - latency numbers on the cloud for a subset of questions. Report the two separately. -2. Many namespaces share one table lock. The 500 namespaces of one run are in one table, so - their loads serialize. Runners also run questions in parallel: the mem0 runner used ten - workers, and MemoryBench takes a concurrency flag. Phase 2 states whether one `writer` - is safe across threads in one process. Extraction can then run in parallel while one - writer sends the loads. -3. The search must reach the `episode` table. The raw-turn baseline in 6.1 searches - episodes, and a fact links back to its episode through `sources`. Today `Store.search` - names no table. Phase 2 states how a caller searches episodes. A filter on - `kind="episode"` is one way, and it needs no new surface. -4. The table grows to benchmark size. The `episode` table for one LongMemEval-S run holds - 122,416 user turns, or 246,750 with assistant turns [counted]. In phase 0, a plain vector - index on 100,000 rows took 65.8 s to build in the cloud and 21.9 s locally [measured]. - A load of 100,000 rows with embeddings took 62.9 s in the cloud [measured]. Phase 2 - decides when the driver builds indexes: before the load, or once after a bulk load. -5. Latency measures the store, not the network. A cold engine costs 8 to 10 s on the - first call. The runner warms the engine and reports cold start as a separate number. - The cloud floor of a query request is about 400 ms. Thus a cloud search will not reach - mem0's reported 148 ms p50. A local run shows the engine cost alone. Report both, with - the setup named. - -### 6.3 Questions for phase 3, the memory contract - -1. A buffered write is invisible to `capture`. `capture` calls `recall` before it extracts, - so that the extractor can skip a known fact and supersede an old one. If the writer - flushes once per question, every session in the question is extracted against an empty - memory. Then supersession across sessions disappears, and knowledge-update questions - test nothing. There are three options. - - Flush per session. This is slow in the cloud and fast locally (6.2, item 1). - - Let `recall` inside one `Memory` see its own unflushed buffer. This changes the rule - "a write is visible to the next `recall`", and it is a change to `guarantees.md`. - - Extract offline in date order with an in-process view, and load once. The benchmark - then no longer tests `capture` as a consumer calls it. - - The benchmark is the first consumer that ingests history in bulk, so phase 3 decides - this before the runner exists. The first option on the local stack is the simplest - proposal. -2. `recall` hides the past states that some questions need. `recall` keeps the records that - are valid at `as_of`. That is right for knowledge-update questions. Some - temporal-reasoning questions ask about a value before it changed, for example "what was - my commute before I moved". Phase 3 decides whether `recall` takes an option to include - superseded records with their validity spans. The rendered block already prints the - span, so the reader can tell old from new. -3. The `as_of` time needs a stated rule for dates without a time. LongMemEval evidence can - be dated up to 24 hours after `question_date`, on the same day. The runner passes the end - of the day. The contract states whether `as_of` is inclusive and what a caller passes - when the source has a date and no time. -4. The budget is in characters, and the benchmarks count tokens. MemoryBench reports - context tokens per question, and the mem0 paper compares memory layers by token count. - The runner counts the tokens of the rendered block with one named tokenizer, and the - contract keeps characters. -5. A distance is not a score. MemoryBench passes `threshold: 0.3`, which assumes a score - where higher is better. The shim ignores it, or converts the distance as - `guarantees.md` already says that an adapter does. -6. The extractor needs the session as a unit. Benchmarks hand over a session with two - roles. Most LongMemEval tokens are assistant text: 44.4 million of 50.8 million - [counted]. The extractor needs the session, its date, and the role of each turn. - `capture` takes text, so the runner renders a session as text with role labels. Phase - 3 makes sure that this is enough, or adds a structured input. - -### 6.4 Order of the phases - -The order in `roadmap.md` is right, with one addition. A small retrieval slice can join -phase 2 at low cost. Take 20 questions of LongMemEval-S, write their user turns as episodes -on the local stack, and score `recall_any@10` against the gold session ids. The slice needs -no model except the embedder. It proves that the driver, the `sources` convention, and the -namespace per question work together, three phases before the full runner. The -`MemoryStore` oracle test can run the same slice and compare the two drivers. - -Phase 5 then adds extraction, the reader, and the judge on top of a path that already -works. - -## 7. The code setup - -The runner lives outside the library, in a `bench/` folder at the root of the repository. -It is not part of the package, and the offline test suite does not import it. Datasets -download to a folder that git ignores. - -```text -bench/ - README.md how to run, which models, the cost of a run - common/ - ids.py the sources convention: "session:", "turn:_" - clock.py parsers for the session dates of each benchmark, to UTC - embedder.py the embedder callable that both drivers take - extractor.py the extractor callable for capture, one model call per session - reader.py the reader prompt and call - judge.py the judge prompts per question type, copied from the reference code - metrics.py recall_any@k, recall_all@k, ndcg_any@k, accuracy per type, latency p50/p95/p99 - report.py writes one JSON result and one Markdown table per run - longmemeval/ - data.py download, sort sessions by date, the end-of-day as_of rule - retrieval.py the no-model score: episodes in, ranked session ids out - e2e.py capture, recall, reader, judge - locomo/ - data.py parse the session dates, repair the 9 bad evidence ids - e2e.py - memorybench_shim/ - server.py FastAPI over Memory: /ingest, /flush, /search, /clear - provider.ts a MemoryBench provider that calls the shim -``` - -The dependencies go in an optional group in `pyproject.toml`, so that the library keeps no -runtime dependency: - -```toml -[dependency-groups] -bench = ["anthropic", "openai", "huggingface-hub", "tiktoken", "fastapi", "uvicorn"] -``` - -The core loop of `longmemeval/e2e.py`, in outline: - -```python -for q in questions: # one namespace per question - scope = ("bench", "lme_s", run_id, q.id) - for s in sorted(q.sessions, key=lambda s: s.date): - memory.capture( - text=render_session(s), # role-labeled turns - scope=scope, - actor="bench-extractor", - observed_at=s.date, # UTC - extractor=extract, # writes sources=("session:", ...) - ) - t0 = time.perf_counter() - hits = memory.recall(q.question, scopes=[scope], as_of=end_of_day(q.date), budget=8000) - search_ms = (time.perf_counter() - t0) * 1000 - answer = read(q.question, hits.block, current_date=q.date) - verdict = judge(q.type, q.question, q.answer, answer) -``` - -The shim for MemoryBench maps its five calls onto `Memory`. `ingest` calls `capture` for -each session inside a `writer`. `awaitIndexing` flushes the writer. `search` calls `recall` -with no `as_of`. `clear` calls `forget` on the namespace. - -The commands, as phase 5 ships them: - -```sh -make local-up -uv run --group bench --env-file .env bench/longmemeval/retrieval.py --limit 20 -uv run --group bench --env-file .env bench/longmemeval/e2e.py --sample-per-type 9 -uv run --group bench --env-file .env bench/locomo/e2e.py -``` - -A run writes its configuration (models, embedder, k, budget, driver, local or cloud, data -file hash) next to its numbers. A number without its configuration is not reported. - -## 8. Models and cost - -A benchmark run needs four models. hotmemory ships none of them, by the brief. The runner -supplies them. - -| Role | Used in | Proposal | Reason | -|---|---|---|---| -| Embedder | Every search | An OpenAI embedding model, or a local sentence-transformer model | Anthropic offers no embedding model. A local model keeps the retrieval score free and repeatable. | -| Extractor | `capture` | Claude Haiku 4.5, with Claude Sonnet 5.5 as a second point | One call per session. This is the largest cost. | -| Reader | End-to-end accuracy | GPT-4o for the published number, and a Claude model for internal runs | Every vendor number on LongMemEval-S uses GPT-4o. | -| Judge | End-to-end accuracy | `gpt-4o-2024-08-06`, as in the reference code | A different judge makes the number incomparable. | - -The keys are `ANTHROPIC_API_KEY` and `OPENAI_API_KEY` in `.env`, beside the Hotdata keys -that the measurement scripts already read. - -The estimates below assume 1,000 tokens of prompt overhead per extraction call, an 8,000 -token reader context, and list prices on 2026-10-08. All values are [estimate]. Make sure -that the prices are current before a paid run. - -| Run | Model calls | Cost | -|---|---|---| -| LongMemEval-S, retrieval on raw turns | embeddings only | under a few dollars, or none with a local embedder | -| LongMemEval-S, extraction per session | 23,867 | about $93 with Haiku 4.5, about $185 with Sonnet 5.5. The batch API halves this | -| LongMemEval-S, reader | 500 | about $5 to $13 | -| LongMemEval-S, judge | 500 | under $1 | -| LongMemEval-S, a sample of 9 per type | 54 questions | about one tenth of the above | -| LoCoMo, a full run | 272 extractions, 1,540 reads and judgments | about $15 to $35 | - -Start every new configuration with the sample of 54 questions. A full run follows only when -the sample works. - -## 9. What was not confirmed - -- No benchmark script was run. Every count above comes from the data files, and the 419 - scored retrieval questions come from the data, not from the reference script. -- The vendor numbers came through web pages and were not reproduced. No vendor states - whether it used the cleaned LongMemEval data. -- The DolphinBench leaderboard loads by script, and the live entries were not seen. The - paper PDF was not read. -- The LongMemEval M split was not downloaded. -- Whether Hotdata accepts reads in parallel during a load was not measured. diff --git a/docs/internal/brief.md b/docs/internal/brief.md deleted file mode 100644 index efe2fab..0000000 --- a/docs/internal/brief.md +++ /dev/null @@ -1,438 +0,0 @@ -# Brief: hotmemory, agent memory as tables on Hotdata - -Status: draft, revised 2026-10-05, phases moved to `roadmap.md` and `plan.md`, and the phase 0 -measurements applied (results in `docs/guarantees.md`). Phase 1 built the storage contract with -`MemoryStore`, and `docs/contracts.md` states the details that it fixed. It fixes the two -contracts, the guarantees behind them, and the harness that proves them, before the first -line of code. The revision applies section 4 of `survey.md`, in this folder. - -Evidence labels. A sentence marked [measured] was run against a real Hotdata workspace and -the number is transcribed. A sentence marked [to measure] names a question that the first -phase answers. Anything unmarked is a design decision. - -This folder is internal in audience, not in visibility. Nothing in it names a private -repository, a customer, or a deployment detail. The first consumer is an incident -investigator built on Hotdata, and it is named only that way here. - -## Where the work is tracked - -This brief holds the design and its reasons. The phases, their status, and anything -deferred are in `roadmap.md`. The current phase, with its tasks, acceptance criteria, -measurements, and verification, is in `plan.md`. Both files are in this folder. A task -becomes a GitHub issue only when its phase is the current one. - -## 1. Goal - -A Python library that stores an agent's long-term memory as rows in a Hotdata managed table, -and gives any agent five operations over them: put, get, list, search, and delete. On top of -that storage, a memory contract gives an agent the operations it uses in practice: remember, -recall, supersede, forget, and profile. The incident investigator is the first consumer. The -library is not specific to it. - -The one claim that makes this a product and not a feature: a memory table is an ordinary -table, so an agent can join its memory to its own data in one SQL query. The README leads -with that sentence. The survey confirms that no surveyed system stores memory as typed -columns with validity spans in the same engine as the consumer's data. - -## 2. Context - -### 2.1 Where the package sits - -The Python stack has four layers. The `hotdata` package is the generated HTTP client. -`hotdata-framework` is the hand-written runtime over it, and it owns managed databases, -parquet upload and load, and index builds. `hotdata-langchain` adds LangChain tools and a -VectorStore. Agent applications sit on top. `hotmemory` sits directly over -`hotdata-framework` and imports nothing from `hotdata-langchain`. Adapters for LangGraph and -for MCP are separate work and are listed under out of scope. - -The library runs inside the consumer's process. The consumer imports it, and it calls the -Hotdata API over HTTPS with the consumer's API key. There is no hotmemory server. The hosted -API that production needs already exists, and it is Hotdata's own API: files, loads, -indexes, and query. The same library runs against a local RuntimeDB on a laptop, but -not against the bare engine image. [measured, M6] A managed table needs a Postgres catalog, -because DuckLake metadata lives only in Postgres, and storage that can presign, because the -framework uploads parquet through presigned URLs. The local stack in `docs/local.md` is -three containers: Postgres, RustFS, and the engine sharing the RustFS network namespace so -the presigned address resolves on both sides. The library then needs `HOTDATA_API_URL` -pointing at the engine, `HOTDATA_WORKSPACE` set to any id, and any non-empty -`HOTDATA_API_KEY`. Every driver call works on that stack except a provider-backed vector -index, because no embedding provider is configured locally. A separate memory service in front of Hotdata is what section 8 rules -out, and the reasons it can become necessary are listed there. - -### 2.2 Platform facts that shape the design - -These come from measurements against Hotdata between 2026-08-27 and 2026-09-18. - -- A write is a file load in every released engine and in production. The caller uploads - parquet and calls a keyed load in replace, append, upsert, update, or delete mode. - [measured] The cloud accepted SQL INSERT on 2026-10-21. [measured, M4] INSERT and CREATE - TABLE, CREATE TABLE AS, and DROP TABLE are implemented. UPDATE and DELETE are refused. An INSERT is an append - load whose source is a query: the planned rows are staged as parquet and appended through - the same locked load path, so it holds the same per-table lock and maintains indexes the - same way. A SQL write needs the read and write claims and can target only the database's - own catalog. -- A write costs about 2.1 seconds per call and is flat in row count. One row costs 1.75 - seconds, ten thousand rows cost 2.5 seconds. Batching is free. The only lever is call - count. [measured] -- There is no conditional write. No mode takes a predicate on the existing value and none - returns the affected row. Two writers on one key is last writer wins. [measured] If the - engine adds UPDATE with a WHERE clause that returns `rows_affected`, that is a - conditional write. The design below assumes today's engine. -- A load holds an exclusive lock on the whole table. A second concurrent load is refused - with `409 RESOURCE_LOCKED`. Distinct keys buy no concurrency. [measured] -- Read after write was never stale across 30 loads. A read costs 0.3 to 0.6 seconds. A cold - engine costs about 8 to 10 seconds on its first call. [measured] -- A table layout is permanent. There is no ALTER. A `mode="upsert"` load with extra columns - succeeds and silently drops them. Only a `mode="replace"` load widens a table. [measured] -- `delete_managed_table` burns the table name forever. [measured] -- `expires_at` on a database records a lifetime and nothing enforces it. [measured] -- A managed database is a hard isolation boundary. Managed databases cannot attach each - other. A registered Postgres connection attaches cleanly. [measured] -- Three kinds of index exist on a managed table: BM25 over a text column, vector over a - float-list column or over a text column with server-side embedding, and sorted. A vector - index has two modes. Plain mode indexes a vector column the caller embedded. Provider- - backed mode takes a text column, embeds it on the server through a Hotdata embedding - provider, and the query passes text. A metric mismatch in plain mode silently reverts to - a full scan. Indexes are invisible to SQL, so only the control plane can say whether one - exists. (`hotdata_framework/client.py:584-600`, `hotdata_langchain/vectorstore.py:541`) -- A provider-backed vector index cannot share its table with any other index. The engine - refuses the second one. A BM25 index, a plain vector index, and a sorted index share a - table. [measured 2026-08-18, confirmed by M5 on 2026-10-05] -- `bm25_search` refuses to run on a column without a BM25 index. There is no scan - fallback. [measured, M5] -- RuntimeDB runs on a laptop with a Postgres container and an S3-compatible storage - container, with no cloud dependency. The bare image, with its SQLite catalog and - filesystem storage, refuses managed tables. [measured, M6] The local stack is a real - Hotdata for development and for measurements that production cannot run. A load there - costs about 24 ms, against about 2.1 seconds in the cloud, so the cloud write cost is - outside the engine. [measured, M4] - -### 2.3 What the design takes from these facts - -- Records are immutable. A revision is a new row that marks the old row superseded. This - is the only way to get history, supersession, and safe concurrency from a store whose - write primitive is a keyed row replace. -- One writer per table. The driver serializes loads and retries on 409. -- Writes are either synchronous and slow, or buffered and flushed. The API exposes both and - the guarantee section says which is visible when. -- Expiry is a column plus a sweeper. Nothing is deleted by the platform. -- The tenant boundary is a database, enforced by the platform through the API token's - workspace allow-list. A namespace is a column filter inside the database, enforced by - the library. The library never claims to enforce more than that. -- The schema is versioned in the table name. A column change is a new version, never an - edit. -- Granularity is one fact per row. A document is not a memory. It is an episode that facts - point back to. Section 3.4 says why. - -## 3. The storage contract - -The storage contract is the public, stable surface. It is a Python protocol named `Store`. -Every driver implements it and the conformance suite runs against every driver. - -### 3.1 The record - -A record is the unit the store holds. Its fields are fixed for schema version 1. - -| Field | Type | Meaning | -|---|---|---| -| `namespace` | tuple of strings | Where the record lives. Stored as a path string joined with `/`, and matched on whole labels, never on a string prefix. A label is not empty and contains no `.` and no `/`. | -| `key` | string | The caller's stable identifier inside the namespace. It is not empty and contains no `/` and no `@`. | -| `revision` | integer | 1 for the first put under a key, then counting up. | -| `kind` | string | One of `fact`, `profile`, `procedure`, `episode`. | -| `subject` | string | What the record is about, for example an alert key, a person, a service. Empty when unknown. | -| `content` | string | The text a model reads. Not empty after normalization. This is the column a provider-backed index embeds. | -| `cues` | tuple of strings | Questions or phrases this record answers. Optional. Embedded separately from `content`. Section 3.4. | -| `payload` | JSON object | Structured data the consumer defines. The store never reads it. | -| `tags` | tuple of strings | Free labels. Filterable. | -| `sources` | tuple of strings | References to where the record came from: a thread id, a document path, a run id, an episode key. The length of this list is the corroboration count. | -| `actor` | string | Who wrote this revision: a user id, an agent name, an extractor name. | -| `created_at` | timestamp | When this revision was written. System clock. | -| `observed_at` | timestamp or null | The time of the source the record came from. A post-mortem loaded a year later keeps the incident date here. | -| `valid_from` | timestamp or null | When the fact became true. Defaults to `observed_at`. Null means unknown. | -| `valid_until` | timestamp or null | When the fact stopped being true. World clock. Null means still true. | -| `expired_at` | timestamp or null | When the store learned that the fact stopped being true. System clock. Null on a current record. | -| `superseded_by` | string or null | The id of the revision that replaced this one. Null on the current revision. | -| `forget_after` | timestamp or null | When a sweeper is allowed to delete the record. Null means keep. | -| `forget_reason` | string | Why, when `forget_after` is set. Empty otherwise. | -| `id` | string | `namespace/key@revision`. Derived. The load key. | - -The record carries no embedding field in the public contract. The Hotdata driver adds an -embedding column when the caller supplies an embedder, and omits it when the caller uses a -provider-backed index. Which one is in use is driver configuration. - -Two clocks, on purpose. `valid_until` is when the world changed. `expired_at` is when the -store found out. A replay that asks what memory held on a given day reads `created_at` and -`expired_at`. A question about the world on that day reads `valid_from` and `valid_until`. - -### 3.2 The operations - -| Operation | Signature, in words | Behaviour | -|---|---|---| -| `put` | namespace, key, record fields | Writes a new revision. If the key exists, the new row gets the next revision and the previous current row gets `superseded_by` set. Returns the id. If the normalized content equals the current revision's, writes nothing and returns the current id, unless the current revision is past its `forget_after`. | -| `get` | namespace, key, optional revision | Returns the current revision, or the named one. Returns None when absent. | -| `history` | namespace, key | Returns every revision, oldest first. | -| `list` | namespace prefix, optional filter, optional since, limit | Returns current revisions under the prefix, newest first. No model, no embedding. The filter is equality on `kind`, `subject`, `tags`, and `actor`, and a range on `valid_from`, `valid_until`, `created_at`, and `expired_at`. | -| `search` | query text or none, namespace prefixes, optional filter, k | Returns up to k current revisions ranked by relevance, each with its content distance. The same filter as `list`. With no query text it is `list`. | -| `delete` | namespace, key | Removes every revision of the key. Hard delete. | -| `list_namespaces` | optional prefix | Returns the distinct namespaces under the prefix. | -| `writer` | optional row count, optional interval | A context manager. Every `put` inside it is buffered. The buffer flushes on exit, or at a row count, or at an interval, and the writer records the ids it flushed. | - -Deleted revisions, superseded revisions, and records past `forget_after` never appear in -`list` or `search`. `history` shows superseded revisions. Nothing shows deleted ones. - -Normalized content means lowercased with whitespace collapsed. Deduplication is exact on -that form and nothing more. A paraphrase is a new record. Merging paraphrases is the -extractor's job, with the help of `candidates` in section 4. - -### 3.3 The drivers - -Two drivers ship in version 1. - -- `MemoryStore`. In process, in memory. It is a real driver and not a mock. It computes - relevance with the same cosine distance the engine uses, and it refuses any filter it - does not model. The offline suite runs against it, and it is the oracle for the other - driver. -- `HotdataStore`. One managed database, two record tables per schema version and two - tables beside them (`cue_v1` and `meta_v1`, from phase 2), keyed loads, a serialized - writer, and the ranking query in section 3.4. `docs/contracts.md` states the layout. The integration leg runs against - it. Against the local RuntimeDB stack it is also the development driver, so no third - driver is needed for working offline. The integration leg can run against that stack in - CI, with plain vector indexes in place of provider-backed ones. [measured, M6] - -### 3.4 Tables and retrieval - -Two record tables per schema version, because facts and raw material have different -shapes and different write patterns. Phase 2 added a cue table and a meta table beside -them, because the engine allows one vector index per table. - -The `memory` table holds facts, profiles, and procedures: small rows, revisioned, searched -often, and the rows that `profile` renders. The `episode` table holds raw material: a -thread, a document, a post-mortem, chunked at a fixed size, append-only, never revisioned, -and searched when a fact is not enough. A fact's `sources` names the episode keys it came -from, so a consumer can walk from a fact to its evidence in one join. Tabular data is never -copied into either table. It stays in the consumer's own tables and a fact points at it. - -Why one fact per row and not one document per row. A document stored as one record makes -every small revision a near-duplicate of the whole, with a cosine distance close to zero -between versions, which is exactly the duplicate problem a memory layer exists to avoid. -Small facts revise cleanly, supersede cleanly, and render into a profile. The surveyed -systems that extract agree on the unit: 15 to 80 words in mem0, one edge per fact in -Graphiti. Markdown remains a fine format for the `content` column of an episode chunk. - -Retrieval runs in one SQL query over the `memory` table, in three stages. - -1. Exact filters first: namespace labels, validity at the as-of time, `kind`, `subject`, - `tags`, and `forget_after`. These are plain predicates and cut the candidate set before - any ranking. -2. Three rankings over the survivors, each served by its own index: BM25 over `content`, - vector distance over `content`, and vector distance over `cues`. Fused by reciprocal rank - fusion, written as plain SQL with common table expressions. The engine has no fusion - primitive and needs none. Because a provider-backed index excludes every other index, - the two vector rankings need plain vector indexes over embedding columns, and so an - embedder supplied by the caller. Without an embedder, the driver can offer BM25 alone or - a provider-backed vector ranking alone, not both. `bm25_search` ranks the whole table - and the filters of stage 1 apply after it, so the BM25 fetch depth must be wide enough - to survive the filters. -3. A sorted index on `created_at` for recency ordering and for the sweeper. Since phase 11 - it is an option, `sorted_index=True`, because a write into a table of 188,893 rows took - 8 to 20 s with it and about 1.4 s without it. [measured, local, 2026-10-22] - -`cues` is the part that is not in any surveyed system and is cheap here. A cue is the -question a record answers, written at capture time by the extractor or by the caller. A -query that resembles the question matches the cue even when it shares no words with the -content. Cues are optional, and the content-only path always works, so a consumer that -writes no cues loses nothing it had. - -Fast reads come from the table being small, not from cleverness. A consumer's memory table -holds thousands of rows, and a filtered scan of that is fast before any index exists. M5 -measured the crossover. In the engine, the vector index starts to pay between one thousand -and ten thousand rows, and at one hundred thousand rows it cuts the filtered vector rank -from 48 ms to 15 ms. The fused query with indexes takes 21 ms there. In the cloud, one -request costs about 400 ms at every size, and that floor hides the saving up to at least -one hundred thousand rows. Only the sorted index on `created_at` showed a cloud gain, 695 ms -to 407 ms at one hundred thousand rows. The driver therefore builds the BM25 index whatever -the size, because `bm25_search` needs it, and treats the vector and sorted indexes as an -optimisation that matters from about ten thousand rows. [measured, M5] Phase 11 found the -cost of that optimisation on writes, so both are options, off by default. - -## 4. The memory contract - -The memory contract is what an agent calls. It is a class named `Memory` built over one -`Store`. It is the surface the first consumer uses and the surface a plain Python agent -uses. It can change more freely than the storage contract while the library is young. - -| Operation | Signature, in words | Behaviour | -|---|---|---| -| `remember` | facts, scope, actor | Writes already-structured facts. Each fact is a record with `kind` set. A deterministic key is derived from the subject and the normalized content hash, so a retried call writes nothing new. | -| `recall` | query, scopes, optional as_of, budget in characters | Searches the allowed scopes, keeps records valid at `as_of`, and returns the top records inside the budget as a list and as one rendered block. The block labels every record with its sources and its validity span, and nothing else. | -| `candidates` | fact, scopes, k | Returns the k nearest current records with their distances. No decision. A caller's consolidator reads this before deciding to `remember` or `supersede`. | -| `supersede` | key of the record to close, new fact, optional valid_from | Closes the named record and writes the new fact as the next revision under its key. The old record's `valid_until` becomes the new record's `valid_from`, and its `expired_at` becomes now. If the old record's `valid_from` is later than the new one's, the call refuses. The library decides nothing by itself. | -| `forget` | ids, or a horizon | Deletes the named records, or every record whose `forget_after` is before the horizon. | -| `profile` | subject, scopes, budget in characters | Returns the current records whose `subject` matches, grouped by `kind`, as one rendered block under the budget. The block ends with the namespaces and record counts that `recall` can reach, so an agent knows what to search for. This is the always-loaded block. | -| `capture` | text, scope, actor, observed_at, extractor | Calls the caller's extractor with the text, `observed_at`, and the current records that `recall` returns for the scope, then calls `remember` on what comes back. The extractor is a plain callable. The library ships no model and names no model. | - -The as-of rule, stated once. A record is valid at time T when `valid_from` is null or at -most T, and `valid_until` is null or after T. A null `valid_from` means the start of time. -A conformance test pins both halves. - -Three rules hold across every operation. - -- A scope is a namespace prefix. The caller supplies the allowed scopes on every call. - The library filters on them and does nothing else to enforce access. -- Memory is a record, not a rule. The rendered block from `recall` and from `profile` says - what was recorded and when. It never says what to do. A consumer that wants instructions - writes them in its own prompt. -- A write is visible to the next `recall` and never inside the current turn. A consumer - that captures and recalls in one step reads what was there before the capture. - -## 5. Guarantees - -Each guarantee names its state. Answered means a measurement already supports it. To -measure means phase 0 must answer it before phase 1 relies on it. Every answered guarantee -gets a conformance test in phase 1, and `docs/guarantees.md` names the test. - -| Question | Answer | State | -|---|---|---| -| Does a second `put` under the same key replace the record? | No. It writes revision n+1 and marks revision n superseded. `get` returns n+1. | answered | -| Can a retried `remember` create duplicates? | No. The key is derived from subject and normalized content hash, and a put whose normalized content equals the current revision's writes nothing. | answered | -| Is a paraphrase a duplicate? | No. Deduplication is exact on normalized content. `candidates` exists so a caller can decide. | answered | -| When a synchronous `put` returns, is the record visible to `list`? | Yes. Read after write was never stale in 30 trials. | answered [measured] | -| When a synchronous `put` returns, is the record visible to `search`? | Yes. Without an index the ranking query scans the table. With a provider-backed vector index or a BM25 index, the first search after the load returned the new row. | answered [measured], M1 | -| When a buffered `put` returns, is the record visible? | No. It is visible after the writer flushes, and the writer records the ids it flushed. | answered | -| What happens when two processes write the same table? | The second load is refused with 409. The driver retries with backoff and gives up after a bound. Within one process the writer serializes. | answered [measured], M3: with two concurrent writers no load needed more than 3 of 8 attempts | -| What happens when two writers put the same key? | Last writer wins at the row level. Because revisions are new rows, both revisions exist and the later one is current. | answered | -| Does `delete` remove retained revisions and derived embeddings? | Yes. The embedding is a column of the row, and after a keyed delete neither a provider-backed vector index nor a BM25 index returned the row. | answered [measured], M2 | -| Which filters work in `list` and `search`? | Equality on `kind`, `subject`, `tags`, and `actor`. Range on `valid_from`, `valid_until`, `created_at`, and `expired_at`. Prefix on namespace labels. Anything else raises. | answered | -| Does a higher score mean more relevant? | The store returns distance, and lower is closer. The memory contract returns records in order and no score. An adapter that needs a score converts. | answered | -| What does `recall(as_of=T)` return? | Records valid at T by the as-of rule in section 4, including records superseded after T when `history` is consulted, and excluding them otherwise. | answered | -| Who enforces scope? | The library filters on the caller's allowed scopes. The platform enforces the database boundary through the API token. A caller holding the token can bypass the library. | answered | -| Can a consumer tell sources from extractions from hypotheses? | Yes through `kind`, `sources`, and `actor`. An extracted fact carries the extractor's name in `actor`. | answered | -| Is a record deleted when `forget_after` passes? | No. It stops appearing in `list` and `search`. A sweeper deletes it when run. | answered [measured] | -| Does a write inside a turn reach a `recall` in the same turn? | No, by contract. A consumer reads what was there before its own capture. | answered | - -Phase 0 measurements. M1 to M3 run against a throwaway database that the run creates and -deletes. M4, M5, and M6 ran against throwaway cloud databases and against the local stack, -because the bare container refuses managed tables. All six were run on 2026-10-05, and -`docs/guarantees.md` holds the numbers. - -- M1. Build a provider-backed vector index on a table, load ten more rows, and search - for one of them. Record whether the new row is served, and after how long. -- M2. Delete a row that an index covers and search for its content. Record whether the - index still returns it. -- M3. Run two processes that load the same table at once with the serialized writer in - each. Record how many 409s occur and whether the retry bound of the driver is enough. -- M4. With SQL writes on, time one hundred single-row INSERT - statements and one hundred single-row upload-plus-load calls into the same table shape. - Record both per-call costs. -- M5. Load one thousand, ten thousand, and one hundred thousand rows into the `memory` - table shape. Time the three-stage retrieval query with and without the BM25 and vector - indexes at each size. Record the size at which an index first beats the scan. -- M6. Against the local container with the three variables above, run every framework - call the driver needs: create a managed database, declare two tables, load with keyed - upsert and delete, build a BM25 index and a provider-backed vector index, and query. - Record which calls work with no API key and no control plane. If all of them do, the - integration leg can run against the container in CI instead of a throwaway cloud - database. - -## 6. The harness - -The library is deterministic. No clock, no network, and no model are needed to run the -offline suite. The harness makes that cheap to prove on every change. - -### 6.1 One command - -`make verify` runs ruff, ruff format in check mode, strict mypy over the library and the -tests, the offline suite, and the documentation checks in section 6.5, in that order. It -runs in under five seconds. CI calls the same target. There are no tiers for the offline -loop, because a loop under five seconds does not need a way to run less than everything. - -### 6.2 The conformance suite - -One suite of tests, parametrized over every driver. Each guarantee in section 5 with the -state answered is one test. The suite is the contract. A driver that fails a conformance -test is not a driver. - -The `MemoryStore` driver is the oracle for the `HotdataStore` driver. A test that builds -the same records in both and compares `search` results proves the SQL the Hotdata driver -emits does what the in-memory driver computes. - -### 6.3 The frozen surfaces - -Four surfaces are frozen as set comparisons in tests. Changing any of them is a public -contract change that needs a changelog entry, and the test says so in its failure message. - -- The names in `__all__`. -- The method set of the `Store` protocol. -- The field set and the field types of the record, per schema version. -- The filter keys that `list` and `search` accept. - -A frozen set compares against a literal set. A test that parametrizes over the thing it -protects turns a deletion into one fewer test instead of a failure. - -### 6.4 The guarantees ledger - -`docs/guarantees.md` lists every guarantee with the test that proves it and the date it was -measured. A guarantee with no test is written as to measure. The ledger is the file a -consumer reads before relying on a behaviour. A test in the suite reads the ledger and -fails when a named test does not exist. - -### 6.5 The documentation checks - -Documentation is verified like code, with checks that need no judgement. - -- Every relative link in `README.md` and under `docs/` resolves to a file. -- Every command in the skill file in section 7 runs against the in-memory driver and exits - zero. -- Every guarantee row in the ledger names a test that exists, once the suite exists. - -### 6.6 The integration leg - -Tests marked `hotdata` run `HotdataStore` against a real engine. They need -`HOTMEMORY_TEST_URL` to name the engine. Each run provisions a throwaway database and -deletes it at the end. They skip when the variable is unset. They run once per pull request, at the end, because -one run costs minutes. They are the only tests that touch a network. - -### 6.7 What is not in the harness - -- No coverage gate. A number that reddens for unrelated reasons gets lowered under - pressure. -- No mutation testing as a gate. Mutation is a one-time exercise run by hand when a - surface is first frozen, to confirm that the freeze catches a change. -- No model in any test. `capture` takes a callable, and the tests pass a fake that - returns fixed facts. -- No bespoke report generator. The output of `make verify` is the report. - -## 7. The skill file - -Agents that use the library through a harness rather than through code get a skill: -`skills/hotmemory/SKILL.md` with a name and a description in front matter, the five memory -operations in prose with one example each, and a `scripts/` folder holding one -command-line entry point per operation. The script is the invocation and the skill says -when to use it. The skill file versions with the library, and section 6.5 runs every -command in it on each `make verify`, so the prose that reaches a model cannot drift from -the API. - -## 8. Out of scope for version 1 - -- A LangGraph `BaseStore` adapter. It lands in `hotdata-langchain` once phase 3 is stable. - The naming in section 3.2 already matches it, so the adapter is a thin one. -- An MCP server over the memory contract. -- A separate memory service in front of Hotdata. The library runs in the consumer's process - and the hosted API it needs is Hotdata's own. A service becomes necessary only for a - client in another language, for extraction that must run centrally, or for scope - enforcement above the API key. None of the three exists yet. -- Server-side extraction. `capture` takes a callable and that is all. -- A third driver. The local RuntimeDB stack covers offline development with the Hotdata - driver itself. -- Access control beyond scope filtering. -- A Rust implementation. The Rust client has no framework layer to build on. - -## 9. Stop and ask if - -- A phase 0 measurement shows that an index does not serve rows loaded after its build, - and the fix changes the storage contract. -- A guarantee in section 5 cannot be given a test that observes it. -- A frozen surface needs to change before version 1 ships. -- The first consumer needs a field that is not in the record. -- M5 shows the three-stage query is slower than a scan at every size, in which case the - index plan in section 3.4 is wrong. diff --git a/docs/internal/plan.md b/docs/internal/plan.md deleted file mode 100644 index 92336f8..0000000 --- a/docs/internal/plan.md +++ /dev/null @@ -1,437 +0,0 @@ -# Plan: the second workstream, memory accuracy - -Status: agreed, 2026-10-13. The owner agreed to each decision and to the target on -2026-10-13. Phase 8 closed on 2026-10-15 with PR #24. On 2026-10-16 the owner added -three phases after it: faster evaluation, the ranking with episode chunks, and a large -memory store. The later phases moved to 12 to 15. Phase 9 closed on 2026-10-16 with PR -#28. Phase 10 closed on 2026-10-19 with PR #30. On 2026-10-19 the owner paused this -workstream before phase 11, to work on other things first. On 2026-10-21 the owner -resumed it with phase 11, a large memory store. This file -holds the phases still to come, 11 to 15. Each phase becomes one GitHub issue and one pull request, worked in task order. When a phase closes, `roadmap.md` gets -its paragraph, and this file keeps only the phases still open. The plan of phase 5 is in -git history, at commit `4a6fb36`. The plan of phase 6, with the decisions taken during -its work, is at commit `8a7318d`. The plan of phase 7 is at commit `ab93da1`. The plan -of phase 8, with the decisions of its task 1, is at commit `b5443ac`. The plan of phase 9 -is at commit `2df6881`. The plan of phase 10 is at commit `a0145ee`. - -## Goal - -When this workstream closes, the memory arm meets the target below, and hotmemory is on -PyPI. At the start of the workstream, the memory arm scored 19 of 54 on sample A (35.2 -percent). After phase 8, it scores 45 of 54 on sample A and 45 on sample B, with -episodes after the facts. After phase 10, with chunks of 1,000 characters at a budget of -16,000, it scores 48 on sample A and 46 on sample B, at about 4,350 reader tokens. The retrieval arm scored 45 of 54 on sample A. - -The target (agreed) is a distance from the lowest published score of a memory layer with -a GPT-4o reader: Zep at 71.2 percent (read, not reproduced). The published range goes up -to 84.2 percent. - -| Level | Distance from 71.2 percent | Sample A and sample B | All 500 questions | -|---|---|---|---| -| Pass | At most 20 points | At least 28 of 54 on each | At least 51.2 percent | -| Goal | At most 10 points | At least 33 of 54 on each | At least 61.2 percent | - -The workstream closes on the pass level, measured in phase 14 on all 500 questions. The -sample counts guide the phases before it. A sample of 54 is uncertain by about 10 points, -so the number on all 500 questions decides. - -The library stays deterministic. No code in `src/hotmemory` calls a model. The extractor -and the embedder stay callables that the caller passes. The benchmark owns its extractor -in `bench/`. - -Latency is not a goal of this workstream. Each phase records the time of its searches and -writes, and does not try to reduce it. The roadmap lists the latency work under Deferred. - -## Where we start - -All numbers are from `docs/benchmarks.md`. The rows of sample A were observed on -2026-10-10, and the rows of sample B on 2026-10-14. The rows of phase 7 were observed on -2026-10-14, the rows of phase 8 on 2026-10-15, and the rows of phase 10 on 2026-10-16 -and 2026-10-18. - -| Item | Value | -|---|---| -| Retrieval, vector ranking, 419 questions | `recall_any@5` 0.9833, `recall_all@10` 0.9928 | -| Retrieval, fused ranking, 419 questions | `recall_any@5` 0.9690, `recall_all@10` 0.9570 | -| Sample of 54, full context | 39 correct, 113,689 reader tokens at the median | -| Sample of 54, retrieval arm, top 10 sessions | 45 correct, 29,878 reader tokens | -| Sample of 54, memory arm | 19 correct, 1,462 reader tokens | -| Memory misses, no stored fact held the answer | 19 of 35 | -| Memory misses, stored but ranked below 30 | 2 of 35, both ranked higher by the vector ranking | -| Memory misses, a needed fact was in the block | 14 of 35: 1 with every needed fact, 13 with some | -| Sample B, full context, retrieval, and memory arms | 43, 48, and 27 correct of 54 | -| Sample B memory misses, no stored fact held the answer | 16 of 27 | -| Sample B memory misses, stored but ranked below 30 | 0 of 27 | -| Sample B memory misses, a needed fact was in the block | 11 of 27: 1 with every needed fact, 10 with some | -| Phase 7, memory arm with `detail` and sources, sample A | 31 correct, 1,655 reader tokens at the median | -| Phase 7, memory arm with `detail` and sources, sample B | 32 correct, 1,664 reader tokens at the median | -| Phase 7 misses with `detail`, no stored fact held the answer | 16 of 23 on sample A, 14 of 22 on sample B | -| Phase 7 misses with `detail`, the block held some needed facts | 6 of 23 on sample A, 7 of 22 on sample B | -| Phase 8, facts only, the baseline of phase 8 | 29 on sample A, 31 on sample B, 1,715 reader tokens at the median | -| Phase 8, `episodes="search"` at 64,000 characters | 45 on sample A, 45 on sample B, 12,849 and 12,676 reader tokens | -| Phase 8, `episodes="search"` at 32,000 characters | 43 on sample A, 5,715 reader tokens | -| Phase 8 misses with `search` at 64,000 characters | 9 on each sample: 5 with no stored fact, and an evidence session in the block in 8 | -| Phase 10, retrieval, fused ranking less BM25 | the same top 10 as the vector ranking on 419 of 419 questions | -| Phase 10, memory arm, vector ranking, `search` at 64,000 characters | 46 on sample A, 13,405 reader tokens | -| Phase 10, chunks of 1,000 characters, `search` at 16,000 characters | 48 on sample A, 46 on sample B, 4,336 and 4,360 reader tokens | -| Phase 10 misses with chunks of 1,000 at 16,000 | 6 on sample A and 8 on sample B, 2 with no stored fact on each | -| Phase 8 run time on sample A, 4 workers, from `duration_s` in the result files | 1,145 s for facts only, 1,507 s for `search` at 64,000 characters, 1,133 to 2,085 s over the seven runs | - -Four facts about the memory arm, read from `bench/e2e.py` and `src/hotmemory/contract.py` -at the start of the workstream, shaped its first phases. Phases 8 and 10 changed facts 2 -and 4: - -1. The extractor is `extractor` in `bench/e2e.py`. It sends each session to - `gpt-4o-mini-2024-07-18` with a prompt variant from `VARIANTS` in `bench/prompts.py`, - and returns `Fact(kind="fact")` with a subject, content, and the source - `session:`. The hash is the first 12 hex digits of the SHA-256 of the session id. - From phase 7, the memory arm runs with `--extractor detail`. -2. `capture` stores facts only. It does not store the session, so the memory arm keeps no - session text in the store. From phase 8, `capture` with `episode` also stores the - session as an episode, and from phase 10 as chunks with `chunk_chars`. -3. `remember` derives each key from the subject and a hash of the content. A changed value - gets a new key, so the old fact stays current beside the new one. -4. `recall` searches the kinds `fact`, `profile`, and `procedure`, keeps k records, and - then applies `as_of` in the process. It never returns an episode. From phase 8, it adds - episodes after the facts when the caller passes `episodes`. - -## Decisions - -The owner agreed to each item on 2026-10-13. - -- Deterministic library (agreed). No model call enters `src/hotmemory`. A model-backed - extractor in an extra, and decision calls such as a classifier for supersede, wait in - Deferred until this workstream closes. -- Two samples (agreed). Sample A is the committed sample of 54, seed `20261010`, in - `bench/sample.json`. Each phase iterates on sample A. Sample B is a second sample of 54, - 9 of each type, with a new fixed seed, and with no question of sample A. A phase runs - sample B once, at its end, to check that a gain on A is not a fit to A. -- What counts as a gain (agreed). One question moves an arm by 1.9 points on 54 - questions. Two conditions make a change count. Sample A and sample B move the same way. - The diagnosis shows the targeted group of misses getting smaller. A phase reports every - variant it tried, not only the best one. -- Fixed models (agreed). The reader and the judge stay `gpt-4o-2024-08-06` at - temperature 0, with the reference prompts. The extractor stays - `gpt-4o-mini-2024-07-18`. From phase 7, its prompt is `detail`. Phase 7 tried no other - extractor model, because the owner named none. -- The full-context arm (agreed). It runs once on sample B, in phase 6, as the baseline - of B. No later phase runs it again on 54 questions. It costs the most, at about 114,000 - tokens per question. -- The cost limit (agreed in phase 5, kept). The runner prints an estimate and stops over - 50 dollars. Each phase records what it spent. -- The stack (kept). End-to-end runs use `HotdataStore` on the local stack, with the vector - ranking, its default from phase 10. A phase that - changes a driver runs `make integration` and keeps the oracle test of `MemoryStore` - against `HotdataStore`. -- Changes to a surface (kept). A change to a frozen surface updates `CHANGELOG.md`, - `docs/contracts.md`, and `docs/guarantees.md` in the same commit, and `MemoryStore` gets - the same behavior as `HotdataStore`. - -## Phase 11. A large memory store - -Status on 2026-10-22: open, on the branch `phase-11-large-store` from `main`. Tasks 1 to 6 -are done, and the full LongMemEval-S run is done. The numbers are in `docs/benchmarks.md`, -in the sections "The full benchmark" and "A large memory store". The work so far goes into -one pull request. No issue exists for this phase. The next task is task 7, the large store -on LongMemEval-M, in the form that the owner chose on 2026-10-22. - -Goal: proof that hotmemory keeps its accuracy when the memory layer is large. The owner set -the aim on 2026-10-21. A large store means many sensible memories in the layer, from -100,000 to 1,000,000, that recall can reach, and new memories added to it during the run. -It does not mean only many questions. The test that answers it runs samples A and B as in -phase 10, over a store that is filled first, and compares the scores with the phase 10 -scores of 48 of 54 and 46 of 54 from a fresh store. The full LongMemEval sets add variety to -the evaluation, and the scale test is the volume test of the layer. - -Two hypotheses to test. The owner's: hotmemory does well at 100,000 to 1,000,000 memories -now that the engine accepts SQL writes. The second: cues help retrieval on facts. In phase -10, the fused ranking lost to the vector ranking on episodes, which have no cues. - -### Decisions of the owner on 2026-10-21 - -- The work runs on one branch, with the doc fixes after phase 10 as its first commit. -- `provision` builds no vector index by default. `vector_index=True` is an experimental - option that builds the three vector indexes and searches through them. It stays, so the - index can come back when the engine updates an index as data arrives and fixes the - filtered search through it. -- The local stack runs the latest engine image, `sha256:75403e84eb47`, built 2026-10-19. -- Task 6 changes from "capture every question into one database" to the prefilled store - below, because a namespace that recall never reads cannot change an answer. -- The numbers go into the docs now, and the commit waits for the later numbers. - -### Decisions of the owner on 2026-10-22 - -- The background of task 6 holds only chunks of sessions that no history of samples A and - B holds. The cached facts stay out, because they copy the facts of the questions. -- `provision` builds no sorted index by default. `sorted_index=True` builds it. -- LongMemEval-S runs in full, on new databases, as the comparable score. -- The large store test moves to LongMemEval-M, where each question has one consistent user. - BEAM comes later. -- The runs move to a stable machine later. - -### What the phase found - -Each item was observed on 2026-10-21 unless marked. - -- The cloud accepts SQL `INSERT`. One row with a vector took 531 ms by `INSERT` and - 1,682 ms by load. At 10 rows they were equal. At 100 rows the load was five times - faster. At 1,000 rows the `INSERT` was refused with `Payload Too Large`, at 32 MB of SQL. - The engine refuses `UPDATE` and `DELETE`. -- The engine applies the scope filter inside `bm25_search`, before its depth, locally and - in the cloud. So BM25 loses no rows of a narrow scope in a shared table. The plan of - phase 9 to measure that loss is void. -- A plain vector index is the main cost of a write. A load of 1,000 rows into a table of - 10,000 rows took about 8.5 s with the index and about 1 s without it, in the cloud. -- The read of the current revisions before a write once sent one condition for each key. - On a flush of 2,020 rows, the local engine stopped with a stack overflow. The read now - uses lists of at most 500 keys. -- A query result holds at most 10,000 rows. A load that is in flight when its process - dies still commits. -- The local image of 2026-10-08 returned fewer than k rows for a vector search over 50,000 - rows or more, when the query selected `content`. The image of 2026-10-19 and the cloud - did not. -- The scale test, over one pool of every LongMemEval-S session as chunks, up to 254,025 - rows: the vector ranking matched a brute-force ranking at every size. `recall_any@10` - over the whole pool fell from 0.8588 at 10,000 rows to 0.4235 at 254,025 by vector, and - to 0.4824 fused. A scoped search stayed fast: 199 ms at 1 percent of 254,025 rows. A - flush of 1,000 rows grew from 1.3 s to 14.3 s, and the cause past the read was not - measured. -- LongMemEval-M retrieval, 419 questions, each in its own namespace of about 470 sessions: - `recall_any@5` 0.9189 fused and 0.9045 by vector. The two tied at `recall_any@10`, at - 0.9451. On LongMemEval-S the vector ranking scored higher on every metric. So the fused - ranking gains as a store grows. -- Transient upload failures, `the upload transfer to storage failed before any response`, - happened about once in every 100 cloud loads. -- Observed on 2026-10-21 and 2026-10-22: the sorted index on `created_at` made each write - into a table of 188,893 rows take 8 to 20 s, for 1 row as for 12 rows. BM25 alone took - about 1.4 s, and no index about 0.4 s. SQL `INSERT` paid the same as a load. Every write - was visible at once to a read by id, a newest-first query, and `bm25_search`. So the - engine rebuilds the index after each write [inferred]. This was the cause of the growth - of a flush in the scale test. -- Samples A and B over a store with 188,893 chunks of other sessions scored 39 and 35, - against 48 and 46 from a fresh store. Other users' chunks took the places of the - question's chunks, and contradicted its facts. `recall` applies `as_of` after the top k, - so 3 blocks were half empty. -- LongMemEval-S in full: 439 of 500, 87.8%. The 392 questions outside the samples scored - 88.0%. -- One write into the episode table was refused with `RESOURCE_LOCKED` after 8 tries in - about 16 s, with one writer only. The cause was not found. - -### Tasks - -1. Done, `9e96753`. SQL writes in the cloud: `scripts/measure_local.py --only m4`, with - flushes of 1, 10, 100, and 1,000 rows. The results are in `docs/guarantees.md`, M4. -2. Done, `59ac141`. `HotdataStore` writes a table by a guarded `INSERT` when its rows in - the write are all new and number at most 5 (`INSERT_ROWS`). The `INSERT` writes only - the ids that the table lacks, so a retry writes nothing twice. A refused `INSERT` makes - the store write by load from then on. -3. Done, `129e501`. `provision` reads every page of the database list, so - hotdata-dev/sdk-python-framework#89 no longer affects it. -4. Done, `d3300cc` and later. The scale harness, `bench/scale.py`. Results: - `scale-local-local-new-engine-436c7217.json` and - `scale-cloud-cloud-no-vector-index-ec9d7e68.json` in `bench/results/`. -5. Done, `f25d7ea`. Retrieval on LongMemEval-M: `python -m bench.retrieval --data m`. The - file is in `bench/data/`, 2.74 GB, SHA-256 `9d79e552...`, and the loader keeps only the - user turns. Result: `retrieval-m-hotdata-local-isolated-fused-vector.json`. -6. Done. The prefilled store: `bench/scale.py --prefill`, `bench/e2e.py --background`, - the time of each write in each row, and `bench/diagnose.py` for a background run. The - sorted index became an option. Samples A and B scored 39 and 35, and a diagnosis of 25 - misses explains the loss. Cost $1.56 for the runs and $0.30 for the diagnosis. -7. Done. LongMemEval-S in full: `bench/e2e.py --sample all`, after a replay of samples A - and B on new databases that gave 48 and 46 at $0. 439 of 500. Cost $14.84. -8. Next. The large store on LongMemEval-M. Two arms on samples A and B, each question with - its own namespace of about 500 sessions: the chunks alone, with no extracted facts, and - the full memory arm on sample A only. The full arm on both samples costs about $46 in - extractor calls [extrapolated], too close to the cap of $50. Write the cost and time - estimates before each run. -9. The cue test on facts. Compare the fused ranking, with cues, and the vector ranking on - facts only, on samples A and B. Up to about $2 for each sample if the fused ranking - changes the capture prompt. -10. Record the numbers in `docs/benchmarks.md`, `docs/guarantees.md`, and - `docs/internal/architecture.md`, close the phase in `roadmap.md`, and commit. - -### How to run and check - -- `make local-up` starts the local stack with a random `RUNTIMEDB_SECRET_KEY`. The engine - image of 2026-10-19 refuses an empty key. To swap only the engine, run - `RUNTIMEDB_SECRET_KEY="$(openssl rand -base64 32)" docker compose up -d --no-deps runtimedb`. -- Scale: `uv run --group bench --env-file .env python -m bench.scale --sizes 10000,100000`, - with `--location cloud`, `--keep`, `--vector-index`, and `--tag`. It writes its result - file after each size, and retries a flush three times after a transient upload error. -- Run one heavy job on the local engine at a time, or the timings mix. -- Kill a run with SIGINT, so that it deletes its database. SIGKILL leaves the database. A - run started with `&` from a shell that is not interactive ignores SIGINT. Stop such a - run with SIGTERM to its process group, then delete its worker databases by name. -- Ask the owner before you stop, restart, or delete anything that a run uses. -- The local databases `agent-memory` and `local-memory` stay untouched. Each held one - memory row and one meta row before and after the engine upgrade. - -### Not in this phase - -These are under Deferred in `roadmap.md`: - -- The vector rankings on the vector index by default. They wait for the engine's fix of - the filtered search. When it lands, the expected failure - `test_a_search_through_the_vector_index_finds_every_later_write` fails, and the index can - become the default. -- `UPDATE` and `DELETE` by SQL, until the engine has them. -- The comparison with other memory layers, such as MemoryBench. -- BEAM at 10 million tokens. - -## Phase 12. Supersede in capture - -Goal: when a session changes a fact, the old fact closes, so the reader sees one current -value. Today a changed value gets a new key, and both values stay current. - -The library decides nothing by itself. The extractor decides which fact a new fact -replaces, and the library applies that decision with the rules of `supersede`. - -Tasks: - -1. Propose the surface in `docs/contracts.md` and stop for the owner's agreement. The - proposal below is a starting point: - - `Fact` gains `replaces`, the key of a current record in the same scope. `remember` - writes such a fact as the next revision under that key, with `close_previous`, at - the `valid_from` of the fact. A key that is not current raises `ValueError`, and the - batch writes nothing. - - The extractor gets the keys of the current records that `capture` passes to it. - `capture` can also pass the `candidates` of each extracted fact, but that needs a - second extractor call, so it waits for the owner. -2. Build the agreed surface for both drivers, with conformance tests that use a fixed - extractor and no model: a changed value, a value that changes twice in one capture, a - `replaces` that names a closed record, and a retried capture. -3. The benchmark extractor prompt gets the keys, and it can set `replaces`. -4. Report the knowledge-update type on both samples, and count the facts closed per - question. A working supersede still moves the benchmark little. The judge - of LongMemEval accepts an answer that gives both the old value and the new one. -5. Add a small offline test set of update pairs to `bench/`, written by hand from - knowledge-update sessions. It checks that the extractor sets `replaces` where a value - changed, with the cached extractor replies. - -Acceptance criteria: - -- A changed value leaves one current record and one closed record with `valid_until` and - `expired_at`, in both drivers. -- `recall` with `as_of` before the change returns the old value only after phase 13. - This phase states that limit in `docs/contracts.md`. - -Stop and ask if: - -- The extractor sets `replaces` on records that did not change in more than one case of - ten in the offline test set. - -## Phase 13. Search over history - -Goal: `recall(as_of=T)` returns what was true at T, including a fact that a later -revision replaced. Today it reads current revisions only, and it applies `as_of` after -it keeps k records. - -Tasks: - -1. Write the rule in `docs/contracts.md` and stop for the owner's agreement. The rule - picks, for each key, the revision valid at T. A revision closed without - `close_previous` has no `valid_until`. The rule needs a second test for such a - revision, for example the `created_at` of the next revision. The proposal names that - test. -2. Move the as-of filter into the SQL of both rankings, so that k counts records valid at - T. `MemoryStore` gets the same order. -3. Add a caller-supplied date window to `recall` that filters records by their validity, - with no date parsing in the library. -4. Conformance tests for each case of the rule, on both drivers, and the oracle test. -5. Report temporal-reasoning and knowledge-update on both samples. Every session of a - LongMemEval question comes before the question, so this phase is expected to move the - benchmark little. Its proof is the conformance suite and the replay case of the first - consumer. - -Acceptance criteria: - -- A fact replaced after T is in the block of `recall(as_of=T)`, and a fact replaced - before T is not, in both drivers. -- If at least k records are valid at T, the search with `as_of` returns k records. - -Stop and ask if: - -- The rule needs a new record field or a new `Store` method. -- The as-of filter in SQL makes a local search slower than twice its time without the - filter. - -## Phase 14. The full benchmark and a second benchmark - -Goal: the final configuration has numbers on all 500 questions, and on a second -benchmark. - -Tasks: - -1. Run the retrieval arm and the memory arm on all 500 questions, with the configuration that - phases 7 to 13 agreed. Print the estimate first. The run needs the owner's yes, because - it costs about ten times a sample run. -2. Add a LoCoMo runner to `bench/`, as a smoke test of 10 conversations and 1,540 - questions. Its labels have known errors, which the research notes list. The report - states this. -3. Update `docs/benchmarks.md` and the README section "Benchmark", with the configuration - of each number. - -Acceptance criteria: - -- `docs/benchmarks.md` reports both benchmarks, each number with its configuration, and - the phase 5 numbers stay as the record of their configuration. -- On all 500 questions, the memory arm scores at least 51.2 percent, the pass level of - the goal. The report states whether it reaches the goal level of 61.2 percent. - -Stop and ask if: - -- An estimate is over 50 dollars. The full run needs the owner's yes either way. -- The memory arm scores below the pass level on all 500 questions. The owner then decides - whether the workstream closes or gets another phase. - -## Phase 15. Release on PyPI - -Goal: `pip install hotmemory` installs the first release. - -Tasks: - -1. Copy the release flow of the sibling repository `hotdata-langchain`: - - `RELEASING.md`, and `scripts/release.sh` with `prepare` and `publish`. - - A workflow that publishes on a `vX.Y.Z` tag through PyPI trusted publishing. - - A workflow that writes the GitHub Release from `CHANGELOG.md`. - - A check that refuses a version change without a matching changelog section. -2. Check the wheel and the sdist: no `bench/`, no tests, no `.env`, and no - `docs/internal/`. Make the README links absolute, so that they work on PyPI. -3. The owner configures trusted publishing on PyPI, creates the `pypi` environment on - GitHub, picks the version, and runs the publish. - -Acceptance criteria: - -- A clean environment installs the release with each extra, and the examples of the - README run. - -## Not in this workstream - -These wait, and `roadmap.md` lists them under Deferred: - -- Latency: a conditional write, text loaded only for ranked rows, and the per-request - time of the cloud. Phase 11 measures SQL writes and the search at scale. -- A model-backed extractor in an extra, and decision calls with a probability, such as a - classifier that decides supersede or routes a recall. -- The other ideas from the hindsight study of 2026-10-09: a chunk-hash delta on - re-capture, "no time" distinct from "omitted", word and number checks on entity names, - forget cascading to derived records, and a parquet export. -- Adapters: a LangGraph `BaseStore` adapter and an MCP server. - -## Verification - -- Per commit: `make verify`. -- Per commit that touches a driver: `make local-up`, then `make integration`. -- Each benchmark run writes its configuration and results to one file in `bench/results/`. - The numbers in `docs/benchmarks.md` come from those files. -- Before each pull request: the docs audit of the `pr-workflow` skill, and a search of the - diff for the names of private repositories and deployment details. - -## Questions for the owner - -- Phase 11: the yes to build and run task 6, the prefilled store. -- Phase 11: the cloud workspace for a run at 1,000,000 rows, and its cost limit apart - from the model spend. The cloud runs so far used the workspace of `.env`. -- Phase 11: a target for the search time, such as p95 in the cloud at 1,000,000 rows. - -- Phase 15: the first version number, and whether to publish to TestPyPI first. diff --git a/docs/internal/roadmap.md b/docs/internal/roadmap.md deleted file mode 100644 index 85c05a4..0000000 --- a/docs/internal/roadmap.md +++ /dev/null @@ -1,207 +0,0 @@ -# Roadmap - -Every phase of the work, one row each, with its status. Phases 0 to 5 were the first -roadmap, which reached the basic memory layer on 2026-10-12. This file changes when a -phase opens or closes, and not in between. The detail of the open phase lives in -`plan.md`. The design and its reasons live in `brief.md`. - -Rules. One phase is open at a time. A phase closes when every issue it opened is closed, or -is listed under Deferred below with a reason. The next phase opens only then. Each phase is -one pull request in each repository that it touches, and ends with the verification layer -that proves it. - -## The first roadmap: the basic memory layer - -| Phase | Delivers | Status | -|---|---|---| -| 0. Documents and measurements | The public contracts and guarantees files, CONTRIBUTING, the Makefile with the first documentation check, the local container setup, and measurements M1 to M6. | closed 2026-10-05, PR #2 | -| 1. The storage contract, offline | The record, the `Store` protocol, the in-memory driver, the conformance suite, the frozen surfaces, the ledger test, and CI. | closed 2026-10-06, PR #5 | -| 2. The Hotdata driver | `HotdataStore` with four tables, the serialized writer, the three-stage retrieval query, the sweeper, the integration leg against the local stack in CI, and the oracle test. | closed 2026-10-08, PR #7 | -| 3. The memory contract | `Memory` with remember, recall, candidates, supersede, forget, profile, and capture, the rendered blocks pinned by tests, and the skill file with its command check. | closed 2026-10-08, PR #9 | -| 4. The first consumer | A document loader in hotmemory. A post-mortem loader and a recall at investigation start in the consumer's own repository, proven by a replay case. Two pull requests, one in each repository. | closed 2026-10-10, PR #11 and the consumer's pull request | -| 5. A public benchmark | A LongMemEval-S runner measuring retrieval recall at k first and end-to-end accuracy second, with the numbers in `docs/benchmarks.md`. | closed 2026-10-12, PR #14 | - -## The second workstream: memory accuracy - -Agreed on 2026-10-13. `plan.md` holds the tasks, the acceptance criteria, and the stop -conditions of each phase. The library stays deterministic: no code in `src/hotmemory` -calls a model. Latency is not a goal, and its work waits under Deferred, except the -measurements at scale of phase 11. On 2026-10-16 the owner added phases 9 to 11, and the -later phases moved to 12 to 15. On 2026-10-19, after phase 10 closed, the owner paused the -workstream to work on other things first. On 2026-10-21 the owner resumed it with phase -11, to test hotmemory as a large memory store: recall from one store that holds 100,000 -memories or more. - -| Phase | Delivers | Status | -|---|---|---| -| 6. The accuracy loop | A second sample of 54 questions, and the memory arm configured from the command line. The diagnosis split for each question type, and the baselines of both samples. | closed 2026-10-14, PR #18 | -| 7. The extractor | A benchmark extractor that keeps what the assistant said, with sources, measured against the 19 misses where no stored fact held the answer. No library change. | closed 2026-10-14, PR #21 | -| 8. Episodes behind facts | `capture` stores its text as an episode that its facts cite. `recall` can add the cited or ranked episodes to its block, within one budget. | closed 2026-10-15, PR #24 | -| 9. Faster evaluation | The runner keeps the memory of a run and reuses it for runs that change only `recall` or the reader. A smoke run of 18 questions on `MemoryStore`. A run keeps its finished questions when it stops. No library change. | closed 2026-10-16, PR #28 | -| 10. The ranking, and episode chunks | An ablation of the fused ranking, and a default ranking chosen by the benchmark. `capture` stores a session as chunks, so the block holds the chunks that match and not whole sessions, for fewer reader tokens. | closed 2026-10-19, PR #30 | -| 11. A large memory store | Recall from one store that holds 100,000 memories or more. SQL `INSERT` measured in the cloud, and taken for appends if it is faster. The time of a write and a search at 10,000, 100,000, and 1,000,000 rows built from real text, by scope size, locally and in the cloud, with each top k checked against a brute-force ranking. Retrieval on LongMemEval-M. Samples A and B answered over a store prefilled with 100,000 or more memories, against their scores from a fresh store. LongMemEval-S in full: 439 of 500 on 2026-10-22. The large store on LongMemEval-M. A test of cues on facts. | open | -| 12. Supersede in capture | A fact can name the current record that it replaces, and `remember` closes that record. The extractor decides, and the library applies the rule. | proposed | -| 13. Search over history | `recall(as_of=T)` returns the revision valid at T, with the as-of filter in SQL, and a date window from the caller. The filter in SQL also keeps a block full when many records are dated after `as_of`, as phase 11 found. | proposed | -| 14. The full benchmark | The final configuration on all 500 LongMemEval-S questions, and a LoCoMo smoke test. The LongMemEval-S run of phase 11 is the first. | proposed | -| 15. Release on PyPI | The release flow of the sibling repository `hotdata-langchain`, and the first release. | proposed | - -## Deferred - -An entry here names the issue or the phase it came from, and the reason it is not being -taken up now. - -- Adapters, planned as phase 6 of the first roadmap: a LangGraph `BaseStore` adapter in - `hotdata-langchain`, then an MCP server. They wait until the memory contract scores well - and is on PyPI. -- Latency, from phase 5: a conditional write for more than one writer, text loaded only - for ranked rows, and the time of each cloud request. The second workstream measures - accuracy first. Phase 11 measures SQL writes and the search at scale. -- Model calls, from the discussion of 2026-10-13: a model-backed extractor in an extra, - and decision calls with a probability, for example to decide a supersede or to route a - recall. The library stays deterministic in the second workstream. -- More extractor prompts, from phase 7. The prompt `detail` cut the misses with no - stored fact from 19 to 16 on sample A, and the stop condition asked for 12 or fewer. - The owner accepted it on 2026-10-14. With `detail`, the extractor still drops what the - assistant said: 7 misses of the type single-session-assistant on sample A have no - stored fact. A second extractor model was not tried, because the owner named none. -- Precise chunk citations, from phase 10. A fact that `capture` writes with chunks cites - every chunk of its text, because the extractor does not say which chunk it came from. - So `episodes="cited"` still gives the whole text. A match of the fact to its chunk, or - an extractor that names it, can make `"cited"` save tokens too. Phase 10 measured - `"search"` only. -- The first-page listing of `hotdata-framework`, from phase 10. `list_managed_databases` - and `resolve_managed_database` read only the first 100 databases, and - `HotdataStore.provision` uses the first. It is filed as - hotdata-dev/sdk-python-framework#89. Phase 11 works around it in `provision`. -- The vector rankings on the vector index, from phase 2. A filtered search through the - vector index misses rows loaded after the build. Until it is fixed, the vector rankings - scan, and phase 11 measures that cost. -- SQL `UPDATE` and `DELETE`, from phase 11. The engine refuses them, so a closed - revision, `forget`, and `sweep` stay on keyed loads. -- The comparison with other memory layers in one harness, such as MemoryBench, from - phase 11. It needs an account and a cost for each service, and it does not test a - large store. -- BEAM at 10 million tokens, from phase 11, as a later test of one very large store. -- The ideas of the hindsight study of 2026-10-09 that the second workstream does not - take, for later planning: - - A chunk-hash delta on re-capture. - - "No time" kept distinct from "omitted". - - Word and number checks on entity names. - - A forget that cascades to derived records. - - A parquet export. - -## Closed phases - -A closed phase keeps its row above with the date it closed and the pull request. - -- Phase 0 closed on 2026-10-05 with PR #2, which closed issue #1. Its measurements are in - `docs/guarantees.md`. It changed the plan for later phases in three ways. The local - development target is a three-container stack (Postgres, RustFS, and the engine), not a - bare container. Fused retrieval needs plain vector indexes and an embedder from the - caller, because a provider-backed index excludes every other index. The integration leg - can run against the local stack in CI. -- Phase 1 closed on 2026-10-06 with PR #5, which closed issue #4. It changed the plan for - later phases in three ways. The record refuses more than the brief said: an empty label, - a label with `/`, a key with `/` or `@`, and empty content. Deduplication skips a - current revision past `forget_after`, so a forgotten fact can come back. Two questions - moved to phase 3: how `supersede` sets fields of the old record, and how a fact gains a - source when its content does not change. -- Phase 2 closed on 2026-10-08 with PR #7, which closed issue #6. It changed the plan for - later phases in four ways. The engine allows one vector index per table, so the cue - vectors have their own table. The engine refuses an index on an empty table, so - `provision` builds over a seed row. A filtered search through a vector index misses rows - loaded after the build, so the vector rankings filter after their fetch. The defect - also occurs in the cloud. One process writes to a - database until the engine has a conditional write. -- Phase 3 closed on 2026-10-08 with PR #9, which closed issue #8. It changed the plan for - later phases in four ways. The vector rankings of `HotdataStore` filter first and rank - by a scan, so the vector indexes serve no query until the engine fixes the filtered - index search. At 100,000 rows of 64 dimensions, a search in a 1 percent scope took 97 ms - locally. `put` gained `close_previous`, and a put of the same content with a new source - writes a merged revision. `recall` reads current revisions only, so a search over - history is still open. A count in the block of `profile` is a lower bound when a - sub-namespace fills the window of `list`. -- Phase 4 closed on 2026-10-10 with PR #11 and the consumer's pull request, which closed - issue #10. The consumer loaded 12 post-mortems, and counted 129 episodes and 256 facts - with SQL. A second load wrote nothing. It changed the plan for later phases in four ways. The pass - rule of the replay case judges the investigation loop from its trace, because the report - step drops the date of a rejected hypothesis. A consumer that renders its own block - passes `budget` and `k` to `recall`, because the default budget of 2,000 characters cut - the list first. A single-part upload in the Hotdata SDK does not retry on a closed - connection, so a long load can fail and must be run again. It is reported to the SDK - repository. A document that gets fewer chunks keeps its old last episodes current. -- Phase 5 closed on 2026-10-12 with PR #14, which closed issue #13. On LongMemEval-S, an - evidence session was in the top 5 for 98.3 percent of the 419 scored questions, with - the vector ranking. On a sample of 54 questions, the reader scored 39 with every - session, 45 with the top 10 sessions, and 19 with the block of `recall`. It changed the - plan for later phases in four ways. The extractor is the main loss of the memory - contract: in 19 of its 35 misses, no stored fact held the answer. On this data the - vector ranking scored higher than the fused ranking, and the reason was not measured. - With every question in one table, the fused ranking scored higher at the top 5 and - slightly lower at the top 10. A BM25 ranking inside the scope can wait. In the cloud, a - fused search took 998 ms at p50 and a vector search 361 ms. -- Phase 6 closed on 2026-10-14 with PR #18, which closed issue #16. A replay of sample A - from the caches reproduced 19 of 54 and each cause of each miss, at no model cost. On - sample B, the reader scored 43 with every session, 48 with the top 10 sessions, and 27 - with the block of `recall`. It changed the plan for later phases in four ways. The - memory arm differs by 14.8 points between the two samples with the same configuration, - so a change counts only if both samples move the same way. The extractor is the main - loss on both samples: no stored fact held the answer in 19 of 35 misses on sample A and - 16 of 27 on sample B. When the block held a needed fact, it held every needed fact in - only 2 of 25 misses. A fact ranked below 30 caused 2 misses on sample A and none on - sample B, so the ranking has little to gain on these samples. The runner gained - `--replay` and `--workers`, and a replay of the memory arm of sample A took 519 s with - 4 workers, against 2,264 s with one process. -- Phase 7 closed on 2026-10-14 with PR #21, which closed issue #19. With the extractor - prompt `detail`, the memory arm scored 31 of 54 on sample A and 32 on sample B, over - the pass level of 28 on each sample. The `baseline` prompt with sources scored 22 on - sample A. It changed the plan for later phases in five ways. Each fact of the benchmark - carries the source `session:` and a hash of the session id, because the block shows the - sources to the reader and LongMemEval names each evidence session `answer_...`. The - sources change the extractor prompt, so the extractor cache of phase 6 does not replay - with them. Most of the gain came from facts that are more complete: 9 of the 13 gains on - sample A were misses where the block of the baseline held some of the needed facts. The - misses with no stored fact went from 19 to 16 on sample A and from 16 to 14 on sample - B, and they are still the largest group. The marker of the diagnosis can miss facts, - so that group can be smaller than the diagnosis shows. A reply that reaches the token - limit gives no facts for its session, and a result file counts such sessions. -- Phase 8 closed on 2026-10-15 with PR #24, which closed issue #22. `capture` stores a - session as an episode that its facts cite, and `recall` adds episodes after the facts. - With `episodes="search"` at 64,000 characters, the memory arm scored 45 of 54 on sample - A and 45 on sample B, against 29 and 31 with facts only, at about 12,800 reader tokens - at the median. The retrieval arm scored 45 on sample A with 29,878 tokens. It changed - the plan for later phases in four ways. The misses with no stored fact went from 17 to - 5 on each sample, and in 8 of the 9 misses on each sample the block held an evidence - session, so the reader is now a larger loss than the extractor. The block shows a - source in its own namespace as `@`, so the run id stays out of every - prompt and a run replays `capture` from the cache. Each run captured the same memory - again for about 20 minutes, which led to phase 9. Whole sessions cost the tokens, which - led to the chunks of phase 10. An internet outage stopped four runs, and the local - engine refused one search with a stale BM25 index, which a second try did not show. -- Phase 9 closed on 2026-10-16 with PR #28, which closed issue #25. With `--keep`, the - runner keeps the memory of each question in a local database of its own, and a run with - `--reuse` recalls from it with no `capture`. On sample A, each reused run took 11 to 15 - seconds and equaled its phase 8 run on all 54 rows. The phase 8 runs took 1,133 to - 2,085 seconds. The smoke run of 18 questions on `MemoryStore` took 12.7 seconds from - the cache. Its first run cost $1.07 and took 2,080 seconds in one process, and it now - takes `--workers`. A run keeps its finished questions in a rows file, and `--resume` - completes it. It changed the plan for later phases in three ways. One database holds one - question, because BM25 ranks the whole table before the namespace filter. The same - order can cost a `recall` over a narrow scope in a shared table, so phase 11 measures - it. A commit that changes `src/hotmemory` makes the kept memory unusable, so a phase - that changes the library captures once again for each configuration. `MemoryStore` - ranks by vector only, so the first smoke run of a configuration pays for extraction. -- Phase 10 closed on 2026-10-19 with PR #30, which closed issue #26. An ablation on the - 419 retrieval questions showed that the BM25 arm made the whole gap between the - rankings: `recall_all@10` was 0.9570 fused, 0.9928 by vector, and 0.8926 for BM25 - alone. So `HotdataStore` ranks by vector by default, as `MemoryStore` does, and - `ranking="fused"` stays as an option. `capture` takes `chunk_chars` and stores a session - as chunks, and each fact cites every chunk. The block shows a run of chunk sources as one - range. With chunks of 1,000 characters at a budget of 16,000, the memory arm scored 48 - of 54 on sample A and 46 on sample B, at about 4,350 reader tokens at the median. Phase - 8 scored 45 on each with about 12,800 tokens. Phase 10 spent $12.77. It changed the plan - for later phases in three ways. The extractor reads a known-fact list of 2,000 - characters, so a longer block line changes its prompt, and a run with chunks paid for - part of its extraction again. A run that reuses kept memory at a new budget pays for - the reader and the judge. The framework listing reads only the first 100 databases, - which the runs worked around by deleting each set of kept databases before the next. diff --git a/docs/internal/survey.md b/docs/internal/survey.md deleted file mode 100644 index ff9cbb8..0000000 --- a/docs/internal/survey.md +++ /dev/null @@ -1,209 +0,0 @@ -# Survey: five memory systems, read against the brief - -Date: 2026-10-01. Each system was read for the same seven questions. Sources are shallow -clones beside this repo (`../mem0`, `../graphiti`, `../langmem`, `../letta-code`) and, for -supermemory, its public docs plus the client code in `../company-brain`. Version and commit -per system are in the first column. Every claim below was checked against a file and line. The -per-system notes with those references are not committed here. - -The purpose is to settle the contracts in `brief.md`, in this folder, not to match features. Section 4 -lists the changes the survey proposes, and section 5 answers whether to fork mem0. - -## 1. What each system is - -| System | What it is | Version read | -|---|---|---| -| mem0 | Python library. One extraction call per `add`, hash dedup, a vector store with an opaque payload, a SQLite history file. | 2.2.1, commit 94c3fe9, 2026-09-25 | -| Graphiti | Python library under Zep. A temporal knowledge graph: entities and facts as edges with world-time and system-time clocks, on Neo4j or FalkorDB. | commit 3c42764, 2026-09-30 | -| LangMem | Python library over the LangGraph `BaseStore`. Extraction managers that search, decide, and write through the store. | 0.0.30, langgraph 1.2.10 | -| supermemory | Hosted API. Documents in, server-side extraction and consolidation, memories partitioned by container tag. | docs and SDK 4.20 as used by company-brain | -| Letta (letta-code) | TypeScript coding agent. Memory is a git repository of Markdown files that the agent edits, with a small always-loaded root and the rest read on demand. | commit of 2026-09-30 | - -Two of the five changed shape recently, and the change is the finding. mem0 2.2.1 removed -the add, update, delete, none decision loop and the graph store. The add path is one -extraction call, an MD5 check against the ten nearest rows, and an insert. The old decision -prompt is still in the file and nothing calls it. Letta removed the MemGPT memory tools and -the archival vector store. Memory is Markdown files under git, and retrieval is grep plus a -hybrid search over the message transcript. Both moved away from model-driven consolidation -over a vector store, and toward simpler writes with the decision left to the caller or to a -later reflection pass. - -## 2. The seven questions - -### 2.1 Who extracts - -| System | Who extracts | What the caller passes | -|---|---|---| -| mem0 | The library, one model call with a fixed prompt and JSON schema. "When in doubt, extract." | Messages, the scope triple, optional metadata. `infer=False` stores raw text. | -| Graphiti | The library, two model calls, entities then facts, each fact with `valid_at` and `invalid_at` from the text. | An episode: name, body, source type, `reference_time`, `group_id`. | -| LangMem | The library through a manager, one model call with the existing memories in the prompt, Pydantic schemas per memory type. | Messages plus the existing records the manager found by search. | -| supermemory | The server, inside a pipeline called dreaming. The caller steers it with a per-container prose prompt. | A document: content, one container tag, a custom id, flat metadata. A direct memory write also exists. | -| Letta | The agent itself, with Edit and git, plus a reflection subagent every 25 steps. | Nothing. Memory is files the agent rewrites. | - -The brief's `capture` takes a callable and names no model. That matches Letta and the -`infer=False` path of mem0, and it is the right default for a library. What every extracting -system also passes, and the brief does not, is context: mem0 passes the ten nearest existing -memories and the last ten messages, LangMem passes the search hits as `existing`, Graphiti -passes the episode time and previous episodes. - -### 2.2 How a new fact meets an existing one - -| System | Mechanism | Who decides | -|---|---|---| -| mem0 | MD5 of the text against the ten nearest rows. A paraphrase is a new row. | Code. No model decision. | -| Graphiti | Exact and MinHash passes on names, cosine search at 0.6 over 15 candidates, then one model call that returns `duplicate_facts` and `contradicted_facts`. Invalidation is pure time arithmetic after that. | Code first, then a model, then code. | -| LangMem | Search top k, one model call that patches, inserts, or removes, concurrent puts. No conflict detection between runs. | The model. | -| supermemory | Hidden in dreaming. The client sees `updates`, `extends`, and `derives` relations, `isLatest`, and `history`. The client's steering prompt says "Hold exactly ONE current answer per subject". | The server's model. Not observable. | -| Letta | The reflection prompt says "fix the stale entry at the source. Do not append the new version alongside the old." | The agent. | - -The one rule all five share is that supersession keeps the old record. supermemory keeps it -with `isLatest=false`, Graphiti with `invalid_at` set, mem0 as a history row, Letta in git -history. The brief's revision row with `superseded_by` is the same shape. - -### 2.3 The temporal model - -| System | Fields | Point-in-time query | -|---|---|---| -| mem0 | `created_at`, `updated_at`, optional `expiration_date`. Time otherwise lives in the text, rewritten as absolute dates. | None. Expired rows are hidden unless asked for. | -| Graphiti | `created_at`, `valid_at`, `invalid_at`, `expired_at`, `reference_time`. Two clocks: when the world changed and when the system learned it. | Date filters on all four fields, OR of AND groups. Nothing excludes expired edges by default. | -| LangMem | `created_at`, `updated_at` on the store item. | None. | -| supermemory | `version`, `isLatest`, `forgetAfter`, `history`. No validity span. Truth time lives in the text as a date suffix. | None. | -| Letta | Git history. Relative dates are made absolute by the reflection pass. | `git log`. | - -Only Graphiti has a real temporal model, and it has the one thing the brief lacks: a second -clock. `invalid_at` is when the fact stopped being true, `expired_at` is when the store -learned that. A replay asks what memory held on a given day, and that is a question on -`expired_at` and `created_at`, not on validity. - -### 2.4 Scope - -| System | Partition | Enforced where | -|---|---|---| -| mem0 | `user_id`, `agent_id`, `run_id` in the payload. One is required. Metadata cannot override them. | The vector store query. The server checks authentication, not ownership of the id. | -| Graphiti | `group_id` on every node and edge. | A property filter on Neo4j. A separate database per group on FalkorDB. | -| LangMem | A namespace tuple. Prefix match is elementwise on the tuple. No `.` in a label. | The store. Tenant isolation is a convention. | -| supermemory | One container tag per document. One tag per search. Each tag is its own vector namespace. | The server. A cross-tag read is one call per tag, merged by the client. | -| Letta | One git repository per agent. Shared memory is a second repository, never in context. | The filesystem. | - -Nobody enforces scope above the store. The brief's answer, the database as the platform -boundary and the namespace as a library filter, is the same answer with a stronger lower -half. - -### 2.5 Retrieval and rendering - -| System | Search | Rendering into a prompt | -|---|---|---| -| mem0 | Semantic over-fetch, optional BM25, entity boost, sigmoid scoring, optional reranker. | None in the library. The proxy joins texts with newlines. | -| Graphiti | Cosine, BM25, graph traversal, reranked by RRF, MMR, cross-encoder, or node distance. Sixteen recipes. | A JSON context string with `valid_at` and `invalid_at`, no budget. | -| LangMem | `BaseStore.search` with a query, a filter, limit, offset. Score is cosine, higher is better. | The search tool returns item JSON. No budget. | -| supermemory | Hybrid, threshold 0.5 default, optional rerank, filters nested to 8 levels. A profile endpoint returns static, dynamic, and bucketed memories. | company-brain ignores the profile endpoint and renders its own ambient profile from list calls, with no cap. | -| Letta | grep over the files, hybrid search with RRF over the transcript. | The root files are compiled into the system prompt, under 65,536 characters total and 20,000 per file. | - -Only Letta has a budget, and only Letta has an always-loaded block. supermemory's profile -exists and its most serious client did not use it. - -### 2.6 Forgetting - -| System | Mechanism | -|---|---| -| mem0 | Hard delete with a history row. No TTL, no decay. | -| Graphiti | Nothing decays. Invalidation never deletes. `remove_episode` is the only delete. | -| LangMem | `delete` is a put with no value. TTL is a store option, refreshed on read. | -| supermemory | Soft delete with `isForgotten`. `forgetAfter` horizons set by the extractor. The client filters on every read. | -| Letta | Retired content moves to an `ARCHIVE.md`. Git keeps everything. | - -The brief's column plus sweeper matches supermemory's client-side filter. The hard delete -matches mem0 and Graphiti. - -### 2.7 Storage and whether SQL reaches it - -| System | Backend | SQL beside the consumer's data | -|---|---|---| -| mem0 | 24 vector store providers. pgvector stores `id, vector, payload JSONB`. History in a separate SQLite file. | Yes on pgvector, through JSONB paths. History cannot be joined. | -| Graphiti | Neo4j, FalkorDB, Kuzu, Neptune. | No. | -| LangMem | InMemoryStore, PostgresStore. | Yes on Postgres, through the store's own tables. | -| supermemory | Hosted. Postgres and Turbopuffer per code comments. | No. Export is a client-side markdown dump. | -| Letta | Git and Markdown. Transcript on the Letta server. | No. | - -So the README claim holds with one qualification. Two systems let a consumer reach memory -with SQL when they run on Postgres, but in both the memory fields sit inside a JSON column -and the history sits elsewhere. None stores memory as typed columns with validity spans in -the same engine as the consumer's data. - -### 2.8 Benchmarks - -The field measures itself on LongMemEval-S and LoCoMo. supermemory publishes a LongMemEval-S -table against Zep and an open suite named MemoryBench under MIT. Graphiti ships a -LongMemEval evaluation with a model judge and no numbers. mem0 points to a separate -benchmarks repository and claims a LoCoMo delta that this clone cannot reproduce. None of -these measures the question the first consumer asks, which is whether a prior incident's cause reaches the -first round of a repeat. The replay case in the brief stays the first proof. A LongMemEval -run is a later proof that the layer is a general memory and not only one consumer's feature. - -## 3. The fork question - -The owner asked whether forking mem0 onto a Hotdata backend gets the orchestration with fewer -steps. The code answers no, and a provider is not worth it either. - -The vector store is pluggable without a fork. `VectorStoreBase` has eleven abstract methods -and 24 providers. A Hotdata store can enter through the `langchain` provider, which wraps -any LangChain VectorStore, so the existing one in `hotdata-langchain` works today, or by -assigning the attribute after construction. The history store is not pluggable: it is one -SQLite class with no base class, no factory, and no configuration key, and it also holds the -last ten messages the extraction prompt reads. The model and embedder validators hold -literal provider lists, so a plain callable is not accepted anywhere. The license is Apache -2.0. - -What a provider buys is one extraction call and an MD5 check, which is the whole of mem0's -orchestration in 2.2.1. What it costs is mem0's contract: the scope triple as the only -partition, a fixed prompt and JSON schema, a required model and embedder at construction, -a history file beside the process, and an opaque payload in which `update` overwrites in -place. None of the brief's temporal or revision model fits in that payload, and that model is -the part that makes a memory table joinable. A fork keeps the same prompts and adds a -3,900-line `main.py` whose write path changed shape between versions. mem0 is a reference -for the extractor contract, and nothing more. - -## 4. Changes the survey proposes to the brief - -Eight changes, each tied to what was read. They are proposals until the owner accepts them. - -1. Add `expired_at` to the record beside `valid_until`. `valid_until` is when the world - changed and `expired_at` is when the store learned it. `supersede` sets both. (Graphiti) -2. Add `observed_at` to the record and to `capture`. It is the time of the source the record - came from, separate from `created_at` and `valid_from`. The extractor receives it and - `valid_from` defaults to it. A post-mortem loaded next year has `created_at` next year - and `observed_at` the incident date. (Graphiti, mem0) -3. Give the extractor context. `capture` passes the text, `observed_at`, and the current - records that `recall` returns for the scope, so the extractor can skip a known fact. - (mem0, LangMem) -4. Make `supersede` name the record it closes by key, and use Graphiti's closing rule: close - the old record at the new record's `valid_from`, and refuse when the old `valid_from` is - later. Write the as-of rule for `recall` down and test it, including the case where - `valid_from` is null. The library decides nothing by itself. (Graphiti, mem0) -5. State that deduplication is exact on normalized content, and that a paraphrase is a new - record. Add `candidates(fact, k)` to `Memory`, which returns near records with distances - so a caller's consolidator can decide. (Graphiti, mem0) -6. Make `profile` an always-loaded block. One rendered block per subject under a character - cap the consumer sets, each record with its validity span, ending with the namespaces and - counts that `recall` can reach so the agent knows to search. (Letta, company-brain) -7. Match the LangGraph `BaseStore` where it costs nothing. Rename `namespaces` to - `list_namespaces`. Let `search` with no query mean `list`. Reject `.` in a namespace - label. Match a namespace prefix on labels, not on a joined string. The adapter converts - distance to a higher-is-better score and routes every put through the writer because the - store's concurrent puts would hit the table lock. (LangMem) -8. Add an `actor` field for who wrote the revision, and derive a corroboration count from - the length of `sources`. `sources` says where a fact came from and not who recorded it. - (mem0, supermemory) - -Two things the survey confirms as right and not worth changing: the hard delete, and keeping -extraction out of the library. Two things it adds to the guarantees ledger without a code -change: a write is visible to the next `recall` and never inside the current turn, which is -Letta's rule, and `search` must prove its value on a real recall case, because Letta dropped -its vector archive for grep over indexed files and did not miss it. - -## 5. What was not read - -The Anthropic memory tool and Google's Memory Bank were covered in conversation from -documentation and not from code. Cognee and the A-MEM paper were not read. The trustcall -package that LangMem uses for patches was not in the clone, so the exact patch format is -unconfirmed. diff --git a/docs/local.md b/docs/local.md deleted file mode 100644 index 698ec6d..0000000 --- a/docs/local.md +++ /dev/null @@ -1,103 +0,0 @@ -# Run against a local RuntimeDB - -RuntimeDB is the Hotdata query engine. The library can use a local RuntimeDB in place of a -cloud workspace, with no API key and no cloud service. This page tells you how to start it -and how to point `HotdataStore` at it. The integration tests and the measurement scripts -also use the stack. - -## Start the stack - -You need Docker Desktop. Run this command from the repository root: - -```sh -make local-up -``` - -The target starts three containers from [compose.yaml](../compose.yaml): - -| Service | Image | Purpose | -|---|---|---| -| `catalog` | `postgres:17` | Holds the RuntimeDB catalog and the DuckLake metadata of each managed table. | -| `storage` | `rustfs/rustfs:latest` | S3-compatible object storage for the table data and for uploads. | -| `runtimedb` | `ghcr.io/hotdata-dev/runtimedb:latest` | The engine, on port 3000. | - -The target creates the `runtimedb` bucket in storage, starts the engine, and waits until -`GET /health` answers. To stop the stack and delete all of its data, run `make local-down`. -To download newer images, run `make local-pull`. - -To use a different port or engine image, set the make variables: - -```sh -make local-up LOCAL_PORT=3100 RUNTIMEDB_IMAGE=runtimedb:local -``` - -Port 9000 on the host must be free, because storage always uses it. - -Warning: do not expose this stack to a network. The engine accepts every request with no -token, and the storage credentials are fixed and public. - -## Why three containers - -A managed table needs two services that the bare engine image does not have. - -- A Postgres catalog. RuntimeDB keeps the DuckLake metadata of a managed table in Postgres. - DuckLake metadata records which parquet files make up each version of the table. With - a Postgres `[catalog]`, the engine derives the DuckLake address from it. The bare image - uses a SQLite catalog, so it refuses to create a managed database with - `managed catalogs require ducklake.metadata_pg_url to be configured`. -- Object storage that can presign. The framework uploads parquet through presigned URLs. - A presigned URL is a storage address, signed by the engine, to which the client sends - the file directly. Filesystem storage cannot presign, and a load from the bare image - fails with `PRESIGN_UNSUPPORTED`. - -A presigned URL carries the host name of the storage endpoint. The engine and the client -on the host must reach the same name. The `runtimedb` service shares the network namespace -of `storage`, so `127.0.0.1:9000` is the storage service both inside the engine and on the -host. - -## Point the library at it - -Set these three variables in the shell that runs the library: - -```sh -export HOTDATA_API_URL=http://localhost:3000 -export HOTDATA_WORKSPACE=local -export HOTDATA_API_KEY=local -``` - -- `HOTDATA_API_URL` sends each API call to the local engine. -- `HOTDATA_WORKSPACE` can be any value. If it is set, the framework does not ask for the - workspace list, which the local engine does not serve. -- `HOTDATA_API_KEY` can be any value that is not empty. The engine does not read it, but - `HotdataClient.from_env()` in `hotdata-framework` refuses an empty key. This statement comes from the framework source. - -Then `HotdataStore.provision` creates or opens a database on the local engine. This -example uses a toy embedder, so it needs no model: - -```python -from hotmemory.hotdata import HotdataStore - - -def embed(texts): - return [[text.count("disk") + 0.1, text.count("cpu") + 0.1] for text in texts] - - -store = HotdataStore.provision("local-memory", embedder=embed, model="toy", dimensions=2) -store.put(("team", "alerts"), "disk", kind="fact", content="The disk fills at night.") -print(store.get(("team", "alerts"), "disk").id) # team/alerts/disk@1 -``` - -`make integration` runs the tests marked `hotdata` and the conformance suite against the -stack. It sets `HOTMEMORY_TEST_URL` to `http://localhost:3000`, and needs none of the -variables above. To run the tests against another engine, set `HOTMEMORY_TEST_URL`. - -`scripts/measure_local.py` needs none of these. It reads `HOTMEMORY_LOCAL_URL`, which -defaults to `http://localhost:3000`. - -## What works locally - -Measurement M6 in [guarantees.md](guarantees.md) records each call. On 2026-10-05, every -call that the driver needs worked against this stack, except one. A provider-backed vector -index fails with `Embedding provider 'sys_emb_openai' not found`, because the stack -configures no embedding provider. A plain vector index, over embeddings that the caller -supplies, works. diff --git a/pyproject.toml b/pyproject.toml index ed09539..734ac1e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -74,4 +74,4 @@ ignore_missing_imports = true [tool.pytest.ini_options] testpaths = ["tests", "bench/tests"] addopts = ["--strict-markers"] -markers = ["hotdata: runs against a RuntimeDB engine; skips unless HOTMEMORY_TEST_URL is set"] +markers = ["hotdata: runs against a Hotdata engine; skips unless HOTMEMORY_TEST_URL is set"] From 067676b836b6b1396422911fda6448d314cd71dc Mon Sep 17 00:00:00 2001 From: Rohan Dsouza Date: Fri, 23 Oct 2026 09:33:58 +0530 Subject: [PATCH 4/7] Ignore the local state of Claude Code, and raise the pyarrow floor of the measurement scripts The scripts declare pyarrow 16.0, as the hotdata extra does. .gitignore lists the worktrees, settings.local.json, and the lock file of Claude Code, so the rest of .claude/ can be shared. --- .gitignore | 5 +++++ scripts/measure_cloud.py | 2 +- scripts/measure_local.py | 4 ++-- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 0ab03b2..1a47d06 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,11 @@ bench/data/ bench/cache/ bench/results/ +# Local state of Claude Code. The rest of .claude/ (agents, skills, settings.json) is shared. +.claude/worktrees/ +.claude/settings.local.json +.claude/scheduled_tasks.lock + # Notes of the owner, kept out of git: the plan, the roadmap, the design brief, the # research, and the local stack. docs/internal/ diff --git a/scripts/measure_cloud.py b/scripts/measure_cloud.py index 8a71235..3af000b 100644 --- a/scripts/measure_cloud.py +++ b/scripts/measure_cloud.py @@ -2,7 +2,7 @@ # requires-python = ">=3.11" # dependencies = [ # "hotdata-framework>=0.14.1", -# "pyarrow>=14.0", +# "pyarrow>=16.0", # ] # /// """Run measurements M1, M2, and M3 against a throwaway Hotdata cloud database. diff --git a/scripts/measure_local.py b/scripts/measure_local.py index e030cec..5371a3a 100644 --- a/scripts/measure_local.py +++ b/scripts/measure_local.py @@ -2,10 +2,10 @@ # requires-python = ">=3.11" # dependencies = [ # "hotdata-framework>=0.14.1", -# "pyarrow>=14.0", +# "pyarrow>=16.0", # ] # /// -"""Run measurements M6, M4, and M5 against a local RuntimeDB container or the cloud. +"""Run measurements M6, M4, and M5 against a local engine container or the cloud. By default the script targets a running local container. It does not start the container. Start it with `make local-up`. With --cloud, it targets the Hotdata cloud API From 2d5eee66c5eca866397370afaf83d1f38530c98c Mon Sep 17 00:00:00 2001 From: Rohan Dsouza Date: Fri, 23 Oct 2026 10:06:43 +0530 Subject: [PATCH 5/7] README: how to install the skill The package does not include skills/hotmemory/, so the README tells the reader to install hotmemory and copy the folder into the skills folder of the agent. --- README.md | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 5e73f1d..0eddfbb 100644 --- a/README.md +++ b/README.md @@ -118,7 +118,30 @@ episode = memory.store.get(("team", "alerts"), "disk-incident-0001") print(episode.content.splitlines()[0]) # # Disk incident ``` -[skills/hotmemory/SKILL.md] gives an agent the same operations as commands. +## Use the skill + +[skills/hotmemory/SKILL.md] gives an agent the same operations as commands. The skill is +a folder with the skill file and one script for each operation. The package does not +include it, so you copy it from the repository. + +1. Install hotmemory in the Python that the agent runs. The extras are necessary only for + a Hotdata database: + + ```sh + pip install "hotmemory[hotdata,openai]" + ``` + +2. Copy the skill folder into the skills folder of the agent. For Claude Code, this is + `~/.claude/skills/` for every project, or `.claude/skills/` in one project: + + ```sh + git clone --depth 1 https://github.com/hotdata-dev/hotmemory + cp -r hotmemory/skills/hotmemory ~/.claude/skills/ + ``` + +The agent runs the scripts from the skill folder. With `--memory-file`, they need no +network and no key. With `--database`, they read the Hotdata and OpenAI keys from the +environment. ## Store memory in Hotdata From 3522d1f1ba9768e6d74850926bd829e502eac0f2 Mon Sep 17 00:00:00 2001 From: Rohan Dsouza Date: Fri, 23 Oct 2026 10:08:44 +0530 Subject: [PATCH 6/7] README: name no specific coding agent in the skill steps --- README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 0eddfbb..13b77f7 100644 --- a/README.md +++ b/README.md @@ -131,12 +131,13 @@ include it, so you copy it from the repository. pip install "hotmemory[hotdata,openai]" ``` -2. Copy the skill folder into the skills folder of the agent. For Claude Code, this is - `~/.claude/skills/` for every project, or `.claude/skills/` in one project: +2. Copy the skill folder into the folder where your coding agent reads skills. Many agents + use `~/./skills/` for every project, or `./skills/` in one project. The + documentation of your agent gives the correct folder. ```sh git clone --depth 1 https://github.com/hotdata-dev/hotmemory - cp -r hotmemory/skills/hotmemory ~/.claude/skills/ + cp -r hotmemory/skills/hotmemory / ``` The agent runs the scripts from the skill folder. With `--memory-file`, they need no From 5b41ebb7d3ec9504a772dee53685c6adcdb44e27 Mon Sep 17 00:00:00 2001 From: Rohan Dsouza Date: Fri, 23 Oct 2026 10:20:58 +0530 Subject: [PATCH 7/7] publish.yml stops when the tagged commit is not on main --- .github/workflows/publish.yml | 8 ++++++++ RELEASING.md | 7 ++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 42a67b4..d6ec1f2 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -18,6 +18,14 @@ jobs: timeout-minutes: 10 steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + fetch-depth: 0 + - name: Check that the tagged commit is on main + run: | + if ! git merge-base --is-ancestor "$GITHUB_SHA" origin/main; then + echo "The tag $GITHUB_REF_NAME points at $GITHUB_SHA, which is not on main" >&2 + exit 1 + fi - uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v6 with: python-version: "3.12" diff --git a/RELEASING.md b/RELEASING.md index 955d385..fb6e1f7 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -40,8 +40,8 @@ The publish uses PyPI trusted publishing. PyPI accepts the upload from the workf The tag starts the workflow `publish.yml`. It runs three jobs in order: -1. `build` makes sure that the tag matches the version in `pyproject.toml` and that - `CHANGELOG.md` has a section for the version. Then it builds the wheel and the sdist with +1. `build` makes sure that the tagged commit is on `main`, that the tag matches the + version in `pyproject.toml`, and that `CHANGELOG.md` has a section for the version. Then it builds the wheel and the sdist with `uv build` and checks them with `twine check --strict`. 2. `pypi` uploads the wheel and the sdist to PyPI. It runs in the environment `pypi`. 3. `github-release` creates the GitHub Release. The release notes are the changelog section @@ -57,7 +57,8 @@ cannot be uploaded again. To fix a bad release, release a new version. `CHANGELOG.md` has no section for the new version, it fails. - If the Unreleased section is empty, `release.sh prepare` stops. - If the tag exists or the changelog section is missing, `release.sh publish` stops. -- If the tag and the version differ, the `build` job stops. +- If the tagged commit is not on `main`, or the tag and the version differ, the `build` + job stops. ## What the package contains