From 581e788ac2e83b994e59e48384faf7a981368253 Mon Sep 17 00:00:00 2001 From: Ayla Croft Date: Thu, 8 Oct 2026 03:54:42 -0400 Subject: [PATCH] feat(s133): embedding spaces, operator pinning and the static floor Slice 133, to G3 apart from what waits on the owner: the legal review of the static weights (D7), and the label audit before the eval's scored run. - Vectors move out of memories into memory_embeddings, keyed (memory_id, space_id). Each space is an embedding_spaces row holding its 14-field manifest, identified by SHA-256 over RFC 8785 JSON. On Postgres the column is an untyped vector with one partial HNSW index per space. Existing vectors migrate under a legacy space with their bytes unchanged. - One active space per store, moved only by the operator: trinity.space.retier builds the new space beside the old and moves the pointer in one transaction, and trinity.space.drop needs --confirm and refuses the active space. A row that does not fit the active space makes the store refuse rather than rank. - Locality is declared: in_process, within_boundary or external. Under :regulated a fault in configuration refuses the boot; a runtime fault turns semantic memory off with its reason while full-text search keeps answering. - Trinity.Memory.Embedders.Static is a pure-Elixir WordPiece tokenizer and EmbeddingBag mean pool, with no NIF. Its weights are a derived artifact whose digest is pinned in code. The weights are not in the tree. - The int8 scorer, the sign and Hamming prefilter, and their measurement (docs/perf.md): prefilter p95 10.3 ms and exact p95 24.5 ms at 10^4 x 256, under the 50 ms line, so no scoring NIF. - TRINITY_WITHOUT_ML=1 builds without nx, bumblebee and exla, and that release boots, embeds and recalls. - The eval harness, corpus pipeline and label freezing. The scored run refuses until the owner's audit is done. Weight-dependent tests are tagged static_weights, excluded when TRINITY_STATIC_MODEL_DIR is unset and failing when it is set and the weights are absent. Red evidence R1 to R9 is in the slice NOTES. Signed-off-by: Ayla Croft --- CHANGELOG.md | 25 + README.md | 9 +- config/config.exs | 13 + docs/01-architecture.md | 14 +- docs/02-tech-stack.md | 1 + docs/05-data-model.md | 28 +- docs/perf.md | 55 + lib/mix/tasks/trinity.eval.corpus.ex | 29 + lib/mix/tasks/trinity.eval.freeze.ex | 26 + lib/mix/tasks/trinity.eval.memory.ex | 64 + lib/mix/tasks/trinity.space.drop.ex | 44 + lib/mix/tasks/trinity.space.list.ex | 34 + lib/mix/tasks/trinity.space.retier.ex | 60 + lib/mix/tasks/trinity.static.build.ex | 137 + lib/trinity.ex | 8 + lib/trinity/application.ex | 12 + lib/trinity/eval.ex | 12 + lib/trinity/eval/corpus.ex | 503 +++ lib/trinity/eval/labels.ex | 216 + lib/trinity/eval/memory_run.ex | 400 ++ lib/trinity/memory.ex | 11 + lib/trinity/memory/embedder.ex | 85 +- lib/trinity/memory/embedder_config.ex | 109 + lib/trinity/memory/embedders/bumblebee.ex | 18 + lib/trinity/memory/embedders/fake.ex | 42 +- lib/trinity/memory/embedders/hosted.ex | 36 + lib/trinity/memory/embedders/static.ex | 273 ++ lib/trinity/memory/entry.ex | 17 +- lib/trinity/memory/observer.ex | 38 +- lib/trinity/memory/re_embed_worker.ex | 31 + lib/trinity/memory/retriever.ex | 41 +- lib/trinity/memory/scorer.ex | 136 + lib/trinity/memory/semantic.ex | 429 +- lib/trinity/memory/space.ex | 212 + lib/trinity/memory/space_row.ex | 29 + lib/trinity/memory/spaces.ex | 394 ++ lib/trinity/memory/static_artifact.ex | 191 + lib/trinity/memory/vector.ex | 25 + lib/trinity/memory/vector_store.ex | 71 +- lib/trinity/memory/vector_stores/brute.ex | 127 +- lib/trinity/memory/vector_stores/pgvector.ex | 134 +- lib/trinity/memory/word_piece.ex | 254 ++ lib/trinity/memory/word_piece/tables.ex | 3684 +++++++++++++++++ lib/trinity/smoke.ex | 4 + mix.exs | 40 +- ...0261008120000_create_memory_embeddings.exs | 257 ++ scripts/eval/reference_vectors.py | 88 + scripts/scorer_bench.exs | 101 + scripts/static/reference_fixtures.py | 200 + scripts/static/wordpiece_tables.py | 144 + scripts/static_floor_release.sh | 66 + scripts/vector_bench.exs | 40 +- test/property/space_identity_test.exs | 110 + test/release/static_floor_release_test.exs | 52 + test/smoke_test.exs | 3 +- test/support/data_case.ex | 3 + test/support/excluded_counter.ex | 54 + test/support/fake_embedder_b.ex | 39 + test/support/spaces_helper.ex | 27 + test/support/static_weights.ex | 120 + test/test_helper.exs | 14 +- test/trinity/embedder_boot_node_test.exs | 281 ++ test/trinity/eval/memory_eval_test.exs | 243 ++ test/trinity/memory/embedder_config_test.exs | 98 + test/trinity/memory/embedder_test.exs | 3 +- test/trinity/memory/migration_test.exs | 298 ++ test/trinity/memory/no_nif_test.exs | 113 + test/trinity/memory/no_nif_xref_test.exs | 92 + test/trinity/memory/observer_test.exs | 4 +- test/trinity/memory/pgvector_spaces_test.exs | 161 + test/trinity/memory/pinning_test.exs | 105 + test/trinity/memory/retier_test.exs | 254 ++ test/trinity/memory/runtime_faults_test.exs | 136 + test/trinity/memory/spaces_test.exs | 136 + test/trinity/memory/static_floor_test.exs | 182 + test/trinity/memory/static_parity_test.exs | 109 + test/trinity/memory/thresholds_test.exs | 70 + test/trinity/memory/vector_store_test.exs | 98 +- .../trinity/memory/word_piece_parity_test.exs | 94 + test/trinity/memory/word_piece_test.exs | 73 + test/trinity/scheduler/curator_test.exs | 25 +- test/trinity/tools/recall_test.exs | 5 + test/trinity_web/live/semantic_tab_test.exs | 5 +- 83 files changed, 11664 insertions(+), 360 deletions(-) create mode 100644 lib/mix/tasks/trinity.eval.corpus.ex create mode 100644 lib/mix/tasks/trinity.eval.freeze.ex create mode 100644 lib/mix/tasks/trinity.eval.memory.ex create mode 100644 lib/mix/tasks/trinity.space.drop.ex create mode 100644 lib/mix/tasks/trinity.space.list.ex create mode 100644 lib/mix/tasks/trinity.space.retier.ex create mode 100644 lib/mix/tasks/trinity.static.build.ex create mode 100644 lib/trinity/eval.ex create mode 100644 lib/trinity/eval/corpus.ex create mode 100644 lib/trinity/eval/labels.ex create mode 100644 lib/trinity/eval/memory_run.ex create mode 100644 lib/trinity/memory/embedder_config.ex create mode 100644 lib/trinity/memory/embedders/static.ex create mode 100644 lib/trinity/memory/re_embed_worker.ex create mode 100644 lib/trinity/memory/scorer.ex create mode 100644 lib/trinity/memory/space.ex create mode 100644 lib/trinity/memory/space_row.ex create mode 100644 lib/trinity/memory/spaces.ex create mode 100644 lib/trinity/memory/static_artifact.ex create mode 100644 lib/trinity/memory/vector.ex create mode 100644 lib/trinity/memory/word_piece.ex create mode 100644 lib/trinity/memory/word_piece/tables.ex create mode 100644 priv/repo/migrations/20261008120000_create_memory_embeddings.exs create mode 100644 scripts/eval/reference_vectors.py create mode 100644 scripts/scorer_bench.exs create mode 100644 scripts/static/reference_fixtures.py create mode 100644 scripts/static/wordpiece_tables.py create mode 100755 scripts/static_floor_release.sh create mode 100644 test/property/space_identity_test.exs create mode 100644 test/release/static_floor_release_test.exs create mode 100644 test/support/excluded_counter.ex create mode 100644 test/support/fake_embedder_b.ex create mode 100644 test/support/spaces_helper.ex create mode 100644 test/support/static_weights.ex create mode 100644 test/trinity/embedder_boot_node_test.exs create mode 100644 test/trinity/eval/memory_eval_test.exs create mode 100644 test/trinity/memory/embedder_config_test.exs create mode 100644 test/trinity/memory/migration_test.exs create mode 100644 test/trinity/memory/no_nif_test.exs create mode 100644 test/trinity/memory/no_nif_xref_test.exs create mode 100644 test/trinity/memory/pgvector_spaces_test.exs create mode 100644 test/trinity/memory/pinning_test.exs create mode 100644 test/trinity/memory/retier_test.exs create mode 100644 test/trinity/memory/runtime_faults_test.exs create mode 100644 test/trinity/memory/spaces_test.exs create mode 100644 test/trinity/memory/static_floor_test.exs create mode 100644 test/trinity/memory/static_parity_test.exs create mode 100644 test/trinity/memory/thresholds_test.exs create mode 100644 test/trinity/memory/word_piece_parity_test.exs create mode 100644 test/trinity/memory/word_piece_test.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d5f19a..47816ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,31 @@ evidence for each increment is retained by the maintainers and summarised here. tree, engineering rules consolidated into `docs/03-conventions.md`, and this changelog added as the public record of delivered work. +### `slice/133`: semantic memory without native code, and vectors that know where they came from + +Semantic recall used to need EXLA, a 463 MB native library that does not load in the Linux desktop +bundle and has no Windows build, so on those builds it was simply off. This adds a floor that needs +no native code at all: a static embedding model (`static-retrieval-mrl-en-v1`, 256 dimensions, int8, +7.8 MB of weights) run in plain Elixir, with its own implementation of the BERT tokenizer that gives +the reference library's token ids on 5,000 natural sentences and 1,000 hostile ones. Its vectors +match the reference implementation's to a cosine of 0.9999 or better on 500 fixtures, and scoring +10,000 of them takes 10 to 25 ms at the 95th percentile on the owner's machine, depending on the +method. A release built without the neural libraries boots, embeds and recalls through it. The +weights are not distributed with Trinity while their licence and training-data terms are reviewed. + +Every stored vector now belongs to a named embedding space, identified by a digest over fourteen +fields (model, revision, weights and tokenizer digests, width, pooling, quantization, prompts, +runtime, declared locality and more), in a table of its own. A store answers from exactly one space, +chosen by the operator: configuring a different model turns semantic recall off with a reason +instead of quietly finding nothing, moving to a new model is an explicit re-embed built beside the +old vectors, and the old vectors stay until the operator confirms their removal. A store row that +does not fit its space makes recall refuse rather than rank. An embedder that reaches an endpoint +must declare whether it is inside the deployment's boundary; a regulated node refuses to boot on a +missing or wrong declaration, and full-text recall keeps working through every runtime fault. + +The evaluation that decides the default model is built and waiting: a generated corpus, 300 queries, +and labels that the scored run will not read until 100 of them have been audited and frozen. + ## 2026-09-26 ### `slice/128`: the checks that guard attribution now say what they check diff --git a/README.md b/README.md index f53a544..a891e8d 100644 --- a/README.md +++ b/README.md @@ -221,9 +221,12 @@ one of its tools asks for input mid-call, is answered there too), `/tasks` the s their results, `/oban` the jobs (in development, or when configured), `/settings` the export. `mix trinity.export` and `mix trinity.import` do what `/settings` does from a terminal; `docs/backup.md` explains the archive. -Semantic recall needs the local embedding model (91 MB, downloaded from the memory page on your -say-so, never on its own). Without it the tier is off and full-text search still works; -`docs/perf.md` has what it costs. +Semantic recall needs a local embedding model: all-MiniLM-L6-v2 (91 MB, downloaded from the memory page on your +say-so, never on its own) or the static floor (a 7.8 MB weights file, pure Elixir, no native code, installed +under the data directory; not yet distributed with Trinity while its licence is reviewed). Without one the tier +is off and full-text search still works. Every stored vector belongs to one named embedding space, and a store +answers only from the space it is pinned to: changing models is an explicit re-embed (`mix trinity.space.retier`), +never a silent switch. `docs/perf.md` has what it costs. `mix gate` runs the full quality gate and has to pass before every commit: format check, compile with warnings as errors, Credo, Sobelow, dependency audits, version verification, the naming and diff --git a/config/config.exs b/config/config.exs index 7e20d42..0ad8467 100644 --- a/config/config.exs +++ b/config/config.exs @@ -234,6 +234,19 @@ config :logger, :default_formatter, # Use Jason for JSON parsing in Phoenix config :phoenix, :json_library, Jason +# Slice 133: the static floor is the embedder where the neural group cannot run. A build without +# it (`TRINITY_WITHOUT_ML=1`, mix.exs) has nothing else in process; the Linux desktop bundle +# leaves exla out (its NIF never loaded in Burrito's musl ERTS) and a Windows host never declares +# it. Chosen at build time, the way mix.exs composes exla per `BURRITO_TARGET`: this is the +# configuration such a build ships with, not a fallback, and the store still answers only from +# the space it is pinned to. Until the weights may be bundled (slice 133 NOTES, D7) these builds +# report `:weights_missing` unless an operator installs the artifact (`Embedders.Static`). +# Everywhere else the default stays `:local` (MiniLM) until the eval decides (D5). +if System.get_env("TRINITY_WITHOUT_ML") == "1" or + System.get_env("BURRITO_TARGET") in ["linux_x86_64", "windows_x86_64"] do + config :trinity, :memory, embedder: :static +end + # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. import_config "#{config_env()}.exs" diff --git a/docs/01-architecture.md b/docs/01-architecture.md index 7744b6c..63a2314 100644 --- a/docs/01-architecture.md +++ b/docs/01-architecture.md @@ -88,7 +88,7 @@ without anything failing. | `Trinity.Effects` | The membrane; the runner in force (`Effects.Runner`, the executor `Tools.Runner` takes as a function); decision and query receipts; the boot receipt | **Tools**, Permissions, Authority, Receipts, Repo | | `Trinity.Authority` | Behaviour; `Local` implementation (the one caller of `execute/2` for effectful tools); selection at boot; `Staged` | Receipts, Repo | | `Trinity.Receipts` | Local chain (one supervised writer per scope, ADR-0013), the signer seam (Ed25519, P-384, ML-DSA-87), key custody and the registry, checkpoints, the verifier, the alarm | Repo (`Repo.Receipts`) | -| `Trinity.Memory` | Always-on tiers with their budget and consolidator (030), search (031), semantic store and retrieval (032), compaction (023) | LLM (summaries/embeddings), Repo | +| `Trinity.Memory` | Always-on tiers with their budget and consolidator (030), search (031), semantic store and retrieval (032), compaction (023); at 133 the embedding spaces (`Space`, the fourteen-field identity; `Spaces`, the one active space, re-tier, drop), the static floor (`Embedders.Static`, `WordPiece`, `StaticArtifact`), the int8 `Scorer`, the embedder configuration's faults (`EmbedderConfig`) and the operator's `ReEmbedWorker` | LLM (summaries/embeddings), Repo | | `Trinity.Skills` | SKILL.md parsing, registry, loader, manager, scanner (as built at 040: parser, sources, registry, index, the three tools; at 041: staging, promotion, manager, scanner, diff, learn, `skill_manage` and `learn`) | Repo, Permissions, **Effects**, **Receipts**, Sandbox (as built at 041: Tools, Memory, Permissions, Receipts and LLM; Effects is not a dependency: the promotion is not a tool call, it writes its own effect receipt; Tools never depends on Skills) | | `Trinity.Scheduler` | Oban workers for agent tasks, delivery (as built at 050: the `tasks` and `task_runs` rows, `next_run_at` computed from a cron expression or a one-shot datetime, `Workers.Tick` (the one Cron plugin entry, every minute, enqueuing `Workers.RunTask` for what is due), `RunTask` (one turn in a fresh `origin: "cron"` session, the summary, the delivery), `Delivery` with `Delivery.Desktop`, `Parse.human/2` (a phrase to cron through the model, checked by Oban's parser); Oban itself on the app's repo, Lite on SQLite and Basic on Postgres, queues `agent_tasks`, `memory`, `maintenance`; the observer as `Trinity.Memory.ObserverWorker` and the curator as `Trinity.Memory.Curator` live in Memory) | Sessions, LLM, **Repo** (Gateways at 070) | | `Trinity.MCP` | Client manager, tool bridge, server (as built at 059: the boundary alone, holding the core's version, its JSON depth and its telemetry event names; at 060: `Servers` and `ServerConfig` (the rows), `Supervisor` and `Boot` (one `Client` per enabled row), `Client` with its `Wire` (the outbound request, the headers, the core's decoder and validator) and two transports (`Transport.Stdio`, a child on a Port; `Transport.HTTP`, one POST per request), `Bridge` (the one tool module every MCP tool runs through) and `Client.Auth` (062's seam, a static token at 060); at 061: `Server` (the module above the core, `@behaviour BeamMCP.Server`, handed to both transports through `:server`; `tools/call` answered through the membrane, everything else the core's), `Server.Catalog` (the exported tools, sorted), `Server.Exports` (the configured names, `:catalog` refused), `Server.Session` (the `origin: "mcp"` session with its own persona), `Server.Envelope` and `Server.Replay` (the sealed `requestState` and the nonce table), `Server.Auth.Local` (the static bearer, 062's seam on the server side), `Server.Plug` (the `POST /mcp` endpoint, mounted in `TrinityWeb.Endpoint` ahead of the parsers) and `Server.Stdio` with `mix trinity.mcp.stdio`; at 062: `Trinity.MCP.Auth`, a boundary of its own under `Trinity.MCP` with `deps: []` (the behaviour `authorize/2` answering a `Principal`, never the token; `Config`, `Token`, `JWKS`, `Discovery`, `Scopes`; the profiles `Local`, `External` and `Embedded` with `Embedded.Keys`; the client role `Client` with `Client.Store`), and `AuthHost` in `Trinity.MCP` (the configuration from the tree's paths and the authority selection, the personal profile's boot, the refusal receipts, the client role's flow for the page); `Server.Auth.Local` is gone, `Server.Plug` authorizes ahead of the transport and answers `401` with a challenge, `Server` checks scope before the gate) | Tools, **Effects**, **Permissions**, Memory (as built at 059: a top-level boundary, like `Trinity.Smoke`, with `deps: [Trinity, BeamMCP.JSON]`; the boundary compiler checks every call into the `beam_mcp` application and this boundary alone lists its modules; at 060 the deps are `[Trinity, BeamMCP.JSON, BeamMCP.Schema]`, Trinity's tools, permissions and receipts reached through `Trinity`'s exports, and `Trinity.Application` and `TrinityWeb` list `Trinity.MCP`; at 061 the core's `Server`, `Catalog`, `ToolSpec` and both transports are listed too, and `Trinity` exports `Tools.Registry` and `Effects.Runner` to it) | @@ -114,8 +114,8 @@ through those modules only. `boundary` `exports:` lists enforce this. | `Trinity.LLM.Provider` | `stream/3`, `generate/3`, `embed/2`, `models/0`, `capabilities/1` | config `:providers` list | | `Trinity.Tools.Tool` | `name/0`, `description/0`, `schema/0`, `risk/0`, `execute/2` | config `:tools` list + MCP dynamic registration | | `Trinity.Gateways.Adapter` | `child_spec/1`, `deliver/2`, `capabilities/0` | config `:gateways` list | -| `Trinity.Memory.VectorStore` | `upsert/3`, `search/3`, `delete/1`, `count/1` (as built at 032: the scope filter is `search/3`'s required argument) | the database adapter, at compile time (`Brute` on SQLite, `Pgvector` on Postgres; `Hnswlib` when a slice measures past 10^5 rows) | -| `Trinity.Memory.Embedder` | `embed/1`, `dim/0`, `model_id/0`, `availability/0` | `config :trinity, :memory, embedder:` (`:local` default, `:hosted` opt-in only, `:fake` in the suite) | +| `Trinity.Memory.VectorStore` | `upsert/3`, `search/3`, `delete/1`, `count/1` (as built at 032: the scope filter is `search/3`'s required argument; as built at 133: `search/3` and `count/1` over one named space, `{:error, {:mixed_space, n}}` rather than a ranking when a row does not fit it, writes through `Spaces.put_vector/5`) | the database adapter, at compile time (`Brute` on SQLite, `Pgvector` on Postgres; `Hnswlib` when a slice measures past 10^5 rows) | +| `Trinity.Memory.Embedder` | `embed/1`, `dim/0`, `model_id/0`, `availability/0`; at 133 `space/0` and `thresholds/0` | `config :trinity, :memory, embedder:` (`:local` default, `:static` the floor, `:hosted` opt-in only with a declared `locality:`, `:fake` in the suite, or a module; a list names several, and the store is served by the one whose space is pinned) | | `Trinity.Skills.Loader` | `load/1`, `validate/1` | config `:skill_loaders` (not built at 040: the three roots are fixed and `Trinity.Skills.Sources` scans them; a loader behaviour waits for a fourth kind of source) | | `Trinity.Permissions.Policy` | `decide/3` → `:allow | :deny | {:ask, prompt}` | config | @@ -130,6 +130,14 @@ UI/Gateway ──user_message──▶ Session(gen_statem) → LLM.stream/3 ──chunks──▶ PubSub "session:" ──▶ LiveView + gateways → on tool_call: Permissions.decide → (ask → approval_wait) → Tools.execute in Task → result appended → loop until final text → persist → Memory.Observer.observe(turn) (a Task under Trinity.Memory.TaskSupervisor, as built at 032) → idle + +**Embedding spaces (Slice 133, as built):** every vector is a `memory_embeddings` row under the space that produced +it, and the store has one active space, moved only by the operator's re-tier (`Spaces.retier/2`, built beside the +old space, the pointer moved in one transaction when it is complete, the old space kept until `mix +trinity.space.drop --confirm`). `Semantic.status/0` serves the store through the configured embedder whose +space is active and answers OFF with a named reason otherwise: a configuration fault (`EmbedderConfig`; under +`:regulated` the boot is refused instead), a space mismatch, the embedder's availability, or the last runtime +fault (an unreachable endpoint, a store row that does not fit its space). Full-text recall goes on in every case. ``` Every state transition is persisted before it is broadcast. A crash between persist and broadcast is safe diff --git a/docs/02-tech-stack.md b/docs/02-tech-stack.md index 46c0ab8..9e5e99a 100644 --- a/docs/02-tech-stack.md +++ b/docs/02-tech-stack.md @@ -15,6 +15,7 @@ Versions live in `VERSIONS.md`. This file explains *why* each choice was made an | Embeddings | Bumblebee + EXLA (all-MiniLM-L6-v2) | Local, private, 384-dim | If EXLA binary size is unacceptable on desktop → hosted embeddings via req_llm, or Ortex (risk: stalled) | | Vector search | sqlite_vec (brute force) behind `VectorStore` behaviour | Fine to ~10^5 vectors; no extra process | Scale → hnswlib (pre-1.0) or pgvector HNSW | | Vector search, as built at 032 | brute force in Elixir over the row's `embedding` bytes (SQLite); pgvector HNSW (Postgres) | sqlite_vec's one release needs `nx ~> 0.9` and cannot share a tree with nx 0.13 (VERSIONS.md); measured in docs/perf.md | Scale → hnswlib 0.1.10 | +| Embeddings and vector search, as built at 133 | the static floor: `static-retrieval-mrl-en-v1` at 256 dimensions int8 in pure Elixir (`Embedders.Static`, its own WordPiece), scored by the int8 scorer in Elixir; MiniLM through Bumblebee and EXLA stays the default where EXLA loads; every vector under a named, operator-pinned embedding space (`memory_embeddings`) | No NIF on the floor's path, so semantic memory can run on Windows, Burrito's musl and a build without the neural group (`TRINITY_WITHOUT_ML=1`); the eval (slice 133, AC14) decides the default; the weights wait on legal review | Scorer p95 over 50 ms at 10^4 x 256 (measured 10.3 ms with the prefilter, 24.5 ms exact: docs/perf.md) → a scoring-only Rustler crate | | Shell tool | MuonTrap | Guaranteed child kill on process death; cgroups on Linux | none | | Sandbox | Luerl (`sandbox` pkg) | In-VM, reduction-limited, no OS access | Untrusted native code → container/microVM (out of scope) | | Modularity | `boundary` + behaviours + `Registry` | Compile-time enforcement of context deps | none | diff --git a/docs/05-data-model.md b/docs/05-data-model.md index 51676f5..d555ffa 100644 --- a/docs/05-data-model.md +++ b/docs/05-data-model.md @@ -63,7 +63,7 @@ every term for FTS5, so operators are text. | scope | string | "global" \| "persona:" \| "project:" | | key | string, nullable | for always_on/profile: short stable key, unique within (tier, scope) | | body | text | | -| embedding | vector(384), nullable | semantic tier only; sqlite_vec virtual table `memories_vec` on SQLite | +| embedding | vector(384), nullable | semantic tier only; sqlite_vec virtual table `memories_vec` on SQLite (as planned; as built see below: slice 032 put it on the row, slice 133 moved it to `memory_embeddings`) | | source_message_id | fk, nullable | provenance | | confidence | float | agent-assigned | | last_used_at | utc_datetime_usec | for decay/pruning | @@ -91,6 +91,32 @@ The change log's `action` vocabulary grows by `pin` (a semantic memory promoted `AlwaysOn.add/2`, so the budget applies) and `by` by `observer`. Recall marks its memory hits' `last_used_at`, which the retriever's recency decay reads. +As built at slice 133 (embedding spaces): the three `embedding*` columns, the index on them and Postgres's +`embedding_vector vector(384)` are gone from `memories`. Migration `20261008120000_create_memory_embeddings` +moved every row that had a vector (the count before equals the count after, held by +`test/trinity/memory/migration_test.exs`) into two tables: + +- **`embedding_spaces`**: `id` (string, primary key: the space ID, 64 lowercase hex, the SHA-256 of the + manifest's RFC 8785 JSON), `manifest` (map: the fourteen identity fields of `Trinity.Memory.Space`), `dim`, + `quantization` (`f32` or `int8`), `state` (`building` while a re-tier fills it, `complete` after), `active` + (boolean), `completed_at`, timestamps. A unique partial index (`embedding_spaces_one_active`, `WHERE active`) + lets the database hold at most one active space: the store's pinned space. A row written before slice 133 is + under a legacy space whose unrecorded fields read `"unrecorded"` (`Space.legacy/2`); the space of the most + recently embedded row was made active. +- **`memory_embeddings`**: `memory_id` (references `memories`, `ON DELETE CASCADE`) and `space_id` (references + `embedding_spaces`, `ON DELETE RESTRICT`) together the primary key, `dim`, `vector` (binary: float32 + little-endian, or for an `int8` space one signed byte a component, the vector scaled to a largest magnitude + of 127), `inserted_at`, and an index on `space_id`. A memory may hold a vector in several spaces at once, + which is what a re-tier builds. On Postgres there is also `embedding_vector`, an **untyped** pgvector + column, and one partial expression HNSW index per space, + `((embedding_vector::vector()) vector_cosine_ops) WHERE space_id = ''`, named + `memory_embeddings_hnsw_` and built when the space is registered. + +Vectors are written by `Trinity.Memory.Spaces.put_vector/5` alone (bytes and, on Postgres, the pgvector value in +one statement, the width checked against the space). A search names one space; a space with a row whose width +or byte length does not fit it is refused rather than ranked. `mix trinity.space.list`, `mix +trinity.space.retier` and `mix trinity.space.drop --confirm` are the operator's commands. + ### memory_changes (Slice 030) `persona_id`, `action` (add | replace | remove | promote | consolidate), `tier`, `scope`, `key`, `before`, `after`, `by` (tool | ui | consolidator), `session_id`, `proposal_id`, `inserted_at`. Every write to the diff --git a/docs/perf.md b/docs/perf.md index f8e79bc..226534e 100644 --- a/docs/perf.md +++ b/docs/perf.md @@ -96,3 +96,58 @@ index, `<=>`), the same script against a local container on 2026-09-21: |---|---|---|---| | 1,000 | 921 ms | 4.2 ms | 5.5 ms | | 10,000 | 13,402 ms | 1.9 ms | 2.3 ms | + +## The static floor and its scorer (slice 133) + +MEASURED on 2026-10-08 on the owner's machine: AMD RYZEN AI MAX+ 395 w/ Radeon 8060S, 32 logical CPUs, 125 GiB, +Linux 6.14.0-37-generic x86_64; OTP 28 (ERTS 16.4.0.5), Elixir 1.20.4, 32 schedulers; under a 24 GiB cgroup, +with other work on the machine (load average 4.7 to 5.8 on 32 CPUs during the run). + +### Scoring at 10^4 x 256 int8 (AC13, D-scoring) + +`TRINITY_STATIC_MODEL_DIR= MIX_ENV=test mix run --no-start scripts/scorer_bench.exs`: the static model's +own 256-dimension int8 embeddings of 10,000 generated memory-shaped sentences, 200 queries one at a time after +20 discarded, the script's output: + +| scorer | p50 | p95 | +|---|---|---| +| sign bits by Hamming distance, then the exact int8 cosine over the 256 nearest (`Scorer.prefilter/4`) | 8.513 ms | 10.296 ms | +| the exact int8 cosine over every row (`Scorer.exact/3`) | 20.179 ms | 24.5 ms | +| deliberately unoptimised: every row decoded to floats per query (AC13's red) | 248.829 ms | 303.452 ms | + +Preparing the index (norms and sign bits for 10,000 rows) took 43 ms, once per load. The prefilter's recall@10 +against the exact search was 0.9995 over the 200 queries. The pre-set line was 50 ms p95: both real scorers are +under it, so no scoring crate is opened; the unoptimised one is over it by six times, which is what shows the +measurement can tell them apart. Slice 032's figure for the old path (the Elixir cosine over decoded float lists +at 10^4 x 384) was 390 ms. + +### The store, end to end + +`scripts/vector_bench.exs` (`TRINITY_BENCH_QUANT=int8`, fake vectors under an int8 space, one persona, SQLite), +after the store stopped reading every entry and reads only ids and vectors, then the k that rank: + +| rows | int8 search p50 | max | float32 search p50 (EXLA product) | max | +|---|---|---|---|---| +| 1,000 | 4.571 ms | 5.704 ms | 5.678 ms | 7.812 ms | +| 10,000 | 71.959 ms | 87.604 ms | 50.033 ms | 53.817 ms | + +At 10^4 the int8 path's time is mostly the database: the rows' load and the fit check that refuses a mixed +space. A cached, prepared index per space would let the store use the prefilter; it is a follow-up for past 10^4. + +### The weights + +| artifact | bytes | SHA-256 | +|---|---|---| +| upstream `0_StaticEmbedding/model.safetensors` at `f60985c706f192d45d218078e49e5a8b6f15283a` (30,522 x 1024 float32) | 125,018,208 | `164fc63ee9f9267be7378fcbd7df99d09788a2f45244c92aa99ae5a574925716` | +| `static-retrieval-mrl-en-v1-256-int8.tsw` (`mix trinity.static.build`) | 8,167,604 | `5990c1104963d8e2402854c80b9a8f8e3a490085c037f25ced63642f4578c520` | +| `static-retrieval-mrl-en-v1-1024-f32.tsw` | 125,249,996 | `c8e427cc6aa3d55755d9c4085f6ab7572e95a509ff9e665f352f7d231db66299` | + +Sizes from `ls -la`, digests from `sha256sum` and the build task. The 256 int8 file is the matrix (7,813,632 +bytes), a float32 scale per token row (122,088) and the vocabulary. Building it from the snapshot took under two +seconds, and rebuilding gave the same digest. + +### A release without the neural group + +`scripts/static_floor_release.sh` (`TRINITY_WITHOUT_ML=1`): the headless release assembled from a build that never +had nx, exla, xla, axon, bumblebee or tokenizers holds none of them in `lib/`, boots, writes two memories and +recalls one through the static space (slice 133, AC8). Slice 130 measured exla at 443 MB of a 522 MB release. diff --git a/lib/mix/tasks/trinity.eval.corpus.ex b/lib/mix/tasks/trinity.eval.corpus.ex new file mode 100644 index 0000000..60654cc --- /dev/null +++ b/lib/mix/tasks/trinity.eval.corpus.ex @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Mix.Tasks.Trinity.Eval.Corpus do + @shortdoc "Generates the memory eval's corpus, queries, labels and audit sample" + + @moduledoc """ + Slice 133, AC14. Writes the eval directory from a seed: `corpus.jsonl` (10^4 memories, the + 10^2 and 10^3 corpora their prefixes), `queries.jsonl` (300), `labels.jsonl`, `generation.json` + and `audit-100.md`. Offline and deterministic: the same seed writes the same bytes. + + mix trinity.eval.corpus --out [--seed 20261007] + + Next: the owner audits `audit-100.md`, then `mix trinity.eval.freeze --dir `; only then + will `mix trinity.eval.memory` score anything. + """ + use Boundary, classify_to: Trinity.Eval + use Mix.Task + + @impl Mix.Task + def run(argv) do + {opts, _, _} = OptionParser.parse(argv, strict: [out: :string, seed: :integer]) + dir = opts[:out] || Mix.raise("--out is required") + seed = opts[:seed] || 20_261_007 + record = Trinity.Eval.Labels.write!(dir, seed) + audit = Trinity.Eval.Labels.write_audit!(dir, seed) + Mix.shell().info(Jason.encode!(record, pretty: true)) + Mix.shell().info("audit sample: #{audit}") + end +end diff --git a/lib/mix/tasks/trinity.eval.freeze.ex b/lib/mix/tasks/trinity.eval.freeze.ex new file mode 100644 index 0000000..1b40256 --- /dev/null +++ b/lib/mix/tasks/trinity.eval.freeze.ex @@ -0,0 +1,26 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Mix.Tasks.Trinity.Eval.Freeze do + @shortdoc "Freezes the memory eval's labels by SHA-256, after the audit" + + @moduledoc """ + Slice 133, AC14. Refuses unless `audit-100.md` in the directory has all 100 boxes ticked and + names who audited it; then writes `labels.sha256`. A scored run refuses labels that no longer + match it. + + mix trinity.eval.freeze --dir + """ + use Boundary, classify_to: Trinity.Eval + use Mix.Task + + @impl Mix.Task + def run(argv) do + {opts, _, _} = OptionParser.parse(argv, strict: [dir: :string]) + dir = opts[:dir] || Mix.raise("--dir is required") + + case Trinity.Eval.Labels.freeze(dir) do + {:ok, digest} -> Mix.shell().info("labels frozen: #{digest} labels.jsonl") + {:error, reason} -> Mix.raise("refused: #{inspect(reason)}") + end + end +end diff --git a/lib/mix/tasks/trinity.eval.memory.ex b/lib/mix/tasks/trinity.eval.memory.ex new file mode 100644 index 0000000..331fc12 --- /dev/null +++ b/lib/mix/tasks/trinity.eval.memory.ex @@ -0,0 +1,64 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Mix.Tasks.Trinity.Eval.Memory do + @shortdoc "Scores the memory eval (refuses until its labels are frozen)" + + @moduledoc """ + Slice 133, AC14: the scored run. Refuses a directory whose labels are not frozen or have + changed since. Writes `results.json` beside them and prints the verdict against the threshold + slice 133 set in advance. + + TRINITY_STATIC_MODEL_DIR= mix trinity.eval.memory --dir \\ + [--vectors ] [--sizes 100,1000,10000] [--resamples 10000] [--only fts5,static-256-int8] + + The default candidates are `Trinity.Eval.MemoryRun.default_candidates/0`: MiniLM and + potion-retrieval-32M from vectors `scripts/eval/reference_vectors.py` wrote into `/vectors`. + """ + use Boundary, classify_to: Trinity.Eval + use Mix.Task + + alias Trinity.Eval.MemoryRun + + @impl Mix.Task + def run(argv) do + {opts, _, _} = + OptionParser.parse(argv, + strict: [ + dir: :string, + sizes: :string, + resamples: :integer, + only: :string, + vectors: :string + ] + ) + + dir = opts[:dir] || Mix.raise("--dir is required") + Mix.Task.run("app.config") + + candidates = + case opts[:only] do + nil -> + MemoryRun.default_candidates() + + names -> + Enum.filter(MemoryRun.default_candidates(), &(elem(&1, 0) in String.split(names, ","))) + end + + run_opts = + [candidates: candidates] + |> put_opt( + :sizes, + opts[:sizes] && Enum.map(String.split(opts[:sizes], ","), &String.to_integer/1) + ) + |> put_opt(:resamples, opts[:resamples]) + |> put_opt(:vectors_dir, opts[:vectors]) + + case MemoryRun.run(dir, run_opts) do + {:ok, results} -> Mix.shell().info(Jason.encode!(results["verdict"], pretty: true)) + {:error, reason} -> Mix.raise("refused: #{inspect(reason)}") + end + end + + defp put_opt(opts, _key, nil), do: opts + defp put_opt(opts, key, value), do: Keyword.put(opts, key, value) +end diff --git a/lib/mix/tasks/trinity.space.drop.ex b/lib/mix/tasks/trinity.space.drop.ex new file mode 100644 index 0000000..1e80b55 --- /dev/null +++ b/lib/mix/tasks/trinity.space.drop.ex @@ -0,0 +1,44 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Mix.Tasks.Trinity.Space.Drop do + @shortdoc "Deletes an inactive embedding space's vectors (requires --confirm)" + + @moduledoc """ + Slice 133. Deletes a space's vectors and its record. Refuses without `--confirm`, and refuses + the active space whatever the flags. The id may be the first twelve characters `mix + trinity.space` prints. + + mix trinity.space.drop --confirm + """ + use Boundary, classify_to: Trinity + use Mix.Task + + @impl Mix.Task + def run(argv) do + {opts, args, _} = OptionParser.parse(argv, strict: [confirm: :boolean]) + + id = + case args do + [id] -> id + _ -> Mix.raise("usage: mix trinity.space.drop --confirm") + end + + Mix.Task.run("app.start") + + case Trinity.Memory.Spaces.drop(id, confirm: Keyword.get(opts, :confirm, false)) do + {:ok, %{space: space, vectors: n}} -> + Mix.shell().info( + "dropped space #{Trinity.Memory.Space.short(space)}: #{n} vectors deleted" + ) + + {:error, :confirm_required} -> + Mix.raise("refused: dropping a space deletes its vectors; pass --confirm to do it") + + {:error, :active} -> + Mix.raise("refused: #{id} is the active space; re-tier to another space first") + + {:error, reason} -> + Mix.raise("refused: #{inspect(reason)}") + end + end +end diff --git a/lib/mix/tasks/trinity.space.list.ex b/lib/mix/tasks/trinity.space.list.ex new file mode 100644 index 0000000..f0cd81a --- /dev/null +++ b/lib/mix/tasks/trinity.space.list.ex @@ -0,0 +1,34 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Mix.Tasks.Trinity.Space.List do + @shortdoc "Lists the store's embedding spaces" + + @moduledoc """ + Slice 133. The store's embedding spaces, the active one first: id (the first twelve + characters), state, width, quantization, vector count and model. + + mix trinity.space.list + + The other commands: `mix trinity.space.retier` and `mix trinity.space.drop`. + """ + use Boundary, classify_to: Trinity + use Mix.Task + + @impl Mix.Task + def run(_argv) do + Mix.Task.run("app.start") + + case Trinity.Memory.Spaces.list() do + [] -> + Mix.shell().info("no embedding spaces: the store holds no vectors") + + rows -> + Enum.each(rows, &Mix.shell().info(line(&1))) + end + end + + defp line(%{row: r, vectors: n}) do + "#{if r.active, do: "*", else: " "} #{Trinity.Memory.Space.short(r.id)} " <> + "#{r.state} #{r.dim}d #{r.quantization} #{n} vectors #{r.manifest["model_id"]}" + end +end diff --git a/lib/mix/tasks/trinity.space.retier.ex b/lib/mix/tasks/trinity.space.retier.ex new file mode 100644 index 0000000..6db70c7 --- /dev/null +++ b/lib/mix/tasks/trinity.space.retier.ex @@ -0,0 +1,60 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Mix.Tasks.Trinity.Space.Retier do + @shortdoc "Re-embeds the store into an embedder's space and moves the active pointer" + + @moduledoc """ + Slice 133, the operator's re-tier. Builds the target embedder's space beside the active one + (every semantic memory embedded into it, in batches), then moves the active pointer to it in + one transaction. The old space's vectors stay until `mix trinity.space.drop --confirm`. + + mix trinity.space.retier [static|local|hosted|fake] + + Without a name, the first embedder `config :trinity, :memory, embedder:` names. For a store + to keep answering while it builds, configure the target ahead of the current embedder + (`embedder: [:static, :local]`): the store is served by whichever configured embedder writes + the active space, so it answers from the old space until the pointer moves and from the new + one after. + """ + use Boundary, classify_to: Trinity + use Mix.Task + + alias Trinity.Memory.{Embedder, Space, Spaces} + + @impl Mix.Task + def run(argv) do + Mix.Task.run("app.start") + + module = + case argv do + [] -> + Embedder.impl() + + [name] -> + case Embedder.from_name(name) do + {:ok, m} -> + m + + {:error, reason} -> + Mix.raise("#{inspect(reason)}; one of #{Enum.join(Embedder.names(), ", ")}") + end + end + + progress = fn n -> if rem(n, 1000) < 64, do: Mix.shell().info(" #{n} embedded") end + + case Spaces.retier(module, on_batch: progress) do + {:ok, %{space: id, previous: prev, embedded: n}} -> + Mix.shell().info( + "space #{Space.short(id)} is active (#{n} embedded); " <> + if(prev, + do: + "the previous space #{Space.short(prev)} is kept until `mix trinity.space.drop #{Space.short(prev)} --confirm`", + else: "there was no previous space" + ) + ) + + {:error, reason} -> + Mix.raise("re-tier refused: #{inspect(reason)}") + end + end +end diff --git a/lib/mix/tasks/trinity.static.build.ex b/lib/mix/tasks/trinity.static.build.ex new file mode 100644 index 0000000..ee4828a --- /dev/null +++ b/lib/mix/tasks/trinity.static.build.ex @@ -0,0 +1,137 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Mix.Tasks.Trinity.Static.Build do + @shortdoc "Builds the static embedder's weights artifact from a local model snapshot" + + @moduledoc """ + Slice 133. Reads a local snapshot of `sentence-transformers/static-retrieval-mrl-en-v1` (the + directory holding `0_StaticEmbedding/model.safetensors` and `tokenizer.json`), checks both + files against the SHA-256 pinned for the revision in `Trinity.Memory.Embedders.Static`, checks + the tokenizer is the BERT uncased pipeline `Trinity.Memory.WordPiece` implements, and writes + the artifact for a variant (`256-int8` by default, `1024-f32` for the eval's ceiling). Offline: + it reads files and makes no request. Prints the artifact's SHA-256, which must equal the digest + pinned for the variant. + + mix trinity.static.build --snapshot --out [--variant 256-int8] + + The weights it reads and writes are not part of this tree (slice 133 NOTES, D7). + """ + use Boundary, classify_to: Trinity + use Mix.Task + + alias Trinity.Memory.Embedders.Static + alias Trinity.Memory.StaticArtifact + + @expected_normalizer %{ + "type" => "BertNormalizer", + "clean_text" => true, + "handle_chinese_chars" => true, + "strip_accents" => nil, + "lowercase" => true + } + + @impl Mix.Task + def run(argv) do + {opts, _, _} = + OptionParser.parse(argv, strict: [snapshot: :string, out: :string, variant: :string]) + + snapshot = opts[:snapshot] || Mix.raise("--snapshot is required") + out = opts[:out] || Mix.raise("--out is required") + variant = opts[:variant] || "256-int8" + + case build(snapshot, out, variant) do + {:ok, path, digest} -> + Mix.shell().info("wrote #{path}\nsha256 #{digest}") + + {:error, reason} -> + Mix.raise("trinity.static.build: #{inspect(reason)}") + end + end + + @doc """ + The build, callable from a test: `{:ok, path, sha256}` or `{:error, reason}`. Refuses + upstream files whose digest is not the pinned one, and a tokenizer that is not the pipeline + the Elixir tokenizer implements. + """ + @spec build(Path.t(), Path.t(), String.t()) :: {:ok, Path.t(), String.t()} | {:error, term()} + def build(snapshot, out, variant) do + weights = Path.join([snapshot, "0_StaticEmbedding", "model.safetensors"]) + tokenizer = Path.join([snapshot, "0_StaticEmbedding", "tokenizer.json"]) + pinned = Static.upstream_digests() + + with {:ok, spec} <- fetch_variant(variant), + {:ok, wbin} <- File.read(weights), + :ok <- digest(wbin, pinned.weights, :weights), + {:ok, tbin} <- File.read(tokenizer), + :ok <- digest(tbin, pinned.tokenizer, :tokenizer), + {:ok, tjson} <- Jason.decode(tbin), + {:ok, vocab} <- vocab(tjson), + {:ok, matrix, rows, cols} <- StaticArtifact.read_safetensors(wbin), + true <- rows == length(vocab) || {:error, {:rows, rows, length(vocab)}} do + bytes = + StaticArtifact.build(matrix, rows, cols, vocab, + dim: spec.dim, + quantization: spec.quantization, + provenance: %{ + "model" => Static.model(), + "revision" => Static.revision(), + "upstream_weights_sha256" => pinned.weights, + "upstream_tokenizer_sha256" => pinned.tokenizer + } + ) + + File.mkdir_p!(out) + path = Path.join(out, Static.file_name(variant)) + File.write!(path, bytes) + {:ok, path, :crypto.hash(:sha256, bytes) |> Base.encode16(case: :lower)} + end + end + + defp fetch_variant(variant) do + case Map.fetch(Static.variants(), variant) do + {:ok, spec} -> {:ok, spec} + :error -> {:error, {:unknown_variant, variant, Map.keys(Static.variants())}} + end + end + + defp digest(bin, expected, what) do + actual = :crypto.hash(:sha256, bin) |> Base.encode16(case: :lower) + if actual == expected, do: :ok, else: {:error, {:upstream_digest, what, actual}} + end + + # The vocabulary in id order, after checking the pipeline is the one WordPiece implements. + defp vocab(%{"normalizer" => n, "pre_tokenizer" => p, "model" => m}) do + cond do + n != @expected_normalizer -> + {:error, {:normalizer, n}} + + p != %{"type" => "BertPreTokenizer"} -> + {:error, {:pre_tokenizer, p}} + + Map.take(m, ~w(type unk_token continuing_subword_prefix max_input_chars_per_word)) != + %{ + "type" => "WordPiece", + "unk_token" => "[UNK]", + "continuing_subword_prefix" => "##", + "max_input_chars_per_word" => 100 + } -> + {:error, {:model, Map.delete(m, "vocab")}} + + true -> + dense_vocab(m["vocab"]) + end + end + + defp vocab(_), do: {:error, :not_a_tokenizer_json} + + # The tokens in id order, when the ids are exactly 0 to n - 1 (a line of the artifact's + # vocabulary is its id, so a gap or a repeat would shift every token after it). + defp dense_vocab(vocab) do + sorted = Enum.sort_by(vocab, &elem(&1, 1)) + ids = Enum.map(sorted, &elem(&1, 1)) + + if ids == Enum.to_list(0..(map_size(vocab) - 1)), + do: {:ok, Enum.map(sorted, &elem(&1, 0))}, + else: {:error, :vocab_ids_not_dense} + end +end diff --git a/lib/trinity.ex b/lib/trinity.ex index ff1fa58..36b53e6 100644 --- a/lib/trinity.ex +++ b/lib/trinity.ex @@ -65,6 +65,14 @@ defmodule Trinity do Memory.Semantic, Memory.Retriever, Memory.Embedders.Bumblebee, + # Slice 133: the application asks it what the regulated profile refuses; the memory page + # and the space commands read the spaces. + Memory.EmbedderConfig, + Memory.Spaces, + Memory.Space, + # Slice 133: the eval (its own top-level boundary) reads these. + Memory.Scorer, + Memory.Embedders.Static, Tools, # Slice 060: what the MCP bridge (a top-level boundary) implements and returns. Tools.Tool, diff --git a/lib/trinity/application.ex b/lib/trinity/application.ex index 59e47f3..274d0d7 100644 --- a/lib/trinity/application.ex +++ b/lib/trinity/application.ex @@ -125,6 +125,18 @@ defmodule Trinity.Application do Trinity.Profile.check_mcp_auth(profile, mcp_auth), Trinity.Profile.check_embedded_as(profile, mcp_auth), check_authority(profile), + # Slice 133: the embedder's configuration faults (a missing or wrong locality, an + # external embedder without the operator's opt-in, an endpoint off the allow-list) refuse + # the boot under `:regulated`; under `:default` they turn semantic memory off with the + # same reason (`Trinity.Memory.Semantic.status/0`) and the node starts. Asked before the + # models' own endpoint check, so an embedding model off the allow-list is refused with + # the embedder's reason (`:not_allow_listed`), which is the one that names its locality. + Trinity.Memory.EmbedderConfig.check( + profile, + Application.get_env(:trinity, :memory, []), + Keyword.get(llm, :models, []), + Trinity.Profile.raw_endpoints() + ), Trinity.Profile.check_llm_endpoints( profile, Trinity.Profile.raw_endpoints(), diff --git a/lib/trinity/eval.ex b/lib/trinity/eval.ex new file mode 100644 index 0000000..315eabe --- /dev/null +++ b/lib/trinity/eval.ex @@ -0,0 +1,12 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.Eval do + @moduledoc """ + The memory-recall eval (slice 133, AC14): its corpus generator (`Trinity.Eval.Corpus`), its + files, audit and label freeze (`Trinity.Eval.Labels`), and its scored run + (`Trinity.Eval.MemoryRun`), driven by `mix trinity.eval.corpus`, `mix trinity.eval.freeze` and + `mix trinity.eval.memory`. A boundary of its own: it reads the memory context's embedders, + scorer and search, and nothing in the product calls it. + """ + use Boundary, top_level?: true, deps: [Trinity], exports: [Corpus, Labels, MemoryRun] +end diff --git a/lib/trinity/eval/corpus.ex b/lib/trinity/eval/corpus.ex new file mode 100644 index 0000000..689b00f --- /dev/null +++ b/lib/trinity/eval/corpus.ex @@ -0,0 +1,503 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.Eval.Corpus do + @moduledoc """ + The memory-recall eval's corpus and queries (slice 133, AC14), generated from a seed. + + The answer sheet's design: memories in fixed proportions (persona facts 25 %, preferences 25 %, + decisions with their rationale 20 %, facts an observer took from a session 30 %), one to three + sentences each; corpora of 10^2, 10^3 and 10^4 memories, each a superset of the smaller, the + larger ones padded with realistic distractors (near-duplicates, the same entity with another + attribute, older decisions a newer one contradicts); 300 queries, 40 % paraphrases with little + lexical overlap, 30 % keyword-like, 20 % multi-hop, 10 % with no answer; each query generated + from the one to three memories that answer it. + + **Generated by templates, not by a model.** The answer sheet asks for the owner's hosted + frontier-class model; slice 133's NOTES (decision 13) say why it was not loaded here. Every + query's targets are in the 10^2 core, so each query is answerable at every size and only the + distractors grow. Everything is a function of the seed: the same seed gives the same bytes, + which `generation.json` records with each file's SHA-256. + + The memory ids are `m00001` to `m10000`; the core is `m00001` to `m00100`. + """ + + @sizes [100, 1_000, 10_000] + + @typedoc "A memory: id, kind, text, and the smallest corpus size that holds it." + @type memory :: %{id: String.t(), kind: String.t(), text: String.t(), size: pos_integer()} + + @typedoc "A query: id, kind, text, and the memories that answer it (none for no-answer)." + @type query :: %{id: String.t(), kind: String.t(), text: String.t(), targets: [String.t()]} + + # Pools. Small on purpose: the distractors reuse them, which is what makes them distractors. + @women ~w(Anna Dara Elif Greta Ines Keiko Maya Nadia Priya Sana Uma Wen Yara Zoe) + @men ~w(Bruno Chen Farid Hugo Jonas Luca Omar Rafael Tomas Viktor) + @people @women ++ @men + @cities ~w(Lisbon Boston Porto Kyoto Nairobi Oslo Montreal Seville Tbilisi Hanoi Lyon Denver + Valparaiso Krakow Adelaide Ghent) + @pet_names ~w(Rex Miso Pepper Biscuit Olive Juno Nacho Ziggy Mochi Basil) + @tools [ + {"Neovim", "a terminal text editor"}, + {"Postgres", "a relational database"}, + {"SQLite", "an embedded database"}, + {"Kubernetes", "a container orchestrator"}, + {"Obsidian", "a note-taking app"}, + {"a paper notebook", "pen and paper"} + ] + @preferences [ + {"oat milk in coffee", "What goes into my morning cup instead of dairy?"}, + {"green tea in the afternoon", "Which hot drink do I have after lunch?"}, + {"short answers with bullet points", "How long should replies to me be?"}, + {"code examples in Elixir", "Which functional language should samples use?"}, + {"metric units", "Kilos or pounds when you give me measurements?"}, + {"British spelling", "Colour or color in my documents?"}, + {"no emoji in replies", "Should messages to me include little pictures?"}, + {"meetings before noon", "When in the day do I like appointments?"}, + {"Neovim for editing", "Which text editor do I use?"}, + {"a paper notebook for planning", "Where do I sketch out my week?"}, + {"window seats on flights", "Aisle or window when I travel by plane?"}, + {"trains over planes for short trips", "How do I like to travel between nearby cities?"}, + {"jazz while working", "What music helps me concentrate?"}, + {"dark mode in every app", "Light or dark themes for me?"}, + {"email over phone calls", "How do I like to be contacted?"}, + {"spicy food", "Do I enjoy a lot of chilli heat?"}, + {"cold showers in the morning", "How do I wake myself up?"}, + {"aisle seats at the cinema", "Where do I sit for a film?"}, + {"reading on paper rather than screens", "Do I like ebooks?"}, + {"a standing desk", "Do I sit while working?"}, + {"quiet cafes for deep work", "Where do I go to focus away from home?"}, + {"weekly summaries on Fridays", "When do I want a recap of the week?"}, + {"plain language without jargon", "How technical should explanations to me be?"}, + {"cycling to work", "How do I commute?"}, + {"sourdough bread from the corner bakery", "Which loaf do I buy?"} + ] + @stopwords ~w(a an the in on at for with over than rather from to of) + + # Twenty projects, one decision each, so no two core decisions answer the same question. + @projects ~w(analytics billing onboarding garden-planner thesis kitchen-renovation podcast + wedding move-abroad marathon-training newsletter tax-return photo-archive + choir-website bike-repair language-course family-reunion home-network + book-club fundraiser) + @reasons [ + "because it handles concurrent writes", + "because the budget was tight", + "because the team already knows it", + "because the deadline moved up", + "because the first option kept failing", + "because it works offline" + ] + @events [ + "the outage last week", + "the call with the landlord", + "the review meeting", + "the doctor's appointment", + "the trip to the coast", + "the budget discussion" + ] + @absent ~w(horse saxophone yacht Reykjavik submarine glacier harpsichord falconry kayak + observatory) + + @doc "The corpus sizes, smallest first; each is a superset of the one before." + @spec sizes() :: [pos_integer()] + def sizes, do: @sizes + + @doc """ + Generates the corpus and the queries from a seed. Returns `%{memories: [memory], queries: + [query]}`, memories in id order, queries in id order. + """ + @spec generate(integer()) :: %{memories: [memory()], queries: [query()]} + def generate(seed) do + :rand.seed(:exsss, {seed, 133, 14}) + core = core_memories() + queries = queries(core) + distractors = distractors(core, 10_000 - length(core)) + + memories = + (core ++ distractors) + |> Enum.with_index(1) + |> Enum.map(fn {m, i} -> + Map.merge(m, %{id: id(i), size: Enum.find(@sizes, &(i <= &1))}) + end) + + # Queries were built against the core's positions; give them the ids now. + ids = Map.new(Enum.with_index(core, 1), fn {m, i} -> {m.ref, id(i)} end) + + queries = + queries + |> Enum.with_index(1) + |> Enum.map(fn {q, i} -> + %{id: "q" <> pad(i, 3), kind: q.kind, text: q.text, targets: Enum.map(q.refs, &ids[&1])} + end) + + %{memories: Enum.map(memories, &Map.delete(&1, :ref)), queries: queries} + end + + defp id(i), do: "m" <> pad(i, 5) + defp pad(i, n), do: i |> Integer.to_string() |> String.pad_leading(n, "0") + defp pick(list), do: Enum.at(list, :rand.uniform(length(list)) - 1) + + ## The core: 100 memories, 25 / 25 / 20 / 30, each carrying what its queries need + + defp core_memories do + facts = for i <- 1..25, do: persona_fact(i) + prefs = for i <- 1..25, do: preference(i) + {decisions, events} = for(i <- 1..20, do: decision_pair(i)) |> Enum.unzip() + observed = events ++ for(i <- 1..10, do: observed_fact(i)) + facts ++ prefs ++ decisions ++ observed + end + + # Persona facts: 25 distinct slots, so no paraphrase can be answered by two core memories. + # Each slot is {template, paraphrase, second paraphrase, keyword}, the paraphrases sharing as + # few content words with the memory as the slot allows. + defp persona_fact(i) do + city = pick(@cities) + + who = + if i in [2, 4], + do: pick(@men), + else: if(i in [1, 3, 8], do: pick(@women), else: pick(@people)) + + year = 2015 + :rand.uniform(10) + name = pick(@pet_names) + + birthplace = pick(@cities) + + {text, para, para2, keyword} = + Enum.at( + [ + {"The person's sister #{who} lives in #{city}.", + "Which town is my female sibling based in?", "Where is #{who}'s home these days?", + "sister #{city}"}, + {"The person's brother #{who} lives in #{city}.", + "In which place has my male sibling settled?", "Where does #{who} stay?", + "brother #{city}"}, + {"The person's mother #{who} lives in #{city}.", "Where is mum's house?", + "Which town is my mom in?", "mother #{city}"}, + {"The person's father #{who} retired to #{city}.", + "Where did dad go after he stopped working?", "Which place is my old man in now?", + "father retired #{city}"}, + {"The person's partner is called #{who}.", "What is my significant other's name?", + "Who am I in a relationship with?", "partner #{who}"}, + {"The person's best friend #{who} is a pilot.", "What job does my closest pal have?", + "Who flies planes among my friends?", "best friend pilot"}, + {"The person's manager is #{who}.", "Who is my boss?", "Who do I report to at work?", + "manager #{who}"}, + {"The person's daughter #{who} started school in #{year}.", + "When did my child begin her education?", "What year did my kid's classes start?", + "daughter school #{year}"}, + {"The person has a dog called #{name}.", "What did I name my pup?", + "What is my hound called?", "dog #{name}"}, + {"The person has a cat called #{name}.", "What is my kitty's name?", + "What do I call my feline?", "cat #{name}"}, + {"The person works as a backend engineer.", "What do I do for a living?", + "Remind me of my profession.", "backend engineer"}, + {"The person is allergic to peanuts.", "Which nut gives me a reaction?", + "Is there a legume I must avoid?", "allergic peanuts"}, + {"The person is allergic to penicillin.", "Which antibiotic is dangerous for me?", + "Which medicine makes me break out?", "allergic penicillin"}, + {"The person moved to #{city} in #{year}.", "When did I relocate, and to which place?", + "Which year did I change cities?", "moved #{city} #{year}"}, + {"The person was born in #{birthplace}.", "What is my hometown?", + "Where did I grow up as a baby?", "born"}, + {"The person's birthday is on the 14th of March.", + "When should I expect birthday wishes?", "Which day do I turn a year older?", + "birthday March"}, + {"The person speaks Portuguese and Japanese.", "Which languages can I talk in?", + "Besides English, what tongues do I know?", "speaks Portuguese Japanese"}, + {"The person drives a blue electric hatchback.", "What kind of car do I have?", + "Which vehicle is mine?", "electric hatchback"}, + {"The person runs ten kilometres every Sunday.", + "What exercise routine do I keep at the weekend?", "How far do I jog on Sundays?", + "runs kilometres Sunday"}, + {"The person plays the cello in a community orchestra.", + "Which instrument do I perform with?", "What do I play in the amateur ensemble?", + "cello orchestra"}, + {"The person is vegetarian.", "Do I eat meat?", "What is my diet like?", "vegetarian"}, + {"The person's flat is on the fourth floor without a lift.", + "How many stairs to my apartment?", "Is there an elevator where I live?", + "fourth floor lift"}, + {"The person is learning to sail.", "Which water sport am I taking lessons in?", + "What am I training to do on boats?", "learning sail"}, + {"The person volunteers at a food bank on Saturdays.", + "Where do I help out at the weekend?", "What charity work do I do?", + "volunteers food bank"}, + {"The person studied architecture before switching to software.", + "What was my degree before I changed careers?", "What did I train in originally?", + "studied architecture"} + ], + i - 1 + ) + + fact(i, "fact", text, para: para, para2: para2, keyword: keyword) + end + + # Preferences: 25 distinct slots, as the persona facts are. + defp preference(i) do + {thing, para} = Enum.at(@preferences, i - 1) + + fact(100 + i, "preference", "The person prefers #{thing}.", + para: para, + keyword: thing |> String.split() |> Enum.reject(&(&1 in @stopwords)) |> Enum.join(" ") + ) + end + + # A decision and the session event it followed: the pair a multi-hop query needs. + defp decision_pair(i) do + project = Enum.at(@projects, rem(i, length(@projects))) + {tool, _} = pick(@tools) + reason = pick(@reasons) + event = pick(@events) + + decision = + fact( + 200 + i, + "decision", + "After #{event}, the person decided to use #{tool} for the #{project} project, #{reason}.", + para: "Which option did I settle on for the #{project} work, and why?", + keyword: "#{project} #{tool}" + ) + + happened = + fact( + 300 + i, + "observed", + "In a session about the #{project} project, the person described #{event} and its effect on the plan.", + para: "What went wrong around the #{project} work that I told you about?", + keyword: "#{project} #{event}" + ) + + {Map.put(decision, :hop, {project, event, 300 + i}), happened} + end + + defp observed_fact(i) do + who = Enum.at(@people, i - 1) + day = Enum.random(~w(Monday Tuesday Wednesday Thursday Friday)) + project = Enum.at(@projects, i + 9) + + fact( + 400 + i, + "observed", + "During a session, the person mentioned that #{who} reviews the #{project} draft every #{day}.", + para: "Who checks my #{project} writing each week, and on which day?", + keyword: "#{who} #{project} #{day}" + ) + end + + defp fact(ref, kind, text, q), do: %{ref: ref, kind: kind, text: text, q: q} + + ## Queries: 120 paraphrases, 90 keyword-like, 60 multi-hop, 30 with no answer + + defp queries(core) do + # 100 paraphrases, one a core memory, and 20 second phrasings of persona facts. + seconds = core |> Enum.filter(&Keyword.has_key?(&1.q, :para2)) |> Enum.take(20) + para = Enum.map(core, &query(&1, :para)) ++ Enum.map(seconds, &query(&1, :para2)) + + keyword = + core |> Enum.shuffle() |> Stream.cycle() |> Enum.take(90) |> Enum.map(&query(&1, :keyword)) + + hops = core |> Enum.filter(&Map.has_key?(&1, :hop)) + + multi = + hops + |> Stream.cycle() + |> Enum.take(60) + |> Enum.with_index() + |> Enum.map(fn {d, n} -> multi_hop(d, n) end) + + none = for n <- 1..30, do: no_answer(n) + para ++ keyword ++ multi ++ none + end + + defp query(m, :para), do: %{kind: "paraphrase", text: m.q[:para], refs: [m.ref]} + defp query(m, :para2), do: %{kind: "paraphrase", text: m.q[:para2], refs: [m.ref]} + defp query(m, :keyword), do: %{kind: "keyword", text: m.q[:keyword], refs: [m.ref]} + + defp multi_hop(%{ref: ref, hop: {project, event, event_ref}}, n) do + text = + case rem(n, 3) do + 0 -> + "What did I decide about the #{project} project after #{event}?" + + 1 -> + "Once #{event} happened, which way did I go on #{project}?" + + _ -> + "Tell me what changed in my #{project} plans following #{event}, and the choice I made." + end + + %{kind: "multi_hop", text: text, refs: [ref, event_ref]} + end + + # Ten things the corpus never mentions, three phrasings each: thirty distinct questions. + defp no_answer(n) do + thing = Enum.at(@absent, rem(n - 1, 10)) + + text = + case div(n - 1, 10) do + 0 -> "What did I name my #{thing}?" + 1 -> "When am I going to #{thing}?" + _ -> "Remind me what I said about the #{thing}." + end + + %{kind: "no_answer", text: text, refs: []} + end + + ## Distractors: near-duplicates, same entity with another attribute, contradicted decisions, + ## and memories about other people and things + + # A distractor must differ from the memory it imitates in what it says, not only in how: a + # copy, or a copy with a phrase added, is a second right answer, and recall against one id + # would count it as a miss. So a near-duplicate swaps an entity (a person, a place, a pet's + # name, a project, a tool, an event), "same entity, other attribute" changes the relation, and + # a memory that offers neither becomes an unrelated distractor instead. + # No two memories in the corpus share a text either: a store whose observer dedupes does not + # hold the same sentence twice. A distractor that comes out as a text already taken is drawn + # again (up to ten times), and then made unique by the item it is about. + defp distractors(core, n) do + by_kind = Enum.group_by(core, & &1.kind) + + {list, _} = + Enum.map_reduce(1..n, MapSet.new(core, & &1.text), fn i, taken -> + kind = kind_for(i) + + text = + Enum.find_value(1..10, &fresh(by_kind[kind], kind, i, &1, taken)) || + "#{pick(@people)} added item #{i} to the #{pick(@projects)} list." + + {%{kind: kind, text: text, ref: nil}, MapSet.put(taken, text)} + end) + + list + end + + # The fixed proportions, by position: of every 20 memories, 5 facts, 5 preferences, 4 + # decisions and 6 observed facts (25 %, 25 %, 20 %, 30 %). + @kind_cycle List.duplicate("fact", 5) ++ + List.duplicate("preference", 5) ++ + List.duplicate("decision", 4) ++ List.duplicate("observed", 6) + + defp kind_for(i), do: Enum.at(@kind_cycle, rem(i - 1, 20)) + + defp fresh(pool, kind, i, attempt, taken) do + m = pick(pool) + + text = + case rem(i + attempt, 3) do + 0 -> near_duplicate(m.text) + 1 -> other_attribute(m.text) + 2 -> if kind == "decision", do: contradicted(i), else: unrelated(i) + end + + if is_nil(text) or MapSet.member?(taken, text), do: nil, else: text + end + + # The same sentence about someone else: shares most of its words with a memory and does not + # answer that memory's questions. The subject becomes another person ("Omar's sister ...", + # "Omar decided ..."), and any person, place, pet, project, tool or event named is swapped. + # nil when nothing could be changed. + defp near_duplicate(text) do + who = pick(@people) + + text + |> String.replace("The person's ", "#{who}'s ", global: false) + |> String.replace("The person ", "#{who} ", global: false) + |> String.replace("the person ", "#{who} ", global: false) + |> swap_entities(text) + end + + defp swap_entities(text, original) do + tool_names = Enum.map(@tools, &elem(&1, 0)) + + swaps = [ + {@people, pick(@people)}, + {@cities, pick(@cities)}, + {@pet_names, pick(@pet_names)}, + {@projects, pick(@projects)}, + {tool_names, pick(tool_names)}, + {@events, pick(@events)} + ] + + swapped = + Enum.reduce(swaps, text, fn {pool, new}, acc -> + case Enum.find(pool, &String.contains?(acc, &1)) do + nil -> acc + ^new -> acc + old -> String.replace(acc, old, same_kind(old, new)) + end + end) + + if swapped == original, do: nil, else: swapped + end + + # A person is swapped for one of the same gender pool, so "sister" stays with a woman's name. + defp same_kind(old, new) do + cond do + old in @women -> pick(@women -- [old]) + old in @men -> pick(@men -- [old]) + true -> new + end + end + + # The same entity with another attribute; nil when the sentence has no relation to change. + @attribute_swaps [ + {"lives in", "works in"}, + {"retired to", "holidays in"}, + {"prefers", "used to prefer"}, + {"is allergic to", "was once tested for"}, + {"decided to use", "considered using"}, + {"has a dog called", "once looked after a dog called"}, + {"has a cat called", "fed a neighbour's cat called"}, + {"works as a", "once interviewed to be a"}, + {"reviews the", "once proofread the"} + ] + + # Where the sentence also names someone or somewhere, that is swapped too, so the many + # distractors drawn from one memory do not all read the same. + defp other_attribute(text) do + case Enum.find(@attribute_swaps, fn {from, _} -> String.contains?(text, from) end) do + nil -> + nil + + {from, to} -> + changed = String.replace(text, from, to, global: false) + swap_entities(changed, changed) || changed + end + end + + defp contradicted(i) do + project = pick(@projects) + {tool, _} = pick(@tools) + + "An older note says the person decided to use #{tool} for the #{project} project, before changing their mind (#{2010 + rem(i, 12)})." + end + + # Memories about other people and things. Enough combinations that they rarely repeat. + defp unrelated(i) do + who = pick(@people) + other = pick(@people) + city = pick(@cities) + project = pick(@projects) + {tool, _} = pick(@tools) + day = Enum.random(~w(Monday Tuesday Wednesday Thursday Friday Saturday Sunday)) + + case rem(i, 6) do + 0 -> + "#{who} from #{city} recommended a book about #{project} planning." + + 1 -> + "The person read an article about #{city} and its #{Enum.random(~w(trams markets museums parks bridges))}." + + 2 -> + "In passing, the person said #{who} is busy with their own #{project} project." + + 3 -> + "#{who} and #{other} are meeting in #{city} on #{day}." + + 4 -> + "The person saw a talk about #{tool} that #{who} gave in #{city}." + + _ -> + "#{who} asked whether the #{project} notes could be shared by #{day}." + end + end +end diff --git a/lib/trinity/eval/labels.ex b/lib/trinity/eval/labels.ex new file mode 100644 index 0000000..2ca22cb --- /dev/null +++ b/lib/trinity/eval/labels.ex @@ -0,0 +1,216 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.Eval.Labels do + @moduledoc """ + The eval's files, its audit sample, and the freezing of its labels (slice 133, AC14). + + An eval directory holds: + + * `corpus.jsonl`: `{"id", "kind", "size", "text"}` per memory; `size` is the smallest corpus + (10^2, 10^3 or 10^4) that holds it. + * `queries.jsonl`: `{"id", "kind", "text"}` per query, without the answers. + * `labels.jsonl`: `{"query", "targets"}` per query: the gold labels. + * `generation.json`: the seed, the generator's module digest, the counts, and each file's SHA-256. + * `audit-100.md`: 100 queries drawn by kind, each with its target memories' text, for the owner + to check before any scored run. + * `labels.sha256`: written by `freeze/1` and only by it, once the audit file shows all 100 rows + checked and names who audited them. A scored run (`Trinity.Eval.MemoryRun`) refuses a + directory whose `labels.jsonl` does not match it, so labels cannot move after the audit. + """ + + alias Trinity.Eval.Corpus + + Module.register_attribute(__MODULE__, :sobelow_skip, persist: true) + + @audit_quota %{"paraphrase" => 40, "keyword" => 30, "multi_hop" => 20, "no_answer" => 10} + + @doc """ + Generates the corpus from `seed` and writes the files into `dir`. Returns the generation + record, which is also `generation.json`. + """ + # sobelow_skip reason: Traversal.FileModule: the eval directory is the one the operator names on + # `mix trinity.eval.*`, a developer tool run in the project; nothing a request, a model or a + # remote party supplies reaches it, and the file names under it are constants. + @sobelow_skip ["Traversal.FileModule"] + @spec write!(Path.t(), integer()) :: map() + def write!(dir, seed) do + File.mkdir_p!(dir) + %{memories: memories, queries: queries} = Corpus.generate(seed) + + jsonl!(dir, "corpus.jsonl", Enum.map(memories, &Map.take(&1, [:id, :kind, :size, :text]))) + jsonl!(dir, "queries.jsonl", Enum.map(queries, &Map.take(&1, [:id, :kind, :text]))) + jsonl!(dir, "labels.jsonl", Enum.map(queries, &%{query: &1.id, targets: &1.targets})) + + record = %{ + "generator" => "Trinity.Eval.Corpus (templates, no model)", + "generator_source_sha256" => generator_digest(), + "seed" => seed, + "memories" => + Map.new(Corpus.sizes(), &{to_string(&1), Enum.count(memories, fn m -> m.size <= &1 end)}), + "memory_kinds_at_10000" => Enum.frequencies_by(memories, & &1.kind), + "query_kinds" => Enum.frequencies_by(queries, & &1.kind), + "files" => + Map.new( + ~w(corpus.jsonl queries.jsonl labels.jsonl), + &{&1, sha256_file(Path.join(dir, &1))} + ) + } + + File.write!(Path.join(dir, "generation.json"), Jason.encode!(record, pretty: true)) + record + end + + @doc """ + Writes `audit-100.md`: 100 queries drawn by kind (40 paraphrase, 30 keyword, 20 multi-hop, 10 + no-answer), seeded, each with the text of the memories labelled as its answer. + """ + # sobelow_skip reason: Traversal.FileModule: the eval directory is the one the operator names on + # `mix trinity.eval.*`, a developer tool run in the project; nothing a request, a model or a + # remote party supplies reaches it, and the file names under it are constants. + @sobelow_skip ["Traversal.FileModule"] + @spec write_audit!(Path.t(), integer()) :: Path.t() + def write_audit!(dir, seed) do + :rand.seed(:exsss, {seed, 100, 1}) + memories = Map.new(read!(dir, "corpus.jsonl"), &{&1["id"], &1}) + labels = Map.new(read!(dir, "labels.jsonl"), &{&1["query"], &1["targets"]}) + + sample = + dir + |> read!("queries.jsonl") + |> Enum.group_by(& &1["kind"]) + |> Enum.flat_map(fn {kind, qs} -> qs |> Enum.shuffle() |> Enum.take(@audit_quota[kind]) end) + |> Enum.sort_by(& &1["id"]) + + rows = + Enum.map_join(sample, "\n", fn q -> + answers = + case labels[q["id"]] do + [] -> " - (no answer: nothing in the corpus answers this)" + ids -> Enum.map_join(ids, "\n", &" - #{&1}: #{memories[&1]["text"]}") + end + + "- [ ] #{q["id"]} (#{q["kind"]}) #{q["text"]}\n#{answers}" + end) + + path = Path.join(dir, "audit-100.md") + + File.write!(path, """ + # Label audit: 100 queries of #{length(read!(dir, "labels.jsonl"))} + + Labels file: `labels.jsonl`, SHA-256 #{sha256_file(Path.join(dir, "labels.jsonl"))}. + + For each query: does the memory listed under it answer it, and is there no other memory in + the corpus that answers it as well? Tick the box when the label is right. Where it is wrong, + correct `labels.jsonl` and write the correction under "Corrections"; the freeze step reads + the file as it is when every box is ticked. + + Audited by: + Date: + + ## Corrections + + ## Queries + + #{rows} + """) + + path + end + + @doc """ + Freezes the labels: refuses unless `audit-100.md` has all 100 boxes ticked and names who + audited it, then writes `labels.sha256`. `{:ok, digest}` or `{:error, reason}`. + """ + # sobelow_skip reason: Traversal.FileModule: the eval directory is the one the operator names on + # `mix trinity.eval.*`, a developer tool run in the project; nothing a request, a model or a + # remote party supplies reaches it, and the file names under it are constants. + @sobelow_skip ["Traversal.FileModule"] + @spec freeze(Path.t()) :: {:ok, String.t()} | {:error, term()} + def freeze(dir) do + audit = Path.join(dir, "audit-100.md") + + with {:ok, text} <- File.read(audit), + :ok <- audited(text) do + digest = sha256_file(Path.join(dir, "labels.jsonl")) + + File.write!( + Path.join(dir, "labels.sha256"), + "#{digest} labels.jsonl\n#{sha256_file(audit)} audit-100.md\n" + ) + + {:ok, digest} + else + {:error, :enoent} -> {:error, :no_audit_file} + {:error, reason} -> {:error, reason} + end + end + + defp audited(text) do + ticked = length(Regex.scan(~r/^- \[x\] q\d+/mi, text)) + open = length(Regex.scan(~r/^- \[ \] q\d+/m, text)) + + auditor = + case Regex.run(~r/^Audited by:[ \t]*(\S.*)$/m, text) do + [_, who] -> String.trim(who) + _ -> "" + end + + cond do + open > 0 or ticked != 100 -> {:error, {:audit_incomplete, ticked, open}} + auditor == "" -> {:error, :auditor_unnamed} + true -> :ok + end + end + + # The generator's source file, when the build that runs it has it (a Mix project does); its + # digest names the templates that wrote the corpus. A release has no sources and says so. + defp generator_digest do + path = Corpus.module_info(:compile)[:source] |> to_string() + if File.regular?(path), do: sha256_file(path), else: "unavailable (no source in this build)" + end + + @doc "`:ok` when `labels.sha256` exists and `labels.jsonl` still matches it." + # sobelow_skip reason: Traversal.FileModule: the eval directory is the one the operator names on + # `mix trinity.eval.*`, a developer tool run in the project; nothing a request, a model or a + # remote party supplies reaches it, and the file names under it are constants. + @sobelow_skip ["Traversal.FileModule"] + @spec verify(Path.t()) :: :ok | {:error, term()} + def verify(dir) do + case File.read(Path.join(dir, "labels.sha256")) do + {:ok, text} -> + [expected | _] = String.split(text) + actual = sha256_file(Path.join(dir, "labels.jsonl")) + if expected == actual, do: :ok, else: {:error, {:labels_changed, expected, actual}} + + {:error, :enoent} -> + {:error, :labels_not_frozen} + end + end + + @doc "The rows of a JSONL file in the directory." + # sobelow_skip reason: Traversal.FileModule: the eval directory is the one the operator names on + # `mix trinity.eval.*`, a developer tool run in the project; nothing a request, a model or a + # remote party supplies reaches it, and the file names under it are constants. + @sobelow_skip ["Traversal.FileModule"] + @spec read!(Path.t(), String.t()) :: [map()] + def read!(dir, name), + do: dir |> Path.join(name) |> File.stream!() |> Enum.map(&Jason.decode!/1) + + @doc "SHA-256 of a file, lowercase hex." + # sobelow_skip reason: Traversal.FileModule: the eval directory is the one the operator names on + # `mix trinity.eval.*`, a developer tool run in the project; nothing a request, a model or a + # remote party supplies reaches it, and the file names under it are constants. The generator + # digest passes the compiled module's own source path. + @sobelow_skip ["Traversal.FileModule"] + @spec sha256_file(Path.t()) :: String.t() + def sha256_file(path), + do: :crypto.hash(:sha256, File.read!(path)) |> Base.encode16(case: :lower) + + # sobelow_skip reason: Traversal.FileModule: the eval directory is the one the operator names on + # `mix trinity.eval.*`, a developer tool run in the project; nothing a request, a model or a + # remote party supplies reaches it, and the file names under it are constants. + @sobelow_skip ["Traversal.FileModule"] + defp jsonl!(dir, name, rows) do + File.write!(Path.join(dir, name), Enum.map(rows, &[Jason.encode!(&1), "\n"])) + end +end diff --git a/lib/trinity/eval/memory_run.ex b/lib/trinity/eval/memory_run.ex new file mode 100644 index 0000000..70fca49 --- /dev/null +++ b/lib/trinity/eval/memory_run.ex @@ -0,0 +1,400 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.Eval.MemoryRun do + @moduledoc """ + The scored run of the memory-recall eval (slice 133, AC14). **It refuses to run until the + labels are frozen** (`Trinity.Eval.Labels.verify/1`): the owner audits 100 labels first, and a + run against labels that moved afterwards measures nothing anyone agreed to. + + Each candidate is a dense leg fused with the full-text leg by reciprocal rank (k = 60, + `Trinity.Memory.Retriever.rrf/1`), and on its own for diagnosis; `fts5` is the full-text leg + alone. The legs are Trinity's own: + + * **Full text**: an FTS5 table (`porter unicode61`, as `messages_fts`) over the corpus, queried + as `Trinity.Memory.Search` queries: the query's words (`Search.terms/1`), each quoted, all + required, ordered by `bm25()`. The retriever hands its search the person's whole message, and + so does this. + * **Dense**: the query's cosine with every memory of the corpus size, exact; the candidate's own + floor applied as the retriever applies it (`thresholds/0`'s `floor`). An int8 space is scored + by `Trinity.Memory.Scorer`, as the store scores it. + + Recency decay is left out: the synthetic memories carry no dates. Each leg is cut at the + retriever's depth for k = 10 (30). + + Candidates are `{name, source}`: `:fts5`; `{:embedder, module, memory_config}` (the static + floor, at either variant); or `{:vectors, name}`, vectors precomputed outside the tree + (`scripts/eval/reference_vectors.py`: MiniLM, potion-retrieval-32M) as `.json`, + `.corpus.f32` and `.queries.f32` in `vectors_dir:` (the eval directory's `vectors/` + by default; vectors are derived from weights and stay with them, outside the tree, D7). + + Metrics, per corpus size and candidate: recall@5, recall@10 and MRR@10 over the answerable + queries, each with a 95 % bootstrap interval (seeded, 10,000 resamples by default), and the + share of no-answer queries answered with nothing. Pairs (static against MiniLM, potion against + static) get a paired bootstrap interval on the recall@10 difference, and the verdict applies + the threshold slice 133 set in advance. + """ + + alias Trinity.Eval.Labels + alias Trinity.Memory.{Retriever, Scorer, Search} + + Module.register_attribute(__MODULE__, :sobelow_skip, persist: true) + + @depth 30 + @default_resamples 10_000 + + @type candidate :: + {String.t(), :fts5 | {:embedder, module(), keyword()} | {:vectors, String.t()}} + + @doc """ + Runs the eval in `dir`. Options: `candidates:` (required), `sizes:` (the corpus sizes), + `resamples:`, `seed:`. `{:ok, results}` (also written to `results.json` in `dir`), or + `{:error, reason}` before anything is scored. + """ + @spec run(Path.t(), keyword()) :: {:ok, map()} | {:error, term()} + def run(dir, opts) do + with :ok <- Labels.verify(dir), do: {:ok, scored(dir, opts)} + end + + # sobelow_skip reason: Traversal.FileModule: the eval directory is the one the operator names on + # `mix trinity.eval.*`, a developer tool run in the project; nothing a request, a model or a + # remote party supplies reaches it, and the file names under it are constants. + @sobelow_skip ["Traversal.FileModule"] + defp scored(dir, opts) do + memories = Labels.read!(dir, "corpus.jsonl") + queries = Labels.read!(dir, "queries.jsonl") + + ctx = %{ + queries: queries, + labels: Map.new(Labels.read!(dir, "labels.jsonl"), &{&1["query"], &1["targets"]}), + resamples: Keyword.get(opts, :resamples, @default_resamples), + seed: Keyword.get(opts, :seed, 20_261_007) + } + + candidates = Keyword.fetch!(opts, :candidates) + vdir = Keyword.get(opts, :vectors_dir, Path.join(dir, "vectors")) + + dense = + Map.new(candidates, fn {name, src} -> {name, vectors(vdir, src, memories, queries)} end) + + by_size = + opts + |> Keyword.get(:sizes, Trinity.Eval.Corpus.sizes()) + |> Map.new(&{&1, at_size(&1, memories, candidates, dense, ctx)}) + + results = %{ + "labels_sha256" => Labels.sha256_file(Path.join(dir, "labels.jsonl")), + "resamples" => ctx.resamples, + "seed" => ctx.seed, + "sizes" => Map.new(by_size, fn {s, per} -> {to_string(s), per} end), + "verdict" => verdict(by_size, ctx.resamples, ctx.seed) + } + + File.write!(Path.join(dir, "results.json"), Jason.encode!(strip(results), pretty: true)) + results + end + + defp at_size(size, memories, candidates, dense, ctx) do + subset = Enum.filter(memories, &(&1["size"] <= size)) + fts = fts_rankings(subset, ctx.queries) + Map.new(candidates, fn {name, _} -> {name, score(name, dense[name], subset, fts, ctx)} end) + end + + ## Dense vectors per candidate: the corpus's and the queries' + + defp vectors(_dir, :fts5, _memories, _queries), do: nil + + defp vectors(_dir, {:embedder, module, config}, memories, queries) do + previous = Application.get_env(:trinity, :memory, []) + Application.put_env(:trinity, :memory, Keyword.merge(previous, config)) + + try do + {:ok, mv} = module.embed(Enum.map(memories, & &1["text"])) + {:ok, qv} = module.embed(Enum.map(queries, & &1["text"])) + quant = module.space().quantization + + %{ + memories: Map.new(Enum.zip(Enum.map(memories, & &1["id"]), prepare(mv, quant))), + queries: Map.new(Enum.zip(Enum.map(queries, & &1["id"]), prepare(qv, quant))), + floor: module.thresholds().floor, + quantization: quant + } + after + Application.put_env(:trinity, :memory, previous) + end + end + + # sobelow_skip reason: Traversal.FileModule: the eval directory is the one the operator names on + # `mix trinity.eval.*`, a developer tool run in the project; nothing a request, a model or a + # remote party supplies reaches it, and the file names under it are constants. The vectors + # directory is named the same way (`--vectors`). + @sobelow_skip ["Traversal.FileModule"] + defp vectors(vdir, {:vectors, name}, memories, queries) do + manifest = vdir |> Path.join("#{name}.json") |> File.read!() |> Jason.decode!() + dim = manifest["dim"] + read = fn file -> vdir |> Path.join(file) |> File.read!() |> rows(dim) end + + %{ + memories: + Map.new(Enum.zip(manifest["corpus_ids"], prepare(read.(name <> ".corpus.f32"), "f32"))), + queries: + Map.new(Enum.zip(manifest["query_ids"], prepare(read.(name <> ".queries.f32"), "f32"))), + floor: manifest["floor"] || 0.3, + quantization: "f32" + } + |> tap(fn _ -> + unless length(manifest["corpus_ids"]) == length(memories) and + length(manifest["query_ids"]) == length(queries), + do: raise(ArgumentError, "vectors/#{name} does not cover this corpus") + end) + end + + defp rows(bin, dim) do + width = dim * 4 + for <>, do: for(<>, do: f) + end + + # int8 vectors as the store keeps them; float vectors normalised once, so a dot is a cosine. + defp prepare(vs, "int8"), + do: + Enum.map(vs, fn v -> + q = Scorer.quantize(v) + {q, Scorer.norm2(q)} + end) + + defp prepare(vs, _f32) do + Enum.map(vs, fn v -> + n = :math.sqrt(Enum.reduce(v, 0.0, &(&1 * &1 + &2))) + if n == 0.0, do: v, else: Enum.map(v, &(&1 / n)) + end) + end + + ## Full text: Trinity's search semantics over an in-memory FTS5 table + + defp fts_rankings(subset, queries) do + {:ok, db} = Exqlite.Sqlite3.open(":memory:") + + try do + :ok = + Exqlite.Sqlite3.execute( + db, + "CREATE VIRTUAL TABLE c USING fts5(content, id UNINDEXED, tokenize = 'porter unicode61')" + ) + + {:ok, ins} = Exqlite.Sqlite3.prepare(db, "INSERT INTO c (content, id) VALUES (?1, ?2)") + + for m <- subset do + :ok = Exqlite.Sqlite3.bind(ins, [m["text"], m["id"]]) + :done = Exqlite.Sqlite3.step(db, ins) + :ok = Exqlite.Sqlite3.reset(ins) + end + + {:ok, sel} = + Exqlite.Sqlite3.prepare( + db, + "SELECT id FROM c WHERE c MATCH ?1 ORDER BY bm25(c) LIMIT #{@depth}" + ) + + Map.new(queries, fn q -> + case Search.terms(q["text"]) do + [] -> + {q["id"], []} + + terms -> + needle = + Enum.map_join(terms, " ", &("\"" <> String.replace(&1, "\"", "\"\"") <> "\"")) + + :ok = Exqlite.Sqlite3.bind(sel, [needle]) + {:ok, ids} = Exqlite.Sqlite3.fetch_all(db, sel) + :ok = Exqlite.Sqlite3.reset(sel) + {q["id"], Enum.map(ids, &hd/1)} + end + end) + after + Exqlite.Sqlite3.close(db) + end + end + + ## Scoring one candidate at one size + + defp score(name, dense, subset, fts, %{queries: queries, labels: labels} = ctx) do + %{resamples: resamples, seed: seed} = ctx + ids = Enum.map(subset, & &1["id"]) + + rankings = + Map.new(queries, fn q -> + d = dense_ranking(dense, ids, q["id"]) + hybrid = if name == "fts5", do: fts[q["id"]], else: fuse(fts[q["id"]], d) + {q["id"], %{hybrid: hybrid, dense: d}} + end) + + answerable = Enum.filter(queries, &(labels[&1["id"]] != [])) + none = Enum.filter(queries, &(labels[&1["id"]] == [])) + + leg = fn which -> + per_query = + Map.new(answerable, fn q -> + r = rankings[q["id"]][which] + t = labels[q["id"]] + {q["id"], %{r5: recall(r, t, 5), r10: recall(r, t, 10), mrr: mrr(r, t)}} + end) + + %{ + "recall@5" => interval(per_query, :r5, resamples, seed), + "recall@10" => interval(per_query, :r10, resamples, seed), + "mrr@10" => interval(per_query, :mrr, resamples, seed), + "no_answer_empty" => + if(none == [], + do: nil, + else: Enum.count(none, &(rankings[&1["id"]][which] == [])) / length(none) + ), + :per_query => per_query + } + end + + if name == "fts5", + do: %{"hybrid" => leg.(:hybrid)}, + else: %{"hybrid" => leg.(:hybrid), "dense_only" => leg.(:dense)} + end + + defp dense_ranking(nil, _ids, _q), do: [] + + defp dense_ranking(%{quantization: quant, floor: floor} = d, ids, q) do + qv = d.queries[q] + + ids + |> Enum.map(fn id -> {id, cosine(quant, qv, d.memories[id])} end) + |> Enum.filter(fn {_, s} -> s >= floor end) + |> Enum.sort(fn {ia, a}, {ib, b} -> a > b or (a == b and ia <= ib) end) + |> Enum.take(@depth) + |> Enum.map(&elem(&1, 0)) + end + + defp cosine("int8", {q, qn2}, {m, mn2}), do: Scorer.cosine(q, qn2, m, mn2) + defp cosine(_f32, q, m), do: Enum.zip_reduce(q, m, 0.0, fn a, b, acc -> acc + a * b end) + + # Reciprocal rank fusion, as the retriever fuses: a first sighting takes its rank's share, a + # second adds; ties broken by id. + defp fuse(fts, dense) do + [fts, dense] + |> Enum.reduce(%{}, fn list, acc -> + list + |> Enum.with_index(1) + |> Enum.reduce(acc, fn {id, rank}, a -> + Map.update(a, id, Retriever.rrf(rank), &(&1 + Retriever.rrf(rank))) + end) + end) + |> Enum.sort(fn {ia, a}, {ib, b} -> a > b or (a == b and ia <= ib) end) + |> Enum.map(&elem(&1, 0)) + end + + defp recall(ranking, targets, k), + do: Enum.count(targets, &(&1 in Enum.take(ranking, k))) / length(targets) + + defp mrr(ranking, targets) do + case ranking |> Enum.take(10) |> Enum.find_index(&(&1 in targets)) do + nil -> 0.0 + i -> 1 / (i + 1) + end + end + + ## Bootstrap intervals, seeded + + defp interval(per_query, key, resamples, seed) do + values = per_query |> Enum.sort() |> Enum.map(fn {_, m} -> Map.fetch!(m, key) end) + %{"mean" => mean(values), "ci95" => bootstrap(values, resamples, seed)} + end + + @doc false + @spec bootstrap([number()], pos_integer(), integer()) :: [float()] + def bootstrap([], _resamples, _seed), do: [nil, nil] + + def bootstrap(values, resamples, seed) do + :rand.seed(:exsss, {seed, 95, 2}) + t = List.to_tuple(values) + n = tuple_size(t) + + means = + for _ <- 1..resamples do + Enum.reduce(1..n, 0.0, fn _, acc -> acc + elem(t, :rand.uniform(n) - 1) end) / n + end + |> Enum.sort() + + [Enum.at(means, round(resamples * 0.025)), Enum.at(means, round(resamples * 0.975) - 1)] + end + + defp mean([]), do: nil + defp mean(values), do: Enum.sum(values) / length(values) + + ## The verdict, against the threshold set before the run (slice 133 SLICE.md) + + defp verdict(by_size, resamples, seed) do + gap = fn size, a, b -> + with %{} = pa <- get_in(by_size, [size, a, "hybrid"]), + %{} = pb <- get_in(by_size, [size, b, "hybrid"]) do + paired = Enum.map(Map.keys(pa.per_query), &(pa.per_query[&1].r10 - pb.per_query[&1].r10)) + + %{ + "points" => mean(paired) * 100, + "ci95_points" => Enum.map(bootstrap(paired, resamples, seed), &(&1 && &1 * 100)) + } + else + _ -> nil + end + end + + sizes = [1_000, 10_000] + static_vs_minilm = Map.new(sizes, &{to_string(&1), gap.(&1, "static-256-int8", "minilm")}) + + potion_vs_static = + Map.new(sizes, &{to_string(&1), gap.(&1, "potion-retrieval-32M", "static-256-int8")}) + + within? = + Enum.all?(sizes, fn s -> + case static_vs_minilm[to_string(s)] do + %{"points" => p} -> p >= -3.0 + _ -> false + end + end) + + replaces? = + Enum.all?(sizes, fn s -> + case potion_vs_static[to_string(s)] do + %{"points" => p} -> p > 2.0 + _ -> false + end + end) + + %{ + "rule" => + "static-256-int8 hybrid within 3.0 points recall@10 of minilm hybrid at both 10^3 and 10^4; " <> + "potion-retrieval-32M replaces it if more than 2.0 points better at both (subject to tokenizer parity)", + "static_minus_minilm_recall10" => static_vs_minilm, + "potion_minus_static_recall10" => potion_vs_static, + "static_floor_is_regulated_default" => within?, + "potion_replaces_static" => replaces? + } + end + + # The per-query detail stays in memory for the verdict; the file carries the summaries. + defp strip(%{} = map) do + map + |> Enum.reject(fn {k, _} -> k == :per_query end) + |> Map.new(fn {k, v} -> {k, strip(v)} end) + end + + defp strip(other), do: other + + @doc "The candidates the owner's run uses, by name; vectors files where no embedder is in process." + @spec default_candidates() :: [candidate()] + def default_candidates do + static = Trinity.Memory.Embedders.Static + + [ + {"fts5", :fts5}, + {"minilm", {:vectors, "minilm"}}, + {"static-256-int8", {:embedder, static, [static_variant: "256-int8"]}}, + {"static-1024-f32", {:embedder, static, [static_variant: "1024-f32"]}}, + {"potion-retrieval-32M", {:vectors, "potion-retrieval-32M"}} + ] + end +end diff --git a/lib/trinity/memory.ex b/lib/trinity/memory.ex index 331fe47..8fc498d 100644 --- a/lib/trinity/memory.ex +++ b/lib/trinity/memory.ex @@ -21,6 +21,17 @@ defmodule Trinity.Memory do Embedder, Embedders.Bumblebee, Embedders.Fake, + # Slice 133: the static floor, its weights file and tokenizer, and the space identity. + Embedders.Static, + StaticArtifact, + WordPiece, + Space, + Spaces, + SpaceRow, + Vector, + Scorer, + EmbedderConfig, + ReEmbedWorker, Observer, Retriever, Semantic, diff --git a/lib/trinity/memory/embedder.ex b/lib/trinity/memory/embedder.ex index 0b77e3f..089dca6 100644 --- a/lib/trinity/memory/embedder.ex +++ b/lib/trinity/memory/embedder.ex @@ -4,10 +4,17 @@ defmodule Trinity.Memory.Embedder do @moduledoc """ Text to vectors (slice 032). One implementation is in force, chosen by `config :trinity, :memory, embedder:`: `:local` (`Embedders.Bumblebee`, the default: - all-MiniLM-L6-v2 through Bumblebee and EXLA, on this machine), `:hosted` - (`Embedders.Hosted`, a configured alternative only: it sends text to a provider and is - never chosen for the operator), `:fake` (the suite). Every vector records the embedder's - `model_id/0` and `dim/0` on its row, and a store never mixes models (NOTES decision 4). + all-MiniLM-L6-v2 through Bumblebee and EXLA, on this machine), `:static` + (`Embedders.Static`, slice 133: the pure-Elixir floor), `:hosted` (`Embedders.Hosted`, a + configured alternative only: it sends text to a provider and is never chosen for the + operator), `:fake` (the suite), or a module implementing this behaviour. + + Slice 133: every embedder names the embedding space it writes (`space/0`, a + `Trinity.Memory.Space`), and every stored vector carries that space's ID. A store answers + from exactly one space, pinned by the operator (`Trinity.Memory.Spaces`); an embedder whose + space is not the pinned one does not serve that store, and nothing switches spaces + implicitly. Thresholds are the space's own (`thresholds/0`): a cosine that means "the same + memory" for one model means something else for another. """ @type vector :: [float()] @@ -24,21 +31,73 @@ defmodule Trinity.Memory.Embedder do @doc "`:ok`, or the reason this embedder cannot serve here (no model, no backend on this OS, not configured)." @callback availability() :: :ok | {:off, term()} - @doc "The configured choice: `:local` unless configuration says otherwise." - @spec configured() :: :local | :hosted | :fake + @doc "The embedding space this embedder writes and reads (slice 133)." + @callback space() :: Trinity.Memory.Space.t() + + @doc """ + The space's cosine thresholds (slice 133, AC11): `floor`, under which a vector hit is not + recalled, and `dedupe`, at or over which a new memory is the same as an existing one. + """ + @callback thresholds() :: %{floor: float(), dedupe: float()} + + @typedoc "An embedder as configuration names it." + @type name :: :local | :static | :hosted | :fake | module() + + @doc """ + The configured choice: `:local` unless configuration says otherwise. Slice 133: + `embedder:` may also be a list, the first being the one a re-tier targets and an empty store + is first pinned to; the store is served by whichever configured embedder writes its active + space, and by none when no configured embedder does (`Trinity.Memory.Semantic.status/0`). + """ + @spec configured() :: name() | [name()] def configured, do: Application.get_env(:trinity, :memory, []) |> Keyword.get(:embedder, :local) - @doc "The implementation the configuration names." + @doc "Every configured embedder's module, in the configured order." + @spec modules() :: [module()] + def modules, do: configured() |> List.wrap() |> Enum.map(&module/1) + + @doc "The first configured embedder's module." @spec impl() :: module() - def impl do - case configured() do - :local -> Trinity.Memory.Embedders.Bumblebee - :hosted -> Trinity.Memory.Embedders.Hosted - :fake -> Trinity.Memory.Embedders.Fake + def impl, do: hd(modules()) + + @doc "The module an embedder name stands for." + @spec module(name()) :: module() + def module(:local), do: Trinity.Memory.Embedders.Bumblebee + def module(:static), do: Trinity.Memory.Embedders.Static + def module(:hosted), do: Trinity.Memory.Embedders.Hosted + def module(:fake), do: Trinity.Memory.Embedders.Fake + def module(module) when is_atom(module), do: module + + @names %{ + "local" => :local, + "static" => :static, + "hosted" => :hosted, + "fake" => :fake + } + + @doc "The embedder names an operator may type (a re-tier's target)." + @spec names() :: [String.t()] + def names, do: Map.keys(@names) |> Enum.sort() + + @doc """ + The module for a typed name: one of `names/0`, or a configured embedder module by its name. + Never mints an atom from the string. + """ + @spec from_name(String.t()) :: {:ok, module()} | {:error, {:unknown_embedder, String.t()}} + def from_name(name) when is_binary(name) do + case Map.fetch(@names, name) do + {:ok, atom} -> + {:ok, module(atom)} + + :error -> + case Enum.find(modules(), &(inspect(&1) == name)) do + nil -> {:error, {:unknown_embedder, name}} + module -> {:ok, module} + end end end - @doc "Embeds through the implementation in force." + @doc "Embeds through the first configured embedder (`Trinity.Memory.Semantic` embeds through the one serving the store)." @spec embed([String.t()]) :: {:ok, [vector()]} | {:error, term()} def embed(texts) when is_list(texts), do: impl().embed(texts) diff --git a/lib/trinity/memory/embedder_config.ex b/lib/trinity/memory/embedder_config.ex new file mode 100644 index 0000000..c438c3f --- /dev/null +++ b/lib/trinity/memory/embedder_config.ex @@ -0,0 +1,109 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.Memory.EmbedderConfig do + @moduledoc """ + The embedder configuration's faults (slice 133, D2-locality and D3's config half). + + Locality is declared by the operator and **never inferred from an address**: `localhost`, a + private address or a cluster name says nothing about whether a component is inside the + system's authorization boundary. So: + + * An in-process embedder (static, Bumblebee, the suite's fakes) runs in this BEAM by + construction and is `:in_process`; a `locality:` declaring anything else is a fault. + * An embedder with an endpoint (today the hosted one) must carry a declared `locality:`, + `:within_boundary` or `:external`. None is a fault under **both** profiles, whatever the URL; + `:in_process` is a fault (an endpoint is not in this process). + * `:external` needs the operator's opt-in, `external_opt_in: true`, under both profiles: no + memory text leaves for an external embedder unless the operator opted in for this deployment. + * Under `:regulated` only, the endpoint must be stated and on `TRINITY_REGULATED_LLM_ENDPOINTS` + (`:not_allow_listed` otherwise), for `:within_boundary` and `:external` alike. + + Pure, like `Trinity.Profile`: the caller reads the world and hands the values in. Under + `:regulated` `Trinity.Application` refuses the boot on a fault; under `:default` the boot goes + on and `Trinity.Memory.Semantic.status/0` is `{:off, {:config, reason}}`. + """ + + alias Trinity.Memory.Embedder + + @localities [:in_process, :within_boundary, :external] + + @doc """ + `:ok`, or the first fault, for the configured embedders. `memory` is `config :trinity, + :memory`, `models` the LLM registry's models, `raw_endpoints` the allow-list variable. + """ + @spec check(Trinity.Profile.t(), keyword(), [map()], String.t() | nil) :: :ok | {:error, term()} + def check(profile, memory, models, raw_endpoints) do + memory + |> Keyword.get(:embedder, :local) + |> List.wrap() + |> Enum.reduce_while(:ok, fn name, :ok -> + case check_one(profile, name, memory, models, raw_endpoints) do + :ok -> {:cont, :ok} + error -> {:halt, error} + end + end) + end + + defp check_one(profile, name, memory, models, raw) do + declared = Keyword.get(memory, :locality) + + case endpoint(Embedder.module(name), memory, models) do + :in_process -> + if declared in [nil, :in_process], + do: :ok, + else: {:error, {:embedder_locality_mismatch, name, declared}} + + {:endpoint, url} -> + check_endpoint(profile, name, url, declared, memory, raw) + end + end + + defp check_endpoint(_profile, name, url, nil, _memory, _raw), + do: {:error, {:embedder_locality_undeclared, name, url}} + + defp check_endpoint(_profile, name, _url, :in_process, _memory, _raw), + do: {:error, {:embedder_locality_in_process_with_endpoint, name}} + + defp check_endpoint(_profile, name, _url, declared, _memory, _raw) + when declared not in @localities, + do: {:error, {:embedder_locality_invalid, name, declared}} + + defp check_endpoint(profile, name, url, :external, memory, raw) do + if Keyword.get(memory, :external_opt_in) == true, + do: allow_listed(profile, name, url, raw), + else: {:error, {:external_not_opted_in, name}} + end + + defp check_endpoint(profile, name, url, :within_boundary, _memory, raw), + do: allow_listed(profile, name, url, raw) + + defp allow_listed(:default, _name, _url, _raw), do: :ok + + defp allow_listed(:regulated, name, url, raw) do + allowed = Trinity.Profile.allowed_endpoints(raw) + + case URI.parse(to_string(url)) do + %URI{scheme: s, host: h} when is_binary(s) and is_binary(h) and h != "" -> + if {s, h} in allowed, + do: :ok, + else: {:error, {:not_allow_listed, name, "#{s}://#{h}"}} + + _ -> + {:error, {:embedder_endpoint_unstated, name}} + end + end + + # Whether an embedder reaches an endpoint, and which. Only the hosted embedder does today; its + # URL is its registry model's `base_url` (nil when the provider's default endpoint is used, + # which `:regulated` refuses as unstated). + defp endpoint(Trinity.Memory.Embedders.Hosted, memory, models) do + model = Keyword.get(memory, :hosted_model, "nvidia:embed") + + url = + Enum.find_value(models, fn m -> if Map.get(m, :id) == model, do: Map.get(m, :base_url) end) + + {:endpoint, url} + end + + defp endpoint(_in_process, _memory, _models), do: :in_process +end diff --git a/lib/trinity/memory/embedders/bumblebee.ex b/lib/trinity/memory/embedders/bumblebee.ex index 509aa7b..49fedcc 100644 --- a/lib/trinity/memory/embedders/bumblebee.ex +++ b/lib/trinity/memory/embedders/bumblebee.ex @@ -17,6 +17,11 @@ defmodule Trinity.Memory.Embedders.Bumblebee do """ @behaviour Trinity.Memory.Embedder + # Slice 133: a build without the neural group (`TRINITY_WITHOUT_ML=1`, mix.exs) compiles this + # module with none of these present; `availability/0` answers `{:off, {:exla, :not_compiled}}` + # there before anything below is called. + @compile {:no_warn_undefined, [Nx, Nx.Serving, Bumblebee, Bumblebee.Text, EXLA]} + @repo "sentence-transformers/all-MiniLM-L6-v2" @dim 384 @serving Trinity.Memory.Embedding @@ -27,6 +32,19 @@ defmodule Trinity.Memory.Embedders.Bumblebee do @impl true def model_id, do: "bumblebee:" <> @repo + @doc """ + Slice 133: the legacy space (`Trinity.Memory.Space.legacy/2`). Bumblebee loads the model from + the hub's moving `main`, so there is no revision and no digest to pin; rather than invent + them, this embedder keeps the identity rows written before slice 133 carry, so an upgraded + store stays pinned to what it holds. Pinning the revision is a follow-up (NOTES). + """ + @impl true + def space, do: Trinity.Memory.Space.legacy(model_id(), @dim) + + @doc "Slice 032's measured values: unrelated pairs at 0.062, related at 0.858 (AC2)." + @impl true + def thresholds, do: %{floor: 0.3, dedupe: 0.92} + @doc "The Hugging Face repository the weights come from." @spec repo() :: String.t() def repo, do: @repo diff --git a/lib/trinity/memory/embedders/fake.ex b/lib/trinity/memory/embedders/fake.ex index 7663352..853ad43 100644 --- a/lib/trinity/memory/embedders/fake.ex +++ b/lib/trinity/memory/embedders/fake.ex @@ -9,34 +9,64 @@ defmodule Trinity.Memory.Embedders.Fake do cosine is a random walk over 384 signed bytes: within a few hundredths of zero). Texts that share a `#near:` prefix map to nearby vectors (the key is what is hashed; the rest of the text only nudges one component), so a dedupe test has something to find. + + Slice 133: a second fake is a configuration away. `config :trinity, :memory, fake_dim:` (a + multiple of 32; 384 by default) and `fake_seed:` (`""`) change the vectors and the space, so + the suite can hold two spaces at once (a 384 and a 256, as AC2 asks on Postgres) and re-tier + from one to the other. """ @behaviour Trinity.Memory.Embedder - @dim 384 + alias Trinity.Memory.Space @impl true - def dim, do: @dim + def dim, do: Keyword.get(config(), :fake_dim, 384) + + defp seed, do: Keyword.get(config(), :fake_seed, "") + + defp config, do: Application.get_env(:trinity, :memory, []) @impl true - def model_id, do: "fake:sha256-#{@dim}" + def model_id do + case seed() do + "" -> "fake:sha256-#{dim()}" + seed -> "fake:sha256-#{dim()}-#{seed}" + end + end @impl true def availability, do: :ok + @impl true + def space do + %{Space.legacy(model_id(), dim()) | locality: "in_process", runtime: "trinity-fake"} + end + + @impl true + def thresholds, do: %{floor: 0.3, dedupe: 0.92} + @impl true def embed(texts), do: {:ok, Enum.map(texts, &vector/1)} - @doc "The vector for one text." + @doc "The vector for one text, at the configured width and seed." @spec vector(String.t()) :: [float()] - def vector(text) do + def vector(text), do: vector(text, dim(), seed()) + + @doc "The vector for one text at a given width (a multiple of 32) and seed." + @spec vector(String.t(), pos_integer(), String.t()) :: [float()] + def vector(text, dim, salt) do {base, jitter} = case Regex.run(~r/^#near:(\S+)\s*(.*)$/s, text) do [_, key, rest] -> {key, byte_size(rest) * 0.01} _ -> {text, 0.0} end + blocks = div(dim, 32) - 1 + floats = - for i <- 0..11, <> <> base)>>, do: b / 255.0 - 0.5 + for i <- 0..blocks, + <> <> salt <> base)>>, + do: b / 255.0 - 0.5 floats = Enum.with_index(floats, fn f, i -> if i == 0, do: f + jitter, else: f end) norm = :math.sqrt(Enum.sum(Enum.map(floats, &(&1 * &1)))) diff --git a/lib/trinity/memory/embedders/hosted.ex b/lib/trinity/memory/embedders/hosted.ex index 6b40520..d5c0b34 100644 --- a/lib/trinity/memory/embedders/hosted.ex +++ b/lib/trinity/memory/embedders/hosted.ex @@ -25,6 +25,42 @@ defmodule Trinity.Memory.Embedders.Hosted do @impl true def model_id, do: "hosted:" <> model() + @doc """ + Slice 133: the space of a provider's model. The provider pools, normalises and versions the + weights out of sight, so those fields are `"provider"` or `"unrecorded"`; the operator's + declared `locality:` is part of it, so declaring another locality is another space. + """ + @impl true + def space do + u = "unrecorded" + + %Trinity.Memory.Space{ + model_id: model_id(), + revision: u, + weights_digest: u, + tokenizer_digest: u, + dim: dim(), + pooling: "provider", + normalisation: u, + quantization: "f32", + query_prompt: "", + document_prompt: "", + max_input_tokens: u, + truncation: u, + runtime: "req_llm", + runtime_version: to_string(Application.spec(:req_llm, :vsn) || u), + locality: (Application.get_env(:trinity, :memory, [])[:locality] || u) |> to_string(), + num_ctx: "none" + } + end + + @doc "Not measured for any hosted model (slice 032 G1: nemotron's raw vectors do not separate)." + @impl true + def thresholds do + Application.get_env(:trinity, :memory, []) + |> Keyword.get(:hosted_thresholds, %{floor: 0.3, dedupe: 0.92}) + end + @impl true def availability do with :hosted <- Trinity.Memory.Embedder.configured(), diff --git a/lib/trinity/memory/embedders/static.ex b/lib/trinity/memory/embedders/static.ex new file mode 100644 index 0000000..f11f7fc --- /dev/null +++ b/lib/trinity/memory/embedders/static.ex @@ -0,0 +1,273 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.Memory.Embedders.Static do + @moduledoc """ + The static floor (slice 133): `sentence-transformers/static-retrieval-mrl-en-v1` in pure + Elixir. A static embedding model is `EmbeddingBag(vocab, dim, mode = mean)`: look each token + up, take the mean, normalise. No tensor runtime and no NIF, so it runs wherever the BEAM does + (Windows, Burrito's musl, macOS, UBI9, an air-gapped host). + + The weights are a derived artifact (`Trinity.Memory.StaticArtifact`, built by + `mix trinity.static.build` from the upstream snapshot at the pinned revision): by default 256 + dimensions, int8 with a float32 scale per token row, 7.8 MB. Its SHA-256 is pinned here and + checked when the file is first loaded in a VM; a mismatch is `{:off, :weights_digest_mismatch}` + and the application still boots (D3: a runtime fault turns semantic memory off with a reason, + full-text search continues). A missing file is `{:off, :weights_missing}`. + + **No weights are in this tree** (slice 133 NOTES, D7: not before legal review of the model's + licence and training-data terms). The file is looked for, in order: `config :trinity, :memory, + static_dir:`, `TRINITY_STATIC_MODEL_DIR`, the application's `priv/models/static` (where a + bundle would carry it once D7 allows), and `/models/static`. + + Tokens come from `Trinity.Memory.WordPiece` over the artifact's vocabulary. A text with no + tokens (only control or zero-width characters) is the zero vector, as an empty bag's mean is. + """ + @behaviour Trinity.Memory.Embedder + + alias Trinity.Memory.{Space, StaticArtifact, WordPiece} + + Module.register_attribute(__MODULE__, :sobelow_skip, persist: true) + + @model "sentence-transformers/static-retrieval-mrl-en-v1" + @revision "f60985c706f192d45d218078e49e5a8b6f15283a" + # SHA-256 of the upstream `0_StaticEmbedding/model.safetensors` and `tokenizer.json` at the + # revision above (`sha256sum`, 2026-10-08); recorded in the artifact's header as provenance. + @upstream_weights "164fc63ee9f9267be7378fcbd7df99d09788a2f45244c92aa99ae5a574925716" + @upstream_tokenizer "d241a60d5e8f04cc1b2b3e9ef7a4921b27bf526d9f6050ab90f9267a1f9e5c66" + + # The derived artifacts `mix trinity.static.build` writes from that snapshot, by variant, and + # their SHA-256 (`sha256` printed by the task, 2026-10-08; built twice, the same bytes). The + # build is deterministic, so anyone holding the snapshot can rebuild and compare. + @variants %{ + "256-int8" => %{ + dim: 256, + quantization: :int8, + sha256: "5990c1104963d8e2402854c80b9a8f8e3a490085c037f25ced63642f4578c520" + }, + "1024-f32" => %{ + dim: 1024, + quantization: :f32, + sha256: "c8e427cc6aa3d55755d9c4085f6ab7572e95a509ff9e665f352f7d231db66299" + } + } + + @runtime_version "1" + + @doc "The upstream model id." + @spec model() :: String.t() + def model, do: @model + + @doc "The pinned upstream revision." + @spec revision() :: String.t() + def revision, do: @revision + + @doc "The upstream files' SHA-256, as `%{weights: _, tokenizer: _}`." + @spec upstream_digests() :: %{weights: String.t(), tokenizer: String.t()} + def upstream_digests, do: %{weights: @upstream_weights, tokenizer: @upstream_tokenizer} + + @doc "The variants and their pinned digests." + @spec variants() :: %{String.t() => map()} + def variants, do: @variants + + @doc "The configured variant: `config :trinity, :memory, static_variant:` (`\"256-int8\"`)." + @spec variant() :: String.t() + def variant, do: Keyword.get(memory_config(), :static_variant, "256-int8") + + @doc "The artifact's file name for a variant." + @spec file_name(String.t()) :: String.t() + def file_name(variant), do: "static-retrieval-mrl-en-v1-#{variant}.tsw" + + @doc "The directories searched for the artifact, in order." + @spec search_dirs() :: [Path.t()] + def search_dirs do + [ + Keyword.get(memory_config(), :static_dir), + System.get_env("TRINITY_STATIC_MODEL_DIR"), + Application.app_dir(:trinity, "priv/models/static"), + Path.join([Trinity.Paths.data_dir(), "models", "static"]) + ] + |> Enum.reject(&(&1 in [nil, ""])) + end + + @doc "The artifact's path, the first directory that holds it, or nil." + @spec path() :: Path.t() | nil + def path do + name = file_name(variant()) + + Enum.find_value(search_dirs(), fn dir -> + (p = Path.join(dir, name)) |> File.regular?() && p + end) + end + + @impl true + def dim, do: Map.fetch!(@variants, variant()).dim + + @impl true + def model_id, do: "static:" <> @model <> "@" <> variant() + + @impl true + def space do + v = Map.fetch!(@variants, variant()) + + %Space{ + model_id: @model, + revision: @revision, + weights_digest: expected_digest(v.sha256), + tokenizer_digest: @upstream_tokenizer, + dim: v.dim, + pooling: "mean", + normalisation: "l2", + quantization: Atom.to_string(v.quantization), + query_prompt: "", + document_prompt: "", + max_input_tokens: "none", + truncation: "none", + runtime: "trinity-static", + runtime_version: @runtime_version, + locality: "in_process", + num_ctx: "none" + } + end + + @doc """ + The static space's own thresholds (AC11), provisional until the eval re-derives them: measured + on 2026-10-08 (slice 133 NOTES, decision 17), a decision and its negation at 0.9703, the + closest restatement at 0.9832, unrelated pairs near 0. A bag of words cannot see order, so + "tea over coffee" and "coffee over tea" are 1.0 and no threshold separates them. + """ + @impl true + def thresholds, do: %{floor: 0.3, dedupe: 0.98} + + @impl true + def availability do + case load() do + {:ok, _} -> :ok + {:error, reason} -> {:off, reason} + end + end + + @impl true + def embed(texts) when is_list(texts) do + with {:ok, loaded} <- load() do + {:ok, Enum.map(texts, &(loaded |> pooled(&1, :mean) |> normalise()))} + end + end + + @doc """ + The pooled vector before normalisation (what the reference's `StaticEmbedding` returns), for + AC10's comparison. `pooling` is `:mean`, the model's; `:sum` exists only so the parity test + can show that a wrong pooling fails it. + """ + @spec pooled_vector(String.t(), :mean | :sum) :: {:ok, [float()]} | {:error, term()} + def pooled_vector(text, pooling \\ :mean) do + with {:ok, loaded} <- load(), do: {:ok, pooled(loaded, text, pooling)} + end + + @doc "The token ids the embedder looks up for a text." + @spec token_ids(String.t()) :: {:ok, [non_neg_integer()]} | {:error, term()} + def token_ids(text) do + with {:ok, %{tokenizer: tok}} <- load(), do: {:ok, WordPiece.encode(tok, text)} + end + + @doc "The tokenizer over the loaded artifact's vocabulary." + @spec tokenizer() :: {:ok, WordPiece.t()} | {:error, term()} + def tokenizer do + with {:ok, %{tokenizer: tok}} <- load(), do: {:ok, tok} + end + + defp pooled(%{artifact: a, tokenizer: tok}, text, pooling) do + ids = WordPiece.encode(tok, text) + zero = List.duplicate(0.0, a.dim) + + sum = + Enum.reduce(ids, zero, fn id, acc -> + Enum.zip_with(acc, StaticArtifact.row(a, id), &(&1 + &2)) + end) + + case {pooling, length(ids)} do + {_, 0} -> zero + {:sum, _} -> sum + {:mean, n} -> Enum.map(sum, &(&1 / n)) + end + end + + defp normalise(v) do + norm = :math.sqrt(Enum.reduce(v, 0.0, &(&1 * &1 + &2))) + if norm == 0.0, do: v, else: Enum.map(v, &(&1 / norm)) + end + + @doc """ + Loads the configured variant once per VM: reads the file, checks its SHA-256 against the + pinned digest, parses it and builds the tokenizer. `{:error, :weights_missing}`, + `{:error, :weights_digest_mismatch}` or `{:error, {:weights_unreadable, reason}}`. The + result, either way, is kept until `reload/0`. + """ + @spec load() :: {:ok, map()} | {:error, term()} + def load do + key = {__MODULE__, variant(), path()} + + case :persistent_term.get(key, nil) do + nil -> + result = do_load(elem(key, 2), Map.fetch!(@variants, variant())) + :persistent_term.put(key, result) + result + + result -> + result + end + end + + @doc "Forgets what `load/0` read, so the next call reads and checks the file again." + @spec reload() :: :ok + def reload do + for {{__MODULE__, _, _} = key, _} <- :persistent_term.get(), do: :persistent_term.erase(key) + :ok + end + + defp do_load(nil, _variant), do: {:error, :weights_missing} + + defp do_load(path, variant) do + with {:ok, bin} <- read(path), + :ok <- check_digest(bin, variant), + {:ok, artifact} <- parse(bin) do + {:ok, %{artifact: artifact, tokenizer: WordPiece.new(artifact.vocab), path: path}} + end + end + + # sobelow_skip reason: Traversal.FileModule: the path is `path/0`'s: a constant file name under + # the configured static directory, `TRINITY_STATIC_MODEL_DIR`, the application's priv or the + # data directory, all set by the operator; and the bytes read are checked against a pinned + # SHA-256 before anything uses them. + @sobelow_skip ["Traversal.FileModule"] + defp read(path) do + case File.read(path) do + {:ok, bin} -> {:ok, bin} + {:error, reason} -> {:error, {:weights_unreadable, reason}} + end + end + + defp check_digest(bin, %{sha256: expected}) do + expected = expected_digest(expected) + actual = :crypto.hash(:sha256, bin) |> Base.encode16(case: :lower) + if actual == expected, do: :ok, else: {:error, :weights_digest_mismatch} + end + + # The pinned digest, unless configuration names another (the suite's synthetic artifacts, and + # an operator who built a variant this tree does not pin). A configured digest is still a + # digest: the file must match it. + defp expected_digest(pinned) do + case Keyword.get(memory_config(), :static_sha256) do + nil -> pinned + configured -> configured + end + end + + defp parse(bin) do + case StaticArtifact.parse(bin) do + {:ok, a} -> {:ok, a} + {:error, reason} -> {:error, {:weights_unreadable, reason}} + end + end + + defp memory_config, do: Application.get_env(:trinity, :memory, []) +end diff --git a/lib/trinity/memory/entry.ex b/lib/trinity/memory/entry.ex index bc2c187..c08e608 100644 --- a/lib/trinity/memory/entry.ex +++ b/lib/trinity/memory/entry.ex @@ -26,11 +26,8 @@ defmodule Trinity.Memory.Entry do field :source_message_id, Trinity.UUID field :confidence, :float field :last_used_at, :utc_datetime_usec - # Slice 032: the vector, the model that produced it (never mixed: docs/05 § semantic) and - # its width. `nil` until the observer embeds the row. - field :embedding, :binary - field :embedding_model, :string - field :embedding_dim, :integer + # Slice 032 kept the vector on this row; slice 133 moved it to `memory_embeddings`, one row + # a space (`Trinity.Memory.Vector`). # Slice 050: the curator's marks. Stale is untouched for a while and still recalled; # archived leaves recall and stays in the row (nothing is deleted by the curator). field :stale_at, :utc_datetime_usec @@ -57,10 +54,7 @@ defmodule Trinity.Memory.Entry do :body, :source_message_id, :confidence, - :last_used_at, - :embedding, - :embedding_model, - :embedding_dim + :last_used_at ]) |> validate_required([:persona_id, :tier, :scope, :key, :body]) |> validate_inclusion(:tier, @always_on_tiers) @@ -88,10 +82,7 @@ defmodule Trinity.Memory.Entry do :body, :source_message_id, :confidence, - :last_used_at, - :embedding, - :embedding_model, - :embedding_dim + :last_used_at ]) |> put_change(:tier, "semantic") |> validate_required([:persona_id, :scope, :key, :body]) diff --git a/lib/trinity/memory/observer.ex b/lib/trinity/memory/observer.ex index c840b65..b2a0dbc 100644 --- a/lib/trinity/memory/observer.ex +++ b/lib/trinity/memory/observer.ex @@ -9,14 +9,19 @@ defmodule Trinity.Memory.Observer do own model (`Trinity.LLM.generate_object/3`, the one the operator chose for the conversation: no text goes anywhere new, NOTES decision 5) for 0 to 3 durable facts, preferences or decisions, embeds them in one batch, drops each whose cosine to a memory already in the - session's scope chain is at or over the dedupe threshold (0.92, `dedupe_cosine:`) or to - one earlier in the same batch, and inserts the rest as `semantic` rows in the persona's + session's scope chain is at or over the dedupe threshold or to one earlier in the same batch, + and inserts the rest as `semantic` rows in the persona's scope with the source message, the model's confidence and a row in the change log (`by: "observer"`). Off, and nothing runs, when the tier is off (`Trinity.Memory.Semantic.status/0`), when the session has no persona, or when `config :trinity, :memory, observer: false` (the test suite's default: the AC3 test calls `run/2` itself). + + Slice 133: the dedupe threshold is the serving space's own (`Trinity.Memory.Semantic.thresholds/0`: + MiniLM's 0.92, the static floor's 0.98, `dedupe_cosine:` overriding), and the batch is + embedded and stored in one space, named with the vectors, so a re-tier that cuts over between + the embed and the write cannot put one space's vectors under another (AC11; slice 133 NOTES). """ alias Trinity.LLM @@ -25,7 +30,6 @@ defmodule Trinity.Memory.Observer do require Logger - @default_dedupe 0.92 @max_memories 3 @schema %{ @@ -90,20 +94,20 @@ defmodule Trinity.Memory.Observer do @doc """ Extracts, dedupes and stores; synchronous. Returns the entries inserted, `:off`, or the - model's error. The dedupe threshold is `dedupe_cosine:` in `config :trinity, :memory`. + model's error. The dedupe threshold is the serving space's (`Semantic.thresholds/0`). """ @spec run(turn(), [message()]) :: {:ok, [Entry.t()]} | :off | {:error, term()} def run(%{persona_id: persona_id, session_id: session_id} = turn, messages) do with true <- on?(turn) || :off, {:ok, proposed} <- extract(turn, messages), - {:ok, vectors} <- Embedder.embed(Enum.map(proposed, & &1.body)) do + {:ok, vectors, space_id} <- Semantic.embed(Enum.map(proposed, & &1.body)) do source = source_id(messages) inserted = proposed |> Enum.zip(vectors) - |> dedupe(persona_id, AlwaysOn.chain(persona_id, session_id)) - |> Enum.flat_map(fn {m, v} -> insert(m, v, persona_id, session_id, source) end) + |> dedupe(persona_id, AlwaysOn.chain(persona_id, session_id), space_id) + |> Enum.flat_map(fn {m, v} -> insert(m, v, space_id, persona_id, session_id, source) end) {:ok, inserted} else @@ -114,13 +118,12 @@ defmodule Trinity.Memory.Observer do # Drops a proposal whose vector is within the threshold of a memory in the chain or of one # kept earlier in this batch; order preserved. - defp dedupe(pairs, persona_id, chain) do - threshold = - Keyword.get(Application.get_env(:trinity, :memory, []), :dedupe_cosine, @default_dedupe) + defp dedupe(pairs, persona_id, chain, space_id) do + threshold = Semantic.thresholds().dedupe {kept, _} = Enum.reduce(pairs, {[], []}, fn {m, v}, {kept, seen} -> - if duplicate?(persona_id, chain, v, seen, threshold), + if duplicate?(persona_id, chain, v, space_id, seen, threshold), do: {kept, seen}, else: {[{m, v} | kept], [v | seen]} end) @@ -128,7 +131,7 @@ defmodule Trinity.Memory.Observer do Enum.reverse(kept) end - defp insert(m, vector, persona_id, session_id, source) do + defp insert(m, vector, space_id, persona_id, session_id, source) do attrs = %{ persona_id: persona_id, scope: AlwaysOn.persona_scope(persona_id), @@ -138,7 +141,12 @@ defmodule Trinity.Memory.Observer do confidence: m.confidence } - case Semantic.add(attrs, by: "observer", session_id: session_id, vector: vector) do + case Semantic.add(attrs, + by: "observer", + session_id: session_id, + vector: vector, + space_id: space_id + ) do {:ok, entry} -> [entry] @@ -211,9 +219,9 @@ defmodule Trinity.Memory.Observer do defp clean(_), do: [] - defp duplicate?(persona_id, chain, vector, seen, threshold) do + defp duplicate?(persona_id, chain, vector, space_id, seen, threshold) do Enum.any?(seen, &(Embedder.cosine(&1, vector) >= threshold)) or - Semantic.near(persona_id, chain, vector, threshold) != nil + Semantic.near(persona_id, chain, vector, space_id, threshold) != nil end # Provenance: the assistant's last message of the turn, else the user's. diff --git a/lib/trinity/memory/re_embed_worker.ex b/lib/trinity/memory/re_embed_worker.ex new file mode 100644 index 0000000..0220d34 --- /dev/null +++ b/lib/trinity/memory/re_embed_worker.ex @@ -0,0 +1,31 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.Memory.ReEmbedWorker do + @moduledoc """ + An operator's re-tier as a job on the `memory` queue (slice 133), for a running node: + `Trinity.Memory.Spaces.retier/2` toward the named embedder. Nothing in the tree enqueues it on + its own: not a boot, not a configuration change, not a failure (D3; AC3 counts these jobs and + expects none). It exists so an operator can re-tier a node that is serving, and + `mix trinity.space.retier` runs the same function directly. + + `args`: `%{"embedder" => name}`, a name from `Trinity.Memory.Embedder.names/0`. A re-run + resumes a space left `building`. `:already_active` is a completed job, not an error. + """ + use Oban.Worker, + queue: :memory, + max_attempts: 3, + unique: [period: :infinity, states: :incomplete] + + alias Trinity.Memory.{Embedder, Spaces} + + @impl Oban.Worker + def perform(%Oban.Job{args: %{"embedder" => name}}) do + with {:ok, module} <- Embedder.from_name(name) do + case Spaces.retier(module) do + {:ok, _} -> :ok + {:error, :already_active} -> :ok + {:error, reason} -> {:error, reason} + end + end + end +end diff --git a/lib/trinity/memory/retriever.ex b/lib/trinity/memory/retriever.ex index f339e64..29814e8 100644 --- a/lib/trinity/memory/retriever.ex +++ b/lib/trinity/memory/retriever.ex @@ -12,20 +12,24 @@ defmodule Trinity.Memory.Retriever do half its score, so it is demoted below a recent one of the same rank, never lost. A vector search always has `k` nearest rows, however far; a memory enters the vector list - only at a cosine of `recall_min_cosine:` (0.3) or more, the slice's own "unrelated" line - (AC2: the unrelated pair measures 0.062 on the local model, the related one 0.858), so a - question about nothing the person ever said recalls nothing. + only at a cosine of the serving space's floor or more (`Semantic.thresholds/0`; 0.3 for MiniLM, + slice 032's "unrelated" line, where the unrelated pair measures 0.062 and the related one + 0.858; `recall_min_cosine:` overrides), so a question about nothing the person ever said + recalls nothing. + + Slice 133: the vector list comes from the store's one active space; when the store refuses + to rank (a row that does not fit the space) the vector list is empty, the refusal is the + tier's recorded fault, and the full-text list still answers. When the semantic tier is off, the vector list is empty and recall is the full-text list alone: 031 keeps working (NOTES decision 3). Memories that come back are marked used. """ - alias Trinity.Memory.{AlwaysOn, Embedder, Search, Semantic, VectorStore} + alias Trinity.Memory.{AlwaysOn, Search, Semantic} @rrf_k 60 @half_life_days 30 @default_k 8 - @default_min_cosine 0.3 @typedoc "A fused hit: a semantic memory or a past message, with the fused score and which lists found it." @type hit :: %{ @@ -105,20 +109,13 @@ defmodule Trinity.Memory.Retriever do defp vector_hits(persona_id, session_id, query, depth) do with true <- Semantic.on?(), - {:ok, [vector]} <- Embedder.embed([query]) do - chain = AlwaysOn.chain(persona_id, session_id) - - floor = - Keyword.get( - Application.get_env(:trinity, :memory, []), - :recall_min_cosine, - @default_min_cosine - ) - - vector - |> VectorStore.search(depth, Semantic.filter(persona_id, chain)) + chain = AlwaysOn.chain(persona_id, session_id), + {:ok, hits} <- Semantic.search(persona_id, chain, query, depth) do + floor = Semantic.thresholds().floor + + hits |> Enum.filter(&(&1.score >= floor)) - |> Enum.map(fn %{entry: e, score: s} -> + |> Enum.map(fn %{entry: e, score: s, space_id: space_id} -> %{ kind: :memory, id: e.id, @@ -126,7 +123,13 @@ defmodule Trinity.Memory.Retriever do at: e.last_used_at || e.inserted_at, score: s, found_by: [], - ref: %{key: e.key, scope: e.scope, source_message_id: e.source_message_id, cosine: s} + ref: %{ + key: e.key, + scope: e.scope, + source_message_id: e.source_message_id, + cosine: s, + space_id: space_id + } } end) else diff --git a/lib/trinity/memory/scorer.ex b/lib/trinity/memory/scorer.ex new file mode 100644 index 0000000..f80c06c --- /dev/null +++ b/lib/trinity/memory/scorer.ex @@ -0,0 +1,136 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.Memory.Scorer do + @moduledoc """ + Scoring int8 vectors in pure Elixir (slice 133, D-scoring): the path that needs no NIF. + + A stored vector of an int8 space is the L2-normalised embedding scaled so its largest + component is 127 (`quantize/1`), one signed byte a dimension. Two ways to find the nearest: + + * **Exact**: the cosine of the int8 query with every row, `dot / (|q| * |d|)`, integers all the + way until the one division. The dot runs over the two binaries eight bytes at a time, with + no list built and no float boxed per element. + * **Prefilter and rescore**: each row's sign bits (one bit a dimension, 32 bytes at 256d) are + compared with the query's by Hamming distance, sixteen bits at a time through a 65,536-entry + popcount table (OTP has no popcount BIF: erlang/otp#11371 is unmerged), and only the + `candidates` nearest by Hamming are scored exactly. Faster on a prepared index, and + approximate: a true neighbour whose signs disagree a lot can fall outside the candidates. + + AC13 measures both at 10^4 x 256 (`scripts/scorer_bench.exs`, docs/perf.md); over 50 ms p95 + opens the scoring-crate slice. Nothing here calls `nx`, `exla` or any NIF, which AC7's xref + check holds for this module and everything it reaches. + """ + import Bitwise + + @typedoc "A prepared row: its id, its int8 bytes, its squared norm and its sign bits." + @type entry :: {term(), binary(), pos_integer() | 0, binary()} + + @pop16 List.to_tuple(for(x <- 0..65_535, do: x |> Integer.digits(2) |> Enum.sum())) + + @doc """ + A float vector as int8 bytes: scaled so its largest magnitude is 127, rounded. The zero + vector stays zero. + """ + @spec quantize([float()]) :: binary() + def quantize(floats) do + max = floats |> Enum.map(&abs/1) |> Enum.max(fn -> 0.0 end) + + if max == 0.0 do + :binary.copy(<<0>>, length(floats)) + else + for x <- floats, into: <<>>, do: <> + end + end + + @doc "int8 bytes back to floats (unscaled: the cosine does not need the scale)." + @spec to_floats(binary()) :: [float()] + def to_floats(bin), do: for(<>, do: x * 1.0) + + @doc "The integer dot product of two int8 vectors of the same length." + @spec dot(binary(), binary()) :: integer() + def dot(a, b) when byte_size(a) == byte_size(b), do: dot(a, b, 0) + + defp dot( + <>, + <>, + acc + ) do + dot( + ra, + rb, + acc + a0 * b0 + a1 * b1 + a2 * b2 + a3 * b3 + a4 * b4 + a5 * b5 + a6 * b6 + a7 * b7 + ) + end + + defp dot(<>, <>, acc), + do: dot(ra, rb, acc + a * b) + + defp dot(<<>>, <<>>, acc), do: acc + + @doc "The squared norm of an int8 vector." + @spec norm2(binary()) :: non_neg_integer() + def norm2(a), do: dot(a, a, 0) + + @doc "The sign bits of an int8 vector: bit i is 1 when component i is not negative." + @spec signs(binary()) :: binary() + def signs(bin), + do: for(<>, into: <<>>, do: <= 0, do: 1, else: 0)::1>>) + + @doc "The Hamming distance between two sign vectors of the same length (a multiple of 16 bits)." + @spec hamming(binary(), binary()) :: non_neg_integer() + def hamming(a, b), do: hamming(a, b, 0) + + defp hamming(<>, <>, acc), + do: hamming(ra, rb, acc + elem(@pop16, bxor(x, y))) + + defp hamming(<<>>, <<>>, acc), do: acc + + @doc "Rows `{id, int8}` prepared for scoring: the squared norm and the sign bits computed once." + @spec prepare([{term(), binary()}]) :: [entry()] + def prepare(rows), do: Enum.map(rows, fn {id, v} -> {id, v, norm2(v), signs(v)} end) + + @doc "The cosine of an int8 query with an int8 row, from their dot and squared norms." + @spec cosine(binary(), non_neg_integer(), binary(), non_neg_integer()) :: float() + def cosine(q, qn2, d, dn2) do + if qn2 == 0 or dn2 == 0, do: 0.0, else: dot(q, d, 0) / :math.sqrt(qn2 * dn2) + end + + @doc """ + The `k` nearest prepared rows to an int8 query by exact int8 cosine, best first, as + `{id, score}`. Ties are broken by id so the order is the same on every run. + """ + @spec exact([entry()], binary(), pos_integer()) :: [{term(), float()}] + def exact(index, q, k) do + qn2 = norm2(q) + + index + |> Enum.map(fn {id, d, dn2, _} -> {id, cosine(q, qn2, d, dn2)} end) + |> top(k) + end + + @doc """ + Prefilter and rescore: the `candidates` rows nearest to the query by Hamming distance over + sign bits, then the exact int8 cosine over those alone; the `k` best, as `exact/3` returns + them. + """ + @spec prefilter([entry()], binary(), pos_integer(), pos_integer()) :: [{term(), float()}] + def prefilter(index, q, k, candidates) do + qs = signs(q) + qn2 = norm2(q) + + index + |> Enum.map(fn {_, _, _, s} = e -> {hamming(qs, s), e} end) + |> Enum.sort_by(&elem(&1, 0)) + |> Enum.take(candidates) + |> Enum.map(fn {_, {id, d, dn2, _}} -> {id, cosine(q, qn2, d, dn2)} end) + |> top(k) + end + + defp top(scored, k) do + scored + |> Enum.sort(fn {ia, a}, {ib, b} -> a > b or (a == b and ia <= ib) end) + |> Enum.take(k) + end +end diff --git a/lib/trinity/memory/semantic.ex b/lib/trinity/memory/semantic.ex index 7602e7c..d28b82b 100644 --- a/lib/trinity/memory/semantic.ex +++ b/lib/trinity/memory/semantic.ex @@ -3,32 +3,82 @@ defmodule Trinity.Memory.Semantic do @moduledoc """ The semantic tier (slice 032): unbounded rows of `memories` with `tier: "semantic"`, each - carrying the vector its embedder produced, written by the observer after a turn and read by - the retriever and the `recall` tool. Every write is a row in the change log, as the - always-on tiers' are (030). - - Whether the tier is on is `status/0`: the embedder in force can serve. When it cannot, the - tier is off (the observer and the retriever stand down, the recall tool says so, the memory - page says "unavailable"), never rerouted to a hosted embedder (NOTES decision 3). The status - is computed from the embedder's `availability/0` and the serving's presence every time it - is asked; nothing is cached, so a download changes the answer. + with a vector, written by the observer after a turn and read by the retriever and the `recall` + tool. Every write is a row in the change log, as the always-on tiers' are (030). + + **Slice 133: one active space, served or OFF.** Vectors live in `memory_embeddings` under the + space that produced them (`Trinity.Memory.Space`), and the store has exactly one active space + (`Trinity.Memory.Spaces`). The store is served by the configured embedder whose space is the + active one, and by nothing else: when no configured embedder writes the active space, semantic + memory is OFF with `{:space_mismatch, ...}`, the pointer stays, and nothing is re-embedded + (D3). `status/0` is the one answer, from four places, in this order: + + 1. **Configuration** (`Trinity.Memory.EmbedderConfig`): a declared locality that is missing or + wrong is `{:off, {:config, reason}}` (under `:regulated` the boot refused it already). + 2. **The pointer**: `{:off, {:space_mismatch, ...}}`, or `{:off, :no_active_space}` for a store + holding vectors and no active space. + 3. **The embedder**: its `availability/0` (`:weights_missing`, `:weights_digest_mismatch`, + `:model_missing`, ...). + 4. **The last runtime fault** (`:endpoint_unreachable`, `{:mixed_space, n}`), recorded when an + embed or a search fails and cleared by the next that succeeds. + + When the tier is off the observer and the retriever stand down and full-text recall goes on + (NOTES decision 3); nothing is rerouted to another embedder or another space. """ import Ecto.Query - alias Trinity.Memory.{AlwaysOn, Embedder, Entry, VectorStore} + alias Trinity.Memory.{ + AlwaysOn, + Embedder, + EmbedderConfig, + Entry, + Space, + SpaceRow, + Spaces, + VectorStore + } + alias Trinity.Repo + @adapter Application.compile_env(:trinity, :db_adapter, Ecto.Adapters.SQLite3) + @fault_key {__MODULE__, :fault} + @type status :: :on | {:off, term()} + @typedoc "What serves the store: the embedder, its space, and the active row (nil before the first pin)." + @type serving :: %{ + module: module(), + space: Space.t(), + space_id: Space.id(), + row: SpaceRow.t() | nil + } + @doc "The tier's status now." @spec status() :: status() def status do - impl = Embedder.impl() - - case impl.availability() do - :ok -> if serving?(impl), do: :on, else: {:off, :serving_not_started} + with {:ok, %{space_id: id}} <- ready(), + nil <- fault(id) do + :on + else {:off, reason} -> {:off, reason} + {:fault, reason} -> {:off, reason} + end + end + + # Everything but the runtime fault: configuration, the pointer, the embedder's availability. + # An embed or a search is attempted on this, so the attempt that succeeds is what clears a + # fault (an endpoint that came back, a store whose odd row was removed). + defp ready do + case serving() do + {:ok, %{module: module} = s} -> + with :ok <- module.availability(), + :ok <- serving_started(module) do + {:ok, s} + end + + {:off, reason} -> + {:off, reason} end end @@ -36,9 +86,65 @@ defmodule Trinity.Memory.Semantic do @spec on?() :: boolean() def on?, do: status() == :on + @doc """ + What serves the store, or why nothing does: the configured embedder whose space is active; + before the first pin of an empty store, the first configured embedder. + """ + @spec serving() :: {:ok, serving()} | {:off, term()} + def serving do + with :ok <- config_check() do + modules = Embedder.modules() + spaces = Enum.map(modules, fn m -> {m, m.space()} end) + + case Spaces.active() do + nil -> unpinned(spaces) + %SpaceRow{} = row -> pinned(spaces, row) + end + end + end + + # No active space: an empty store is served by the first configured embedder (its first + # write pins it); a store holding vectors is not served until an operator pins it. + defp unpinned([{m, space} | _]) do + if Spaces.empty?(), + do: {:ok, %{module: m, space: space, space_id: Space.id(space), row: nil}}, + else: {:off, :no_active_space} + end + + defp pinned(spaces, %SpaceRow{id: active} = row) do + case Enum.find(spaces, fn {_, space} -> Space.id(space) == active end) do + {m, space} -> + {:ok, %{module: m, space: space, space_id: active, row: row}} + + nil -> + configured = Enum.map(spaces, fn {_, sp} -> Space.short(Space.id(sp)) end) + {:off, {:space_mismatch, %{active: Space.short(active), configured: configured}}} + end + end + + defp config_check do + case EmbedderConfig.check( + Trinity.Profile.current(), + Application.get_env(:trinity, :memory, []), + Application.get_env(:trinity, :llm, []) |> Keyword.get(:models, []), + Trinity.Profile.raw_endpoints() + ) do + :ok -> :ok + {:error, reason} -> {:off, {:config, reason}} + end + end + @doc "The status as a sentence for a page or a tool." @spec describe(status()) :: String.t() - def describe(:on), do: "semantic recall is on (#{Embedder.impl().model_id()})" + def describe(:on) do + case serving() do + {:ok, %{module: m, space_id: id}} -> + "semantic recall is on (#{m.model_id()}, space #{Space.short(id)})" + + _ -> + "semantic recall is on" + end + end def describe({:off, :model_missing}), do: "semantic recall is unavailable: the local model is not downloaded" @@ -52,32 +158,143 @@ defmodule Trinity.Memory.Semantic do def describe({:off, :serving_not_started}), do: "semantic recall is unavailable: the embedding serving is not running" + def describe({:off, :weights_missing}), + do: "semantic recall is unavailable: the static embedder's weights are not installed" + + def describe({:off, :weights_digest_mismatch}), + do: "semantic recall is unavailable: the static embedder's weights failed their digest check" + + def describe({:off, :endpoint_unreachable}), + do: "semantic recall is unavailable: the embedder's endpoint is unreachable" + + def describe({:off, {:space_mismatch, %{active: a}}}), + do: + "semantic recall is unavailable: the store is pinned to space #{a} and no configured " <> + "embedder writes it (an operator re-tier changes the space)" + + def describe({:off, {:mixed_space, n}}), + do: "semantic recall is unavailable: #{n} stored vector(s) do not fit the active space" + + def describe({:off, {:config, reason}}), + do: + "semantic recall is unavailable: the embedder configuration is refused (#{inspect(reason)})" + def describe({:off, reason}), do: "semantic recall is unavailable: #{inspect(reason)}" - @doc "The model id rows written now carry; the filter every search runs under." - @spec model() :: String.t() - def model, do: Embedder.impl().model_id() + @doc """ + The serving space's thresholds (`Trinity.Memory.Embedder.thresholds/0`), with + `config :trinity, :memory, dedupe_cosine:` and `recall_min_cosine:` as explicit overrides. + The first configured embedder's when nothing serves. + """ + @spec thresholds() :: %{floor: float(), dedupe: float()} + def thresholds do + module = + case serving() do + {:ok, %{module: m}} -> m + _ -> Embedder.impl() + end + + config = Application.get_env(:trinity, :memory, []) + base = module.thresholds() + + %{ + floor: Keyword.get(config, :recall_min_cosine) || base.floor, + dedupe: Keyword.get(config, :dedupe_cosine) || base.dedupe + } + end + + @doc """ + Embeds texts for the store: through the serving embedder, tagged with the space the vectors + belong to. A failure is recorded as the tier's runtime fault (an endpoint error as + `:endpoint_unreachable`); a success clears it. + """ + @spec embed([String.t()]) :: {:ok, [Embedder.vector()], Space.id()} | {:error, term()} + def embed(texts) do + with {:ok, %{module: m, space_id: id}} <- serving_on() do + case m.embed(texts) do + {:ok, vectors} -> + clear_fault(id, :embed) + {:ok, vectors, id} + + {:error, reason} -> + {:error, put_fault(id, runtime_reason(reason))} + end + end + end + + defp serving_on do + case ready() do + {:ok, s} -> {:ok, s} + {:off, reason} -> {:error, {:embedder_off, reason}} + end + end + + defp runtime_reason(%Trinity.LLM.Error{transient?: true}), do: :endpoint_unreachable + defp runtime_reason({:endpoint_unreachable, _}), do: :endpoint_unreachable + defp runtime_reason(:endpoint_unreachable), do: :endpoint_unreachable + + defp runtime_reason(%Trinity.LLM.Error{status: status}) when is_integer(status), + do: {:embed_failed, status} + + defp runtime_reason(other), do: {:embed_failed, other} - @doc "The search filter for a persona and the scopes it may see (M6: never wider)." - @spec filter(String.t(), [String.t()]) :: VectorStore.filter() - def filter(persona_id, scopes), do: %{persona_id: persona_id, scopes: scopes, model: model()} + @doc """ + The `k` nearest semantic memories to a query text within a persona's scopes, in the active + space: `{:ok, hits}` (each hit names its `space_id`), or `{:error, reason}` when the tier is + off, the embed failed, or the store refused to rank (`{:mixed_space, n}`). + """ + @spec search(String.t(), [String.t()], String.t(), pos_integer()) :: + {:ok, [VectorStore.hit()]} | {:error, term()} + def search(persona_id, scopes, query, k) do + with {:ok, [vector], space_id} <- embed([query]) do + search_vector(persona_id, scopes, vector, space_id, k) + end + end + + @doc "As `search/4`, for a vector already embedded in `space_id`." + @spec search_vector(String.t(), [String.t()], Embedder.vector(), Space.id(), pos_integer()) :: + {:ok, [VectorStore.hit()]} | {:error, term()} + def search_vector(persona_id, scopes, vector, space_id, k) do + case Spaces.get(space_id) do + nil -> + {:ok, []} + + row -> + case VectorStore.search(vector, k, filter(persona_id, scopes, row)) do + {:ok, hits} -> + clear_fault(space_id, :search) + {:ok, hits} + + {:error, {:mixed_space, _} = reason} -> + {:error, put_fault(space_id, reason)} + + {:error, reason} -> + {:error, reason} + end + end + end + + @doc "The search filter for a persona, the scopes it may see (M6: never wider) and a space." + @spec filter(String.t(), [String.t()], SpaceRow.t()) :: VectorStore.filter() + def filter(persona_id, scopes, %SpaceRow{} = row), + do: %{persona_id: persona_id, scopes: scopes, space: row} @doc """ - Adds a semantic memory: embeds the body (or takes `vector:` when the caller embedded it - already, the observer's batch), inserts the row with its vector and the model that - produced it, logs the change (`by:`, `session_id:`). `{:error, {:embedder_off, _}}` when the - tier is off; `{:error, :exists}` when the key is taken in that scope. + Adds a semantic memory: embeds the body through the serving embedder (or takes `vector:` with + the `space_id:` it was embedded in, the observer's batch), inserts the row, stores the vector + under the active space and logs the change (`by:`, `session_id:`). An empty store is pinned + to the serving space on this first write (`Spaces.pin_if_empty/1`). A vector embedded in a + space that is no longer the active one is embedded again rather than stored under the wrong + space. `{:error, {:embedder_off, reason}}` when the tier is off; `{:error, :exists}` when the + key is taken in that scope. """ @spec add(map(), keyword()) :: {:ok, Entry.t()} | {:error, term()} def add(attrs, opts) do attrs = Map.new(attrs, fn {k, v} -> {to_atom(k), v} end) with nil <- AlwaysOn.get("semantic", attrs[:scope], attrs[:key]), - {:ok, [vector]} <- vector_for(attrs[:body], Keyword.get(opts, :vector)), - {:ok, entry} <- insert(attrs, vector) do - # The bytes column is on the row already (the changeset); the Postgres store also fills - # its vector column. `upsert/3` is the one place that knows about both. - :ok = VectorStore.upsert(entry.id, vector, model()) + {:ok, vector, space_id} <- vector_for(attrs[:body], opts), + {:ok, entry} <- insert(attrs, vector, space_id) do AlwaysOn.log(entry, "add", nil, entry.body, Keyword.put_new(opts, :by, "observer")) {:ok, Repo.reload!(entry)} else @@ -86,15 +303,115 @@ defmodule Trinity.Memory.Semantic do end end - @doc "The nearest existing memory to a vector within the scopes, when its cosine reaches `threshold`." - @spec near(String.t(), [String.t()], Embedder.vector(), float()) :: VectorStore.hit() | nil - def near(persona_id, scopes, vector, threshold) do - case VectorStore.search(vector, 1, filter(persona_id, scopes)) do - [%{score: score} = hit] when score >= threshold -> hit + defp vector_for(body, opts) do + case {Keyword.get(opts, :vector), Keyword.get(opts, :space_id)} do + {v, id} when is_list(v) and is_binary(id) -> + {:ok, v, id} + + _ -> + with {:ok, [v], id} <- embed([body]), do: {:ok, v, id} + end + end + + # One transaction: the row, then the vector under the space that is active now. If the + # vector was embedded in another space (a re-tier cut over in between), embed again with the + # serving embedder; on Postgres the active row is read FOR SHARE, so a cutover waits for this. + defp insert(attrs, vector, space_id) do + Repo.transaction(fn -> + {target, space, vector} = target_space(space_id, vector, attrs[:body]) + + with {:ok, entry} <- %Entry{} |> Entry.semantic_changeset(attrs) |> Repo.insert(), + :ok <- Spaces.put_vector(entry.id, target, space, vector) do + entry + else + {:error, reason} -> Repo.rollback(reason) + end + end) + end + + defp target_space(space_id, vector, body) do + case active_locked() do + %SpaceRow{id: ^space_id} = row -> + {space_id, SpaceRow.space(row), vector} + + nil -> + with {:ok, %{space: space, space_id: ^space_id}} <- serving(), + {:ok, row} <- Spaces.pin_if_empty(space) do + {row.id, space, vector} + else + other -> Repo.rollback({:not_pinned, other}) + end + + %SpaceRow{} -> + case embed([body]) do + {:ok, [v], id} -> {id, SpaceRow.space(Spaces.get(id)), v} + {:error, reason} -> Repo.rollback(reason) + end + end + end + + if @adapter == Ecto.Adapters.Postgres do + defp active_locked, + do: Repo.one(from(s in SpaceRow, where: s.active == true, lock: "FOR SHARE")) + else + defp active_locked, do: Spaces.active() + end + + @doc """ + The nearest existing memory to a vector (embedded in `space_id`) within the scopes, when its + cosine reaches `threshold`; nil otherwise, and nil when the search refuses. + """ + @spec near(String.t(), [String.t()], Embedder.vector(), Space.id(), float()) :: + VectorStore.hit() | nil + def near(persona_id, scopes, vector, space_id, threshold) do + case search_vector(persona_id, scopes, vector, space_id, 1) do + {:ok, [%{score: score} = hit]} when score >= threshold -> hit + _ -> nil + end + end + + ## Runtime faults: the last one per space, until a success clears it + + @doc "The recorded runtime fault for a space, or nil." + @spec fault(Space.id()) :: term() | nil + def fault(space_id) do + case :persistent_term.get(@fault_key, nil) do + {^space_id, reason} -> {:fault, reason} _ -> nil end end + @doc false + @spec put_fault(Space.id(), term()) :: term() + def put_fault(space_id, reason) do + if :persistent_term.get(@fault_key, nil) != {space_id, reason} do + :persistent_term.put(@fault_key, {space_id, reason}) + end + + reason + end + + @doc "Forgets every recorded runtime fault (a test starting clean)." + @spec clear_fault(:all) :: :ok + def clear_fault(:all) do + if :persistent_term.get(@fault_key, nil) != nil, do: :persistent_term.erase(@fault_key) + :ok + end + + # A successful embed clears an embed fault, a successful search a store fault; neither clears + # the other's, so an endpoint coming back does not hide a store that still refuses to rank. + defp clear_fault(space_id, kind) do + case :persistent_term.get(@fault_key, nil) do + {^space_id, reason} -> if fault_kind(reason) == kind, do: :persistent_term.erase(@fault_key) + _ -> :ok + end + + :ok + end + + defp fault_kind({:mixed_space, _}), do: :search + defp fault_kind(_), do: :embed + @doc "Removes a semantic memory; logged." @spec remove(Entry.t(), keyword()) :: {:ok, Entry.t()} | {:error, term()} def remove(%Entry{tier: "semantic"} = entry, opts) do @@ -181,8 +498,9 @@ defmodule Trinity.Memory.Semantic do @doc """ The packaged binary's vector check (slice 032, AC7, `--smoke`): three rows of the fake - embedder's deterministic vectors on a throwaway persona, the nearest expected first through - the store in force, all rolled back. `{:ok, store}` or `{:error, reason}`. + embedder's deterministic vectors on a throwaway persona, in a throwaway space of their own, + the nearest expected first through the store in force, all rolled back; the store's active + space is never touched. `{:ok, store}` or `{:error, reason}`. """ @spec smoke() :: {:ok, module()} | {:error, term()} def smoke do @@ -193,28 +511,30 @@ defmodule Trinity.Memory.Semantic do Trinity.Personas.create(%{name: "smoke-#{System.unique_integer([:positive])}"}) scope = AlwaysOn.persona_scope(persona.id) + space = %{Fake.space() | model_id: "smoke:" <> Fake.model_id(), dim: 384} + {:ok, row} = Spaces.register(space) texts = ["the cat sat", "quarterly tax filing", "a dog barked"] ids = for {t, i} <- Enum.with_index(texts) do attrs = %{persona_id: persona.id, scope: scope, key: "k#{i}", body: t} {:ok, e} = %Entry{} |> Entry.semantic_changeset(attrs) |> Repo.insert() - :ok = VectorStore.upsert(e.id, Fake.vector(t), Fake.model_id()) + :ok = Spaces.put_vector(e.id, row.id, space, Fake.vector(t)) e.id end - filter = %{persona_id: persona.id, scopes: [scope], model: Fake.model_id()} - hits = VectorStore.search(Fake.vector("a dog barked"), 3, filter) - expected = Enum.at(ids, 2) result = - case hits do - [%{id: ^expected, score: score}, _, _] when score > 0.999 -> + case VectorStore.search(Fake.vector("a dog barked"), 3, filter(persona.id, [scope], row)) do + {:ok, [%{id: ^expected, score: score}, _, _]} when score > 0.999 -> {:ok, VectorStore.impl()} - _ -> + {:ok, hits} -> {:error, {:wrong_order, Enum.map(hits, &{&1.entry.body, Float.round(&1.score, 3)})}} + + {:error, reason} -> + {:error, reason} end Repo.rollback(result) @@ -228,24 +548,13 @@ defmodule Trinity.Memory.Semantic do e -> {:error, Exception.message(e)} end - defp vector_for(body, nil), do: Embedder.embed([body]) - defp vector_for(_body, vector) when is_list(vector), do: {:ok, [vector]} - - defp insert(attrs, vector) do - attrs = - Map.merge(attrs, %{ - embedding: Embedder.to_binary(vector), - embedding_model: model(), - embedding_dim: length(vector) - }) - - %Entry{} |> Entry.semantic_changeset(attrs) |> Repo.insert() + defp serving_started(Trinity.Memory.Embedders.Bumblebee) do + if is_pid(Process.whereis(Trinity.Memory.Embedders.Bumblebee.serving_name())), + do: :ok, + else: {:off, :serving_not_started} end - defp serving?(Trinity.Memory.Embedders.Bumblebee), - do: is_pid(Process.whereis(Trinity.Memory.Embedders.Bumblebee.serving_name())) - - defp serving?(_), do: true + defp serving_started(_), do: :ok defp to_atom(k) when is_atom(k), do: k defp to_atom(k) when is_binary(k), do: String.to_existing_atom(k) diff --git a/lib/trinity/memory/space.ex b/lib/trinity/memory/space.ex new file mode 100644 index 0000000..2e1ebc2 --- /dev/null +++ b/lib/trinity/memory/space.ex @@ -0,0 +1,212 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.Memory.Space do + @moduledoc """ + An embedding space (slice 133): everything that decides what a vector means, and the + identifier every stored vector carries. + + Two vectors can be compared only when the same model, at the same revision, with the same + weights and tokenizer, pooled, normalised, quantized and prompted the same way, by the same + runtime, produced both. Before this slice a row recorded `embedding_model` alone, so two + revisions of a model, a truncated and an untruncated output, or the same model behind a + different runtime all looked like one space. A space ID is a digest over the fourteen identity + fields the owner set (slice 133 NOTES, D4-schema): + + | field | what it pins | + |---|---| + | `model_id` | the model's name | + | `revision` | the upstream commit the weights came from | + | `weights_digest` | SHA-256 of the weights file this runtime reads | + | `tokenizer_digest` | SHA-256 of the tokenizer data | + | `dim` | the output width after any truncation | + | `pooling` | `mean`, `cls`, `last`, or `provider` when an endpoint pools | + | `normalisation` | `l2` or `none` | + | `quantization` | `f32` or `int8`, of the weights and the stored vectors | + | `query_prompt` | the template a query is wrapped in (`""` for none) | + | `document_prompt` | the template a document is wrapped in | + | `max_input` | the maximum input tokens and the truncation policy | + | `runtime` | the runtime and its version | + | `locality` | `in_process`, `within_boundary` or `external` | + | `num_ctx` | the context length an endpoint runtime is pinned to | + + The ID is the lowercase hex SHA-256 of the manifest's RFC 8785 canonical JSON, the + canonicaliser approval fingerprints already use (`jcs`). Every value is a string or an + integer: a field this tree cannot know is the string `"unrecorded"`, one that does not apply is + `"none"`, and neither is ever `null`, so "missing" and "not known" cannot be confused. + `test/property/space_identity_test.exs` holds the rule that changing any one field changes + the ID. + """ + + @enforce_keys [ + :model_id, + :revision, + :weights_digest, + :tokenizer_digest, + :dim, + :pooling, + :normalisation, + :quantization, + :query_prompt, + :document_prompt, + :max_input_tokens, + :truncation, + :runtime, + :runtime_version, + :locality, + :num_ctx + ] + defstruct @enforce_keys + + @type value :: String.t() | non_neg_integer() + + @type t :: %__MODULE__{ + model_id: String.t(), + revision: String.t(), + weights_digest: String.t(), + tokenizer_digest: String.t(), + dim: pos_integer(), + pooling: String.t(), + normalisation: String.t(), + quantization: String.t(), + query_prompt: String.t(), + document_prompt: String.t(), + max_input_tokens: value(), + truncation: String.t(), + runtime: String.t(), + runtime_version: String.t(), + locality: String.t(), + num_ctx: value() + } + + @typedoc "A space ID: 64 lowercase hex characters." + @type id :: String.t() + + @identity_fields ~w(model_id revision weights_digest tokenizer_digest dim pooling normalisation + quantization query_prompt document_prompt max_input runtime locality num_ctx) + + @localities ~w(in_process within_boundary external) + + @doc "The fourteen identity fields, in the order the moduledoc lists them." + @spec identity_fields() :: [String.t()] + def identity_fields, do: @identity_fields + + @doc "The three declared localities (D2), as the manifest spells them." + @spec localities() :: [String.t()] + def localities, do: @localities + + @doc """ + The manifest: the fourteen fields as a map with string keys, the two compound fields + (`max_input`, `runtime`) as maps of their parts. This is what the ID digests and what the + `embedding_spaces` row stores. + """ + @spec manifest(t()) :: %{String.t() => value() | map()} + def manifest(%__MODULE__{} = s) do + %{ + "model_id" => s.model_id, + "revision" => s.revision, + "weights_digest" => s.weights_digest, + "tokenizer_digest" => s.tokenizer_digest, + "dim" => s.dim, + "pooling" => s.pooling, + "normalisation" => s.normalisation, + "quantization" => s.quantization, + "query_prompt" => s.query_prompt, + "document_prompt" => s.document_prompt, + "max_input" => %{"tokens" => s.max_input_tokens, "truncation" => s.truncation}, + "runtime" => %{"name" => s.runtime, "version" => s.runtime_version}, + "locality" => s.locality, + "num_ctx" => s.num_ctx + } + end + + @doc "A space back from its stored manifest." + @spec from_manifest(map()) :: t() + def from_manifest(%{} = m) do + %__MODULE__{ + model_id: m["model_id"], + revision: m["revision"], + weights_digest: m["weights_digest"], + tokenizer_digest: m["tokenizer_digest"], + dim: m["dim"], + pooling: m["pooling"], + normalisation: m["normalisation"], + quantization: m["quantization"], + query_prompt: m["query_prompt"], + document_prompt: m["document_prompt"], + max_input_tokens: m["max_input"]["tokens"], + truncation: m["max_input"]["truncation"], + runtime: m["runtime"]["name"], + runtime_version: m["runtime"]["version"], + locality: m["locality"], + num_ctx: m["num_ctx"] + } + end + + @doc """ + The space ID: SHA-256 of the manifest's RFC 8785 canonical JSON, lowercase hex. Takes a + space or a manifest (the migration computes legacy IDs from manifests it writes itself). + """ + @spec id(t() | map()) :: id() + def id(%__MODULE__{} = space), do: id(manifest(space)) + + def id(%{} = manifest), + do: + manifest |> Jcs.encode() |> then(&:crypto.hash(:sha256, &1)) |> Base.encode16(case: :lower) + + @doc """ + The space of a row written before slice 133, which recorded only `embedding_model` and the + dimension: every field it did not record is `"unrecorded"`. The migration that moved those + rows computes the same manifest with its own frozen copy of this function, and + `test/trinity/memory/migration_test.exs` holds the two equal. The Bumblebee embedder answers + this space too, because it loads its model from the hub's moving `main` and so has no + revision or digest to record (slice 133 NOTES, decision 4). + """ + @spec legacy(String.t(), pos_integer()) :: t() + def legacy(embedding_model, dim) when is_binary(embedding_model) and is_integer(dim) do + u = "unrecorded" + + %__MODULE__{ + model_id: embedding_model, + revision: u, + weights_digest: u, + tokenizer_digest: u, + dim: dim, + pooling: u, + normalisation: u, + quantization: "f32", + query_prompt: u, + document_prompt: u, + max_input_tokens: u, + truncation: u, + runtime: u, + runtime_version: u, + locality: u, + num_ctx: u + } + end + + @doc """ + A vector as a row of this space stores it: float32 little-endian, or for an `int8` space the + vector scaled so its largest magnitude is 127, one signed byte a component + (`Trinity.Memory.Scorer.quantize/1`). Takes the quantization or anything carrying one. + """ + @spec encode_vector(t() | String.t(), [float()]) :: binary() + def encode_vector(%__MODULE__{quantization: q}, floats), do: encode_vector(q, floats) + def encode_vector("int8", floats), do: Trinity.Memory.Scorer.quantize(floats) + def encode_vector(_f32, floats), do: for(f <- floats, into: <<>>, do: <>) + + @doc "A stored vector back to floats (an int8 vector unscaled: direction is what it keeps)." + @spec decode_vector(t() | String.t(), binary()) :: [float()] + def decode_vector(%__MODULE__{quantization: q}, bin), do: decode_vector(q, bin) + def decode_vector("int8", bin), do: Trinity.Memory.Scorer.to_floats(bin) + def decode_vector(_f32, bin), do: for(<>, do: f) + + @doc "How many bytes a stored vector of this width and quantization is." + @spec vector_bytes(String.t(), pos_integer()) :: pos_integer() + def vector_bytes("int8", dim), do: dim + def vector_bytes(_f32, dim), do: dim * 4 + + @doc "The first twelve hex characters, for logs and the memory page." + @spec short(id()) :: String.t() + def short(id) when is_binary(id), do: binary_part(id, 0, min(12, byte_size(id))) +end diff --git a/lib/trinity/memory/space_row.ex b/lib/trinity/memory/space_row.ex new file mode 100644 index 0000000..2ad3841 --- /dev/null +++ b/lib/trinity/memory/space_row.ex @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.Memory.SpaceRow do + @moduledoc """ + One row of `embedding_spaces` (slice 133): a space this store has held vectors in, its + manifest (`Trinity.Memory.Space`), whether it is complete (every semantic memory embedded in + it) or still being built by a re-tier, and whether it is the store's active space. At most one + row is active; the database refuses a second (`embedding_spaces_one_active`). + """ + use Ecto.Schema + + @primary_key {:id, :string, autogenerate: false} + + @type t :: %__MODULE__{} + + schema "embedding_spaces" do + field :manifest, :map + field :dim, :integer + field :quantization, :string + field :state, :string, default: "complete" + field :active, :boolean, default: false + field :completed_at, :utc_datetime_usec + timestamps(type: :utc_datetime_usec) + end + + @doc "The space this row records." + @spec space(t()) :: Trinity.Memory.Space.t() + def space(%__MODULE__{manifest: m}), do: Trinity.Memory.Space.from_manifest(m) +end diff --git a/lib/trinity/memory/spaces.ex b/lib/trinity/memory/spaces.ex new file mode 100644 index 0000000..6cd0241 --- /dev/null +++ b/lib/trinity/memory/spaces.ex @@ -0,0 +1,394 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.Memory.Spaces do + @moduledoc """ + The store's embedding spaces and its one active space (slice 133, D3-pinning). + + **The active space changes only by the operator's action.** Nothing here is called on a + boot path, on a failure, or because configuration changed: a store pinned to space A while + configuration names an embedder for space B answers semantic recall OFF with a named reason + (`Trinity.Memory.Semantic.status/0`), keeps the pointer where it is, and enqueues nothing. The + only implicit write is the first pin of an **empty** store (no active space, no vector in + it), which is not a switch: there is nothing to switch from (NOTES, decision 2). + + **Re-tier** (`retier/2`, `mix trinity.space.retier`) builds the target space beside the active + one, embedding every semantic memory into it in batches, each batch its own transaction, while + the store goes on answering from the active space. When no memory lacks a vector in the + target, one transaction marks it complete and moves the pointer. The old space's rows stay. + + **Drop** (`drop/2`, `mix trinity.space.drop --confirm`) deletes a space's vectors and its + row; without `confirm: true` it refuses, and it never drops the active space. + + On Postgres each space has its own partial expression HNSW index over the untyped `vector` + column (`index_name/1`), created when the space is registered. + """ + + import Ecto.Query + + alias Trinity.Memory.{Entry, Space, SpaceRow, Vector} + alias Trinity.Repo + + require Logger + + Module.register_attribute(__MODULE__, :sobelow_skip, persist: true) + + @adapter Application.compile_env(:trinity, :db_adapter, Ecto.Adapters.SQLite3) + + @doc "The active space's row, or nil." + @spec active() :: SpaceRow.t() | nil + def active, do: Repo.one(from(s in SpaceRow, where: s.active == true)) + + @doc "A space's row by id, or nil." + @spec get(Space.id()) :: SpaceRow.t() | nil + def get(id), do: Repo.get(SpaceRow, id) + + @doc "Every space's row with its vector count, the active one first." + @spec list() :: [%{row: SpaceRow.t(), vectors: non_neg_integer()}] + def list do + counts = + from(v in Vector, group_by: v.space_id, select: {v.space_id, count(v.memory_id)}) + |> Repo.all() + |> Map.new() + + from(s in SpaceRow, order_by: [desc: s.active, asc: s.inserted_at]) + |> Repo.all() + |> Enum.map(&%{row: &1, vectors: Map.get(counts, &1.id, 0)}) + end + + @doc "How many vectors a space holds." + @spec count(Space.id()) :: non_neg_integer() + def count(id), do: Repo.aggregate(from(v in Vector, where: v.space_id == ^id), :count) + + @doc "True when the store holds no vector in any space." + @spec empty?() :: boolean() + def empty?, do: not Repo.exists?(from(v in Vector, select: 1)) + + @doc """ + Records a space (idempotent): its row, `state` `"complete"` unless told `"building"`, and on + Postgres its index. Returns the row as stored. + """ + @spec register(Space.t(), String.t()) :: {:ok, SpaceRow.t()} + def register(%Space{} = space, state \\ "complete") when state in ["complete", "building"] do + id = Space.id(space) + + case get(id) do + %SpaceRow{} = row -> + {:ok, row} + + nil -> + now = DateTime.utc_now() + + row = %SpaceRow{ + id: id, + manifest: Space.manifest(space), + dim: space.dim, + quantization: space.quantization, + state: state, + active: false, + completed_at: if(state == "complete", do: now) + } + + {:ok, row} = Repo.insert(row, on_conflict: :nothing, conflict_target: :id) + create_index(id, space.dim) + {:ok, get(id) || row} + end + end + + @doc """ + The first pin of an empty store: when there is no active space and no vector in the store, + `space` becomes active. `{:ok, row}` when the store is (now) pinned to `space`; + `{:error, {:pinned_elsewhere, id}}` when it is pinned to another; `{:error, :store_not_empty}` + when it holds vectors and no active space (only an operator's re-tier pins it then). + """ + @spec pin_if_empty(Space.t()) :: {:ok, SpaceRow.t()} | {:error, term()} + def pin_if_empty(%Space{} = space) do + id = Space.id(space) + + Repo.transaction(fn -> + case active() do + %SpaceRow{id: ^id} = row -> row + %SpaceRow{id: other} -> Repo.rollback({:pinned_elsewhere, other}) + nil -> first_pin(space, id) + end + end) + end + + defp first_pin(space, id) do + if empty?() do + {:ok, _} = register(space) + {1, _} = Repo.update_all(from(s in SpaceRow, where: s.id == ^id), set: [active: true]) + Logger.info("memory: the empty store is pinned to space #{Space.short(id)}") + get(id) + else + Repo.rollback(:store_not_empty) + end + end + + @doc """ + Re-tier: builds `embedder`'s space beside the active one and, when it is complete, moves the + pointer in one transaction. Options: `batch:` (64), `on_batch:` (a function called with the + running count, for progress and for tests that act mid-build). `{:ok, %{space: id, previous: + id | nil, embedded: n}}`, or `{:error, reason}`: `:already_active`, an embedder that is not + available, or an embedding error (the partial space is left `building`, and a re-run resumes it). + """ + @spec retier(module(), keyword()) :: {:ok, map()} | {:error, term()} + def retier(embedder, opts \\ []) when is_atom(embedder) do + space = embedder.space() + id = Space.id(space) + previous = active() + + cond do + previous && previous.id == id -> + {:error, :already_active} + + (availability = embedder.availability()) != :ok -> + {:error, {:embedder_unavailable, availability}} + + true -> + {:ok, _} = register(space, "building") + build(embedder, space, id, previous, Keyword.get(opts, :batch, 64), opts, 0) + end + end + + defp build(embedder, space, id, previous, batch, opts, done) do + case missing(id, batch) do + [] -> + case cutover(id) do + :ok -> + Logger.info( + "memory: re-tier complete, space #{Space.short(id)} active (#{done} embedded)" + ) + + {:ok, %{space: id, previous: previous && previous.id, embedded: done}} + + :more -> + build(embedder, space, id, previous, batch, opts, done) + end + + entries -> + with {:ok, done} <- embed_batch(embedder, entries, id, space, done, opts) do + build(embedder, space, id, previous, batch, opts, done) + end + end + end + + defp embed_batch(embedder, entries, id, space, done, opts) do + with {:ok, vectors} <- embedder.embed(Enum.map(entries, & &1.body)) do + :ok = put_vectors(Enum.zip(Enum.map(entries, & &1.id), vectors), id, space) + done = done + length(entries) + if f = opts[:on_batch], do: f.(done) + {:ok, done} + end + end + + # Semantic memories with no vector in the space yet, oldest first. + defp missing(id, limit) do + from(e in Entry, + as: :e, + where: + e.tier == "semantic" and + not exists( + from(v in Vector, + where: v.memory_id == parent_as(:e).id and v.space_id == ^id, + select: 1 + ) + ), + order_by: [asc: e.inserted_at, asc: e.id], + limit: ^limit + ) + |> Repo.all() + end + + # One transaction: if every semantic memory has a vector in the space, mark it complete and + # make it the only active space; if a memory arrived since the last batch, say so and build on. + defp cutover(id) do + {:ok, result} = + Repo.transaction(fn -> + if missing(id, 1) == [] do + now = DateTime.utc_now() + Repo.update_all(from(s in SpaceRow, where: s.active == true), set: [active: false]) + + {1, _} = + Repo.update_all(from(s in SpaceRow, where: s.id == ^id), + set: [active: true, state: "complete", completed_at: now, updated_at: now] + ) + + :ok + else + :more + end + end) + + result + end + + @doc """ + Stores a memory's vector in a space (replacing one already there), in the space's encoding; + on Postgres the untyped `vector` column too, in the same statement. The space's width is + checked: a vector of another width is refused, never stored under the wrong space. + """ + @spec put_vector(String.t(), Space.id(), Space.t(), [float()], DateTime.t()) :: + :ok | {:error, term()} + def put_vector(memory_id, space_id, %Space{} = space, floats, now \\ DateTime.utc_now()), + do: put_vectors([{memory_id, floats}], space_id, space, now) + + @doc "`put_vector/5` for many memories at once, in one statement; all refused if one has the wrong width." + @spec put_vectors([{String.t(), [float()]}], Space.id(), Space.t(), DateTime.t()) :: + :ok | {:error, term()} + def put_vectors(pairs, space_id, %Space{dim: dim} = space, now \\ DateTime.utc_now()) do + case Enum.find(pairs, fn {_, floats} -> length(floats) != dim end) do + {_, floats} -> + {:error, {:wrong_width, length(floats), dim}} + + nil -> + rows = + for {memory_id, floats} <- pairs do + row(memory_id, space_id, space, floats, now) + end + + Repo.insert_all(table(), rows, + on_conflict: {:replace, replaced()}, + conflict_target: [:memory_id, :space_id] + ) + + :ok + end + end + + if @adapter == Ecto.Adapters.Postgres do + # The pgvector value is the stored vector decoded (for an int8 space, the int8 values' + # direction, which is all a cosine reads), so the index ranks exactly what the bytes hold. + defp row(memory_id, space_id, space, floats, now) do + bytes = Space.encode_vector(space, floats) + + %{ + memory_id: Ecto.UUID.dump!(memory_id), + space_id: space_id, + dim: space.dim, + vector: bytes, + embedding_vector: Pgvector.new(Space.decode_vector(space, bytes)), + inserted_at: now + } + end + + defp replaced, do: [:vector, :dim, :inserted_at, :embedding_vector] + + # Schemaless, for the column the schema does not map; Postgrex binds by the column's type. + defp table, do: "memory_embeddings" + + # Built without parallel workers: a parallel HNSW build allocates its working memory in + # dynamic shared memory, and a container's default 64 MB /dev/shm refuses it ("could not + # resize shared memory segment", the local pgvector container, 2026-10-08). The index of a + # new space is built while the space is empty, so a serial build costs nothing here. + # sobelow_skip reason: SQL.Query: the only thing spliced into the statement is a space ID, + # checked by `valid_id?/1` to be exactly 64 lowercase hex characters before use, and an integer + # width; a partial index needs its predicate as a literal, which a bound parameter cannot be. + @sobelow_skip ["SQL.Query"] + defp create_index(id, dim) when dim <= 2000 do + true = valid_id?(id) + + {:ok, _} = + Repo.transaction(fn -> + Repo.query!("SET LOCAL max_parallel_maintenance_workers = 0") + + Repo.query!( + "CREATE INDEX IF NOT EXISTS #{index_name(id)} ON memory_embeddings " <> + "USING hnsw ((embedding_vector::vector(#{dim})) vector_cosine_ops) WHERE space_id = '#{id}'" + ) + end) + + :ok + end + + defp create_index(_id, _dim), do: :ok + + # sobelow_skip reason: SQL.Query: the only thing spliced into the statement is a space ID, + # checked by `valid_id?/1` to be exactly 64 lowercase hex characters before use, and an integer + # width; a partial index needs its predicate as a literal, which a bound parameter cannot be. + @sobelow_skip ["SQL.Query"] + defp drop_index(id) do + true = valid_id?(id) + Repo.query!("DROP INDEX IF EXISTS #{index_name(id)}") + end + else + defp row(memory_id, space_id, space, floats, now) do + %{ + memory_id: memory_id, + space_id: space_id, + dim: space.dim, + vector: Space.encode_vector(space, floats), + inserted_at: now + } + end + + defp replaced, do: [:vector, :dim, :inserted_at] + + # Through the schema, so the bytes are bound as a BLOB: a schemaless binary reaches SQLite as + # TEXT, and `length/1` of TEXT counts characters, which the fit check would read as a misfit. + defp table, do: Vector + defp create_index(_id, _dim), do: :ok + defp drop_index(_id), do: :ok + end + + @doc "The name of a space's HNSW index on Postgres." + @spec index_name(Space.id()) :: String.t() + def index_name(id), do: "memory_embeddings_hnsw_" <> binary_part(id, 0, 16) + + @doc "True for a well-formed space ID (64 lowercase hex), the only thing ever spliced into SQL." + @spec valid_id?(term()) :: boolean() + def valid_id?(id), do: is_binary(id) and id =~ ~r/\A[0-9a-f]{64}\z/ + + @doc """ + Drops a space: its vectors, its index and its row. Refuses without `confirm: true` + (`{:error, :confirm_required}`), refuses the active space (`{:error, :active}`), and + `{:error, :not_found}` for an unknown id. A unique prefix of the id (12 characters or more, as + `list/0` prints them) is accepted. `{:ok, %{space: id, vectors: n}}`. + """ + @spec drop(String.t(), keyword()) :: {:ok, map()} | {:error, term()} + def drop(id_or_prefix, opts \\ []) do + with {:ok, row} <- resolve(id_or_prefix), + :ok <- confirmed(opts), + :ok <- inactive(row) do + {:ok, n} = + Repo.transaction(fn -> + {n, _} = Repo.delete_all(from(v in Vector, where: v.space_id == ^row.id)) + Repo.delete!(row) + n + end) + + drop_index(row.id) + Logger.info("memory: space #{Space.short(row.id)} dropped (#{n} vectors)") + {:ok, %{space: row.id, vectors: n}} + end + end + + defp confirmed(opts), + do: if(Keyword.get(opts, :confirm) == true, do: :ok, else: {:error, :confirm_required}) + + defp inactive(%SpaceRow{active: true}), do: {:error, :active} + defp inactive(%SpaceRow{}), do: :ok + + @doc "A space by full id or a unique prefix of 12 hex characters or more." + @spec resolve(String.t()) :: {:ok, SpaceRow.t()} | {:error, term()} + def resolve(id_or_prefix) when is_binary(id_or_prefix) do + cond do + valid_id?(id_or_prefix) -> + case get(id_or_prefix) do + nil -> {:error, :not_found} + row -> {:ok, row} + end + + id_or_prefix =~ ~r/\A[0-9a-f]{12,63}\z/ -> + rows = Repo.all(from(s in SpaceRow, where: like(s.id, ^(id_or_prefix <> "%")))) + + case rows do + [row] -> {:ok, row} + [] -> {:error, :not_found} + _ -> {:error, :ambiguous} + end + + true -> + {:error, :not_found} + end + end +end diff --git a/lib/trinity/memory/static_artifact.ex b/lib/trinity/memory/static_artifact.ex new file mode 100644 index 0000000..98448d8 --- /dev/null +++ b/lib/trinity/memory/static_artifact.ex @@ -0,0 +1,191 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.Memory.StaticArtifact do + @moduledoc """ + The static embedder's weights file (slice 133): one file holding the vocabulary and the + embedding matrix, derived deterministically from the upstream model at a pinned revision by + `mix trinity.static.build`, so its SHA-256 can be pinned in code and checked at load. + + Layout, all integers little-endian: + + "TSW1" magic, 4 bytes + header_len :: 32 then the header, RFC 8785 JSON: model, revision, the upstream + files' SHA-256, rows, dim, quantization, source_dim + vocab_len :: 32 then the vocabulary, one token per line, line n is id n + scales int8 only: rows x float32, one scale per row + matrix rows x dim, int8 (q = round(x / scale), scale = max|x| / 127) + or float32 + + Truncation to `dim` is the first `dim` columns, which is what Matryoshka truncation is and what + sentence-transformers' `truncate_dim` does. The int8 scale is per row (per token), so one + large-valued token does not crush the precision of the rest. + """ + + @magic "TSW1" + + @enforce_keys [:header, :vocab, :rows, :dim, :quantization, :scales, :matrix] + defstruct @enforce_keys + + @type quantization :: :int8 | :f32 + @type t :: %__MODULE__{ + header: map(), + vocab: [String.t()], + rows: pos_integer(), + dim: pos_integer(), + quantization: quantization(), + scales: binary() | nil, + matrix: binary() + } + + @doc """ + Builds the file's bytes from a float32 row-major matrix (`source_dim` columns), the vocabulary + and the provenance header fields. Deterministic: the same inputs give the same bytes. + """ + @spec build(binary(), pos_integer(), pos_integer(), [String.t()], keyword()) :: binary() + def build(f32_matrix, rows, source_dim, vocab, opts) do + dim = Keyword.fetch!(opts, :dim) + quantization = Keyword.fetch!(opts, :quantization) + + unless byte_size(f32_matrix) == rows * source_dim * 4 and dim <= source_dim and + length(vocab) == rows do + raise ArgumentError, "the matrix, the vocabulary and the dimensions do not agree" + end + + truncated = for r <- 0..(rows - 1), do: binary_part(f32_matrix, r * source_dim * 4, dim * 4) + + {scales, matrix} = + case quantization do + :f32 -> {"", IO.iodata_to_binary(truncated)} + :int8 -> quantize(truncated) + end + + header = + opts + |> Keyword.get(:provenance, %{}) + |> Map.merge(%{ + "rows" => rows, + "dim" => dim, + "quantization" => Atom.to_string(quantization), + "source_dim" => source_dim + }) + |> Jcs.encode() + + vocab_bin = Enum.join(vocab, "\n") + + IO.iodata_to_binary([ + @magic, + <>, + header, + <>, + vocab_bin, + scales, + matrix + ]) + end + + defp quantize(rows) do + {scales, qs} = rows |> Enum.map(&quantize_row/1) |> Enum.unzip() + {IO.iodata_to_binary(scales), IO.iodata_to_binary(qs)} + end + + # One row: its scale (as the file stores it, float32, so quantizing and dequantizing agree) + # and its bytes. + defp quantize_row(row) do + floats = for <>, do: x + max = floats |> Enum.map(&abs/1) |> Enum.max() + <> = <> + {<>, for(x <- floats, into: <<>>, do: <>)} + end + + defp q8(_x, scale) when scale == 0.0, do: 0 + defp q8(x, scale), do: x |> Kernel./(scale) |> round() |> max(-127) |> min(127) + + @doc "Parses the file's bytes. `{:error, reason}` for anything that is not a whole artifact." + @spec parse(binary()) :: {:ok, t()} | {:error, term()} + def parse(<<@magic, hlen::32-little, rest::binary>>) when byte_size(rest) >= hlen do + <> = rest + + with {:ok, header} <- Jason.decode(header_json), + %{"rows" => rows, "dim" => dim, "quantization" => q} when is_integer(rows) <- header, + <> when byte_size(rest) >= vlen <- rest, + <> <- rest, + {:ok, quantization} <- quantization(q), + {:ok, scales, matrix} <- split_tables(rest, rows, dim, quantization) do + vocab = String.split(vocab_bin, "\n") + + if length(vocab) == rows, + do: + {:ok, + %__MODULE__{ + header: header, + vocab: vocab, + rows: rows, + dim: dim, + quantization: quantization, + scales: scales, + matrix: matrix + }}, + else: {:error, :vocab_rows_mismatch} + else + {:error, reason} -> {:error, reason} + _ -> {:error, :malformed} + end + end + + def parse(_), do: {:error, :not_an_artifact} + + defp quantization("int8"), do: {:ok, :int8} + defp quantization("f32"), do: {:ok, :f32} + defp quantization(other), do: {:error, {:unknown_quantization, other}} + + defp split_tables(rest, rows, dim, :int8) do + {s, m} = {rows * 4, rows * dim} + + case rest do + <> -> {:ok, scales, matrix} + _ -> {:error, :truncated} + end + end + + defp split_tables(rest, rows, dim, :f32) do + m = rows * dim * 4 + + case rest do + <> -> {:ok, nil, matrix} + _ -> {:error, :truncated} + end + end + + @doc "The row for a token id as floats (dequantized for int8)." + @spec row(t(), non_neg_integer()) :: [float()] + def row(%__MODULE__{quantization: :int8, dim: dim, scales: scales, matrix: m}, id) do + <> = binary_part(scales, id * 4, 4) + for <>, do: q * scale + end + + def row(%__MODULE__{quantization: :f32, dim: dim, matrix: m}, id) do + for <>, do: x + end + + @doc """ + Reads the float32 matrix out of a `model.safetensors` holding exactly one 2-D F32 tensor (the + static model's `embedding.weight`). `{:ok, matrix, rows, cols}`. + """ + @spec read_safetensors(binary()) :: + {:ok, binary(), pos_integer(), pos_integer()} | {:error, term()} + def read_safetensors(<>) when byte_size(rest) >= hlen do + <> = rest + + with {:ok, h} <- Jason.decode(header), + [{_name, %{"dtype" => "F32", "shape" => [rows, cols], "data_offsets" => [from, to]}}] <- + Enum.reject(h, fn {k, _} -> k == "__metadata__" end), + true <- to - from == rows * cols * 4 and byte_size(data) >= to do + {:ok, binary_part(data, from, to - from), rows, cols} + else + {:error, reason} -> {:error, reason} + _ -> {:error, :unexpected_safetensors_layout} + end + end + + def read_safetensors(_), do: {:error, :not_safetensors} +end diff --git a/lib/trinity/memory/vector.ex b/lib/trinity/memory/vector.ex new file mode 100644 index 0000000..4f15009 --- /dev/null +++ b/lib/trinity/memory/vector.ex @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.Memory.Vector do + @moduledoc """ + One row of `memory_embeddings` (slice 133): a memory's vector in one space, keyed + `(memory_id, space_id)`. `vector` is the bytes in the space's encoding (float32 little-endian, + or int8 for an int8 space: `Trinity.Memory.Space.encode_vector/2`) and `dim` its width. On + Postgres the same vector is also in the untyped `embedding_vector` column, which this schema + does not map: only `Trinity.Memory.VectorStores.Pgvector` reads and writes it. + """ + use Ecto.Schema + + @primary_key false + @foreign_key_type Trinity.UUID + + @type t :: %__MODULE__{} + + schema "memory_embeddings" do + field :memory_id, Trinity.UUID, primary_key: true + field :space_id, :string, primary_key: true + field :dim, :integer + field :vector, :binary + field :inserted_at, :utc_datetime_usec + end +end diff --git a/lib/trinity/memory/vector_store.ex b/lib/trinity/memory/vector_store.ex index b3d9c40..3fc01ca 100644 --- a/lib/trinity/memory/vector_store.ex +++ b/lib/trinity/memory/vector_store.ex @@ -2,26 +2,38 @@ # SPDX-License-Identifier: Apache-2.0 defmodule Trinity.Memory.VectorStore do @moduledoc """ - Where semantic memories' vectors live and how they are searched (slice 032). The scope - filter is a required argument of `search/3` (M6): a search never sees a scope the caller - did not name. The store in force is the adapter's: `Brute` on SQLite (the persona's rows - of the embedder in force loaded and scored in Elixir; docs/02: fine to about 10^5) and - `Pgvector` on Postgres. Both search only rows whose `embedding_model` is the embedder in - force's (NOTES decision 4). - """ + How semantic memories' vectors are searched (slice 032; slice 133 for spaces). The scope filter + is a required argument of `search/3` (M6): a search never sees a scope the caller did not + name. Slice 133: the filter also names **one space**, and a search ranks only that space's + vectors (`memory_embeddings` rows keyed `(memory_id, space_id)`). Vectors are written by + `Trinity.Memory.Spaces.put_vector/5`, the one writer for both adapters. - @type hit :: %{id: String.t(), score: float(), entry: Trinity.Memory.Entry.t()} - @type filter :: %{persona_id: String.t(), scopes: [String.t()], model: String.t()} + **A store refuses rather than ranks.** Before a search ranks anything it counts the space's + rows whose recorded width or byte length disagrees with the space; any such row is + `{:error, {:mixed_space, n}}` and nothing is returned. A row with the right width but another + model's numbers cannot be told from the row itself; the `(memory_id, space_id)` key and the + single writer are what keep those out. - @doc "Stores a vector on an existing semantic memory row." - @callback upsert(entry_id :: String.t(), vector :: [float()], model :: String.t()) :: - :ok | {:error, term()} + The store in force is the adapter's: `Brute` on SQLite (rows loaded and scored in Elixir: an + int8 space always by `Trinity.Memory.Scorer`, a float32 space by EXLA's product where that NIF + loads), `Pgvector` on Postgres (each space's own partial HNSW index). + """ - @doc "The `k` nearest rows by cosine within the filter, best first." - @callback search(query :: [float()], k :: pos_integer(), filter()) :: [hit()] + @type hit :: %{ + id: String.t(), + score: float(), + entry: Trinity.Memory.Entry.t(), + space_id: String.t() + } + @type filter :: %{ + persona_id: String.t(), + scopes: [String.t()], + space: Trinity.Memory.SpaceRow.t() + } - @doc "Removes a row's vector (the row itself is `AlwaysOn.remove/2`'s to remove)." - @callback delete(entry_id :: String.t()) :: :ok + @doc "The `k` nearest rows by cosine within the filter's persona, scopes and space, best first." + @callback search(query :: [float()], k :: pos_integer(), filter()) :: + {:ok, [hit()]} | {:error, term()} @doc "How many vectors the filter holds." @callback count(filter()) :: non_neg_integer() @@ -37,18 +49,27 @@ defmodule Trinity.Memory.VectorStore do end @doc "Delegates to the store in force." - @spec upsert(String.t(), [float()], String.t()) :: :ok | {:error, term()} - def upsert(id, vector, model), do: impl().upsert(id, vector, model) - - @doc "Delegates to the store in force." - @spec search([float()], pos_integer(), filter()) :: [hit()] + @spec search([float()], pos_integer(), filter()) :: {:ok, [hit()]} | {:error, term()} def search(query, k, filter), do: impl().search(query, k, filter) - @doc "Delegates to the store in force." - @spec delete(String.t()) :: :ok - def delete(id), do: impl().delete(id) - @doc "Delegates to the store in force." @spec count(filter()) :: non_neg_integer() def count(filter), do: impl().count(filter) + + @doc """ + How many of a space's rows disagree with it (a width other than the space's, or a byte + length its encoding cannot have). Both stores ask this before ranking. + """ + @spec misfits(Trinity.Memory.SpaceRow.t()) :: non_neg_integer() + def misfits(%Trinity.Memory.SpaceRow{id: id, dim: dim, quantization: q}) do + import Ecto.Query + bytes = Trinity.Memory.Space.vector_bytes(q, dim) + + Trinity.Repo.aggregate( + from(v in Trinity.Memory.Vector, + where: v.space_id == ^id and (v.dim != ^dim or fragment("length(?)", v.vector) != ^bytes) + ), + :count + ) + end end diff --git a/lib/trinity/memory/vector_stores/brute.ex b/lib/trinity/memory/vector_stores/brute.ex index bc232fb..20e3e1f 100644 --- a/lib/trinity/memory/vector_stores/brute.ex +++ b/lib/trinity/memory/vector_stores/brute.ex @@ -2,80 +2,101 @@ # SPDX-License-Identifier: Apache-2.0 defmodule Trinity.Memory.VectorStores.Brute do @moduledoc """ - The SQLite vector store (slice 032): vectors as float32 bytes on `memories.embedding`, - a search loads the filter's rows (a persona's semantic tier, the named scopes, the - embedder in force's model) and scores them by cosine: as one matrix product on EXLA when - the NIF is loaded (the local embedder's own backend), in Elixir over decoded lists when - it is not (the hosted embedder on a machine without EXLA). Measured at G3 over 10,000 - rows (docs/perf.md): loading the rows 93 ms, the Elixir cosine 571 ms, the EXLA product - 6 ms after its first compile for that row count (102 ms). docs/02's "fine to about 10^5" - holds for the product, not for the Elixir path; hnswlib is the step after. + The SQLite vector store (slice 032; spaces at slice 133): a search loads the filter's rows (a + persona's semantic tier, the named scopes, one space's vectors from `memory_embeddings`) and + scores them all. + + * **An int8 space** (the static floor) is scored by `Trinity.Memory.Scorer`, exact int8 cosine, + in pure Elixir, always: D-scoring makes the floor's path NIF-free, and AC7 holds it to that + (slice 133 NOTES, R5: before this, the static path loaded EXLA through this module). + * **A float32 space** (MiniLM) keeps slice 032's choice: one matrix product on EXLA's host + client when that NIF is loaded, the Elixir cosine over decoded lists when it is not + (docs/perf.md: at 10^4 rows the product is 24 ms after its compile, the lists 390 ms). + + Before ranking, a space with a row that does not fit it is refused + (`Trinity.Memory.VectorStore.misfits/1`). """ @behaviour Trinity.Memory.VectorStore + # The float32 path's EXLA product; a build without the neural group (`TRINITY_WITHOUT_ML=1`) + # compiles this module with none of these present and never reaches them (`exla?/0`). + @compile {:no_warn_undefined, [Nx, Nx.LinAlg, EXLA.Backend]} + import Ecto.Query - alias Trinity.Memory.{Embedder, Entry} + alias Trinity.Memory.{Embedder, Entry, Scorer, Space, SpaceRow, Vector, VectorStore} alias Trinity.Repo @impl true - def upsert(id, vector, model) when is_list(vector) do - {n, _} = - Repo.update_all(from(e in Entry, where: e.id == ^id), - set: [ - embedding: Embedder.to_binary(vector), - embedding_model: model, - embedding_dim: length(vector) - ] - ) - - if n == 1, do: :ok, else: {:error, :no_such_entry} + def search(query, k, %{persona_id: persona_id, scopes: scopes, space: %SpaceRow{} = space}) + when is_list(query) do + case VectorStore.misfits(space) do + 0 -> + rows = load(persona_id, scopes, space.id) + {:ok, rows |> rank(query, k, space) |> with_entries()} + + n -> + {:error, {:mixed_space, n}} + end end - @impl true - def search(query, k, %{persona_id: persona_id, scopes: scopes, model: model}) - when is_list(query) do + # The ids and the vectors only: the rows themselves are read for the `k` that rank, not for + # all of them (at 10^4 rows reading every entry was most of a search's time). + defp load(persona_id, scopes, space_id) do from(e in Entry, + join: v in Vector, + on: v.memory_id == e.id and v.space_id == ^space_id, where: e.persona_id == ^persona_id and e.tier == "semantic" and e.scope in ^scopes and - e.embedding_model == ^model and not is_nil(e.embedding) and is_nil(e.archived_at) + is_nil(e.archived_at), + select: {e.id, v.vector} ) |> Repo.all() - |> score(query) - |> Enum.sort_by(& &1.score, :desc) - |> Enum.take(k) end - @doc "Cosine of the query against every row's vector, as hits; the EXLA product when the NIF is loaded, Elixir otherwise." - @spec score([Entry.t()], Embedder.vector()) :: [Trinity.Memory.VectorStore.hit()] - def score([], _query), do: [] + defp rank([], _query, _k, _space), do: [] - def score(rows, query) do + defp rank(rows, query, k, %SpaceRow{quantization: "int8", id: space_id}) do + rows + |> Enum.map(fn {id, v} -> {id, v, Scorer.norm2(v), nil} end) + |> Scorer.exact(Scorer.quantize(query), k) + |> Enum.map(fn {id, score} -> %{id: id, score: score, space_id: space_id} end) + end + + defp rank(rows, query, k, %SpaceRow{id: space_id}) do + ids = Enum.map(rows, &elem(&1, 0)) scores = if exla?(), do: exla_scores(rows, query), else: elixir_scores(rows, query) - Enum.zip_with(rows, scores, fn e, s -> %{id: e.id, score: s, entry: e} end) + + ids + |> Enum.zip_with(scores, fn id, s -> %{id: id, score: s, space_id: space_id} end) + |> Enum.sort_by(&{-&1.score, &1.id}) + |> Enum.take(k) end - @doc "Cosines in Elixir over decoded lists (the path without EXLA); public so the suite can hold the two paths to the same numbers." - @spec elixir_scores([Entry.t()], Embedder.vector()) :: [float()] + defp with_entries([]), do: [] + + defp with_entries(hits) do + ids = Enum.map(hits, & &1.id) + entries = Map.new(Repo.all(from(e in Entry, where: e.id in ^ids)), &{&1.id, &1}) + for %{id: id} = hit <- hits, entry = entries[id], do: Map.put(hit, :entry, entry) + end + + @doc "Cosines in Elixir over decoded lists (the float32 path without EXLA), for rows `{id, bytes}`." + @spec elixir_scores([{term(), binary()}], Embedder.vector()) :: [float()] def elixir_scores(rows, query), - do: Enum.map(rows, &Embedder.cosine(query, Embedder.from_binary(&1.embedding))) + do: Enum.map(rows, fn {_, v} -> Embedder.cosine(query, Space.decode_vector("f32", v)) end) - @doc "Cosines as one (rows x dim) . dim product on the EXLA host client; a row of another width scores 0.0, as `cosine/2` refuses it." - @spec exla_scores([Entry.t()], Embedder.vector()) :: [float()] + @doc "Cosines as one (rows x dim) . dim product on the EXLA host client, for rows `{id, bytes}`." + @spec exla_scores([{term(), binary()}], Embedder.vector()) :: [float()] def exla_scores(rows, query) do dim = length(query) - fit = Enum.filter(rows, &(byte_size(&1.embedding) == dim * 4)) - scores = if fit == [], do: %{}, else: product(fit, query, dim) - Enum.map(rows, &Map.get(scores, &1.id, 0.0)) - end - defp product(fit, query, dim) do m = - fit - |> Enum.map(& &1.embedding) + rows + |> Enum.map(&elem(&1, 1)) |> IO.iodata_to_binary() |> Nx.from_binary(:f32, backend: EXLA.Backend) - |> Nx.reshape({length(fit), dim}) + |> Nx.reshape({length(rows), dim}) q = Nx.tensor(query, type: :f32, backend: EXLA.Backend) norms = Nx.multiply(Nx.LinAlg.norm(m, axes: [1]), Nx.LinAlg.norm(q)) @@ -84,28 +105,20 @@ defmodule Trinity.Memory.VectorStores.Brute do |> Nx.dot(q) |> Nx.divide(Nx.select(Nx.equal(norms, 0.0), 1.0, norms)) |> Nx.to_flat_list() - |> then(&Map.new(Enum.zip(Enum.map(fit, fn e -> e.id end), &1))) end defp exla?, do: Code.ensure_loaded?(EXLA.Backend) and Trinity.Memory.Embedders.Bumblebee.exla() == :ok @impl true - def delete(id) do - Repo.update_all(from(e in Entry, where: e.id == ^id), - set: [embedding: nil, embedding_model: nil, embedding_dim: nil] - ) - - :ok - end - - @impl true - def count(%{persona_id: persona_id, scopes: scopes, model: model}) do + def count(%{persona_id: persona_id, scopes: scopes, space: %SpaceRow{id: space_id}}) do Repo.aggregate( from(e in Entry, + join: v in Vector, + on: v.memory_id == e.id and v.space_id == ^space_id, where: e.persona_id == ^persona_id and e.tier == "semantic" and e.scope in ^scopes and - e.embedding_model == ^model and not is_nil(e.embedding) and is_nil(e.archived_at) + is_nil(e.archived_at) ), :count ) diff --git a/lib/trinity/memory/vector_stores/pgvector.ex b/lib/trinity/memory/vector_stores/pgvector.ex index 9426eb4..f06afae 100644 --- a/lib/trinity/memory/vector_stores/pgvector.ex +++ b/lib/trinity/memory/vector_stores/pgvector.ex @@ -2,92 +2,98 @@ # SPDX-License-Identifier: Apache-2.0 defmodule Trinity.Memory.VectorStores.Pgvector do @moduledoc """ - The Postgres vector store (slice 032): the same rows, with the vector also in - `memories.embedding_vector` (pgvector, `vector(384)`), searched by cosine distance - (`<=>`) under the HNSW index over the semantic tier, with pgvector 0.8's iterative scan so a - filtered query never comes back short. Keeps `embedding`, `embedding_model` - and `embedding_dim` in step with the SQLite store so an export reads the same columns on - both. Only vectors of 384 dimensions fit the column; another embedder's go to the bytes - column alone and are searched brute force here too. + The Postgres vector store (slice 032; spaces at slice 133). The vectors are in + `memory_embeddings.embedding_vector`, an **untyped** pgvector column, so a 256-wide space and a + 384-wide one share the table; each space has its own partial expression HNSW index, + `((embedding_vector::vector()) vector_cosine_ops) WHERE space_id = ''` + (`Trinity.Memory.Spaces.index_name/1`). A search orders by the same expression under the same + predicate, with the space id written into the statement as a literal so the planner can match + the partial index (a bound parameter cannot prove the predicate for a generic plan); the id is + checked to be 64 lowercase hex before it is ever spliced (`Spaces.valid_id?/1`). AC2's test + reads the plan with `EXPLAIN`. + + pgvector 0.8's iterative scan (`SET LOCAL hnsw.iterative_scan = strict_order`) keeps a + filtered query from coming back short (slice 032 NOTES, finding 12). A space wider than HNSW's + 2,000 dimensions has no index and is scored by `Brute`. """ @behaviour Trinity.Memory.VectorStore import Ecto.Query - alias Trinity.Memory.{Embedder, Entry} + alias Trinity.Memory.{Entry, SpaceRow, Spaces} alias Trinity.Repo - @column_dim 384 + Module.register_attribute(__MODULE__, :sobelow_skip, persist: true) @impl true - def upsert(id, vector, model) when is_list(vector) do - bytes = Embedder.to_binary(vector) + def search(query, k, %{persona_id: persona_id, scopes: scopes, space: %SpaceRow{} = space} = f) + when is_list(query) do + cond do + space.dim > 2000 -> + Trinity.Memory.VectorStores.Brute.search(query, k, f) - {n, _} = - Repo.update_all(from(e in Entry, where: e.id == ^id), - set: [embedding: bytes, embedding_model: model, embedding_dim: length(vector)] - ) + (n = misfits(space)) > 0 -> + {:error, {:mixed_space, n}} - if n == 1 and length(vector) == @column_dim do - Repo.query!("UPDATE memories SET embedding_vector = $1 WHERE id = $2", [ - Pgvector.new(vector), - Ecto.UUID.dump!(id) - ]) + true -> + {:ok, indexed(query, k, persona_id, scopes, space)} end - - if n == 1, do: :ok, else: {:error, :no_such_entry} end - @impl true - def search(query, k, %{persona_id: persona_id, scopes: scopes, model: model}) - when is_list(query) do - if length(query) == @column_dim do - vec = Pgvector.new(query) - - from(e in Entry, - where: - e.persona_id == ^persona_id and e.tier == "semantic" and e.scope in ^scopes and - e.embedding_model == ^model and not is_nil(e.embedding) and is_nil(e.archived_at), - order_by: fragment("embedding_vector <=> ?", ^vec), - limit: ^k, - select: {e, fragment("1 - (embedding_vector <=> ?)", ^vec)} + # `VectorStore.misfits/1`'s rows, and on this store one more kind: a row of an indexed space + # whose pgvector value is missing or of another width cannot be ranked by the index (its + # distance is NULL, or the cast fails); it does not fit the space's storage either. Each row + # counted once. + defp misfits(%SpaceRow{id: id, dim: dim, quantization: q}) do + %{rows: [[n]]} = + Repo.query!( + "SELECT count(*) FROM memory_embeddings WHERE space_id = $1 AND (dim <> $2 OR " <> + "length(vector) <> $3 OR embedding_vector IS NULL OR vector_dims(embedding_vector) <> $2)", + [id, dim, Trinity.Memory.Space.vector_bytes(q, dim)] ) - |> all_iterative() - |> Enum.map(fn {e, score} -> %{id: e.id, score: score * 1.0, entry: e} end) - else - Trinity.Memory.VectorStores.Brute.search(query, k, %{ - persona_id: persona_id, - scopes: scopes, - model: model - }) - end + + n end - # An HNSW scan hands back its `ef_search` (40) nearest index entries and the WHERE clause - # is applied after; a persona's few rows can all fall outside those candidates when the - # index holds many other entries (other personas' rows, and dead tuples from rolled-back - # inserts, which is how the postgres job saw two of three rows: run 35604770525). pgvector - # 0.8's iterative scan keeps walking the index until the LIMIT is met; `SET LOCAL` scopes - # it to this transaction (a savepoint inside the sandbox's). - defp all_iterative(query) do - {:ok, rows} = - Repo.transaction(fn -> - Repo.query!("SET LOCAL hnsw.iterative_scan = strict_order") - Repo.all(query) - end) + @doc """ + The SQL a search runs for a space, with its parameters' positions: `$1` the query vector, + `$2` the persona, `$3` the scopes, `$4` the limit. Public so AC2's test can `EXPLAIN` exactly + what a search executes. + """ + @spec sql(SpaceRow.t()) :: String.t() + def sql(%SpaceRow{id: id, dim: dim}) do + true = Spaces.valid_id?(id) and is_integer(dim) + expr = "(v.embedding_vector::vector(#{dim}))" - rows + "SELECT v.memory_id, 1 - (#{expr} <=> $1::vector(#{dim})) AS score " <> + "FROM memory_embeddings v JOIN memories e ON e.id = v.memory_id " <> + "WHERE v.space_id = '#{id}' AND e.persona_id = $2 AND e.tier = 'semantic' " <> + "AND e.scope = ANY($3) AND e.archived_at IS NULL " <> + "ORDER BY #{expr} <=> $1::vector(#{dim}) LIMIT $4" end - @impl true - def delete(id) do - Repo.update_all(from(e in Entry, where: e.id == ^id), - set: [embedding: nil, embedding_model: nil, embedding_dim: nil] - ) + # The vector is bound as the dequantized floats; for an int8 space those are the stored + # int8 values' direction, which is all a cosine reads. + # sobelow_skip reason: SQL.Query: `sql/1` splices only the space ID, checked to be 64 lowercase + # hex (`Spaces.valid_id?/1`), and the integer width; the planner can match a partial index only + # to a literal predicate. The query vector, the persona, the scopes and the limit are bound + # parameters. + @sobelow_skip ["SQL.Query"] + defp indexed(query, k, persona_id, scopes, space) do + params = [Pgvector.new(query), Ecto.UUID.dump!(persona_id), scopes, k] - Repo.query!("UPDATE memories SET embedding_vector = NULL WHERE id = $1", [Ecto.UUID.dump!(id)]) + {:ok, %{rows: rows}} = + Repo.transaction(fn -> + Repo.query!("SET LOCAL hnsw.iterative_scan = strict_order") + Repo.query!(sql(space), params) + end) - :ok + ids = Enum.map(rows, fn [id, _] -> Ecto.UUID.load!(id) end) + entries = Map.new(Repo.all(from(e in Entry, where: e.id in ^ids)), &{&1.id, &1}) + + for [id, score] <- rows, id = Ecto.UUID.load!(id), entry = entries[id] do + %{id: id, score: score * 1.0, entry: entry, space_id: space.id} + end end @impl true diff --git a/lib/trinity/memory/word_piece.ex b/lib/trinity/memory/word_piece.ex new file mode 100644 index 0000000..c8def59 --- /dev/null +++ b/lib/trinity/memory/word_piece.ex @@ -0,0 +1,254 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.Memory.WordPiece do + @moduledoc """ + The bert-base-uncased tokenizer in pure Elixir (slice 133): what the static embedder turns a + text into before it looks anything up. No NIF, so it runs wherever the BEAM does. + + It reproduces Hugging Face `tokenizers`' pipeline for the static model's `tokenizer.json` + (`BertNormalizer` with clean_text, CJK handling, accent stripping and lowercasing; + `BertPreTokenizer`; `WordPiece` with `[UNK]`, the `##` prefix and the 100-character rule), + encoded without special tokens, as sentence-transformers' `StaticEmbedding` encodes: + + 1. **Added tokens first.** The five special tokens (`[PAD]`, `[UNK]`, `[CLS]`, `[SEP]`, + `[MASK]`) are matched in the raw text, leftmost and longest, before anything is normalised, + and stand for their own ids; the text between them is tokenized on its own. + 2. **Normalise each character** by the table the reference itself produced + (`Trinity.Memory.WordPiece.Tables`): removed, a space, padded as a CJK ideograph, mapped + (NFD, non-spacing marks dropped, lowercased), or kept. Lowercasing is per character, as the + reference's is, so a final sigma stays `σ`. + 3. **Pre-tokenise**: split on whitespace, isolate punctuation. + 4. **WordPiece** each word: greedy longest match from the left, continuations prefixed `##`, + a word over 100 characters or with an unmatched remainder is one `[UNK]`. + + Parity with the reference is AC9's test (`test/trinity/memory/word_piece_parity_test.exs`), + over 5,000 natural sentences and 1,000 adversarial ones, one mismatched id failing it. + """ + + import Bitwise + + alias Trinity.Memory.WordPiece.Tables + + Module.register_attribute(__MODULE__, :sobelow_skip, persist: true) + + @special ~w([PAD] [UNK] [CLS] [SEP] [MASK]) + @max_chars 100 + @prefix "##" + + @enforce_keys [:vocab, :unk_id, :added] + defstruct [:vocab, :unk_id, :added] + + @type t :: %__MODULE__{ + vocab: %{String.t() => non_neg_integer()}, + unk_id: non_neg_integer(), + added: [{String.t(), non_neg_integer()}] + } + + # Sorted, disjoint `{lo, hi}` ranges as tuples, built at compile time, searched by bisection. + @remove List.to_tuple(Tables.remove()) + @space List.to_tuple(Tables.space()) + @cjk List.to_tuple(Tables.cjk()) + @punct List.to_tuple(Tables.punct()) + @ws List.to_tuple(Tables.ws()) + @mapping Tables.mapping() + + @doc """ + A tokenizer over a vocabulary: the tokens in id order (line `n` of a `vocab.txt` is id `n`). + The special tokens present in it are the added tokens. Raises if `[UNK]` is absent, as the + reference refuses such a vocabulary. + """ + @spec new([String.t()]) :: t() + def new(tokens) when is_list(tokens) do + vocab = tokens |> Enum.with_index() |> Map.new() + + unk_id = + Map.get(vocab, "[UNK]") || raise ArgumentError, "a WordPiece vocabulary needs [UNK]" + + added = + for t <- @special, id = Map.get(vocab, t), do: {t, id} + + %__MODULE__{vocab: vocab, unk_id: unk_id, added: added} + end + + @doc "Reads a `vocab.txt` (one token per line, line order is id order)." + # sobelow_skip reason: Traversal.FileModule: a vocabulary file named by the caller (a test, or + # an operator building an artifact); nothing a request or a model supplies reaches it. + @sobelow_skip ["Traversal.FileModule"] + @spec from_vocab_file(Path.t()) :: {:ok, t()} | {:error, term()} + def from_vocab_file(path) do + with {:ok, bin} <- File.read(path) do + {:ok, bin |> String.split("\n") |> drop_trailing_empty() |> new()} + end + rescue + e in ArgumentError -> {:error, {:vocab, Exception.message(e)}} + end + + defp drop_trailing_empty(lines) do + case List.last(lines) do + "" -> Enum.drop(lines, -1) + _ -> lines + end + end + + @doc """ + The token ids for a text, without special tokens around it. `opts[:normalize]` replaces the + normalisation step; it exists so AC9's test can show that a broken step (accents kept) fails + the parity set, and nothing in `lib/` passes it. + """ + @spec encode(t(), String.t(), keyword()) :: [non_neg_integer()] + def encode(%__MODULE__{} = tok, text, opts \\ []) when is_binary(text) do + normalize = Keyword.get(opts, :normalize, &normalize/1) + + text + |> split_added(tok.added) + |> Enum.flat_map(fn + {:added, id} -> + [id] + + {:text, segment} -> + segment |> normalize.() |> pre_tokenize() |> Enum.flat_map(&word(tok, &1)) + end) + end + + @doc "The token strings for a text (the ids looked back up), for tests and inspection." + @spec tokens(t(), String.t()) :: [String.t()] + def tokens(%__MODULE__{vocab: vocab} = tok, text) do + by_id = Map.new(vocab, fn {k, v} -> {v, k} end) + tok |> encode(text) |> Enum.map(&Map.fetch!(by_id, &1)) + end + + ## Added tokens: leftmost, longest, on the raw text + + defp split_added(text, []), do: [{:text, text}] + defp split_added(text, added), do: split_added(text, added, 0, 0, []) + + defp split_added(text, _added, from, at, acc) when at >= byte_size(text) do + Enum.reverse(push_text(acc, text, from, byte_size(text) - from)) + end + + defp split_added(text, added, from, at, acc) do + rest = binary_part(text, at, byte_size(text) - at) + + case longest_added(rest, added) do + {literal, id} -> + acc = acc |> push_text(text, from, at - from) |> then(&[{:added, id} | &1]) + next = at + byte_size(literal) + split_added(text, added, next, next, acc) + + nil -> + split_added(text, added, from, at + 1, acc) + end + end + + defp longest_added(rest, added) do + added + |> Enum.filter(fn {literal, _} -> String.starts_with?(rest, literal) end) + |> Enum.max_by(fn {literal, _} -> byte_size(literal) end, fn -> nil end) + end + + defp push_text(acc, _text, _from, 0), do: acc + defp push_text(acc, text, from, len), do: [{:text, binary_part(text, from, len)} | acc] + + ## Normalisation, one code point at a time, by the reference's own table + + @doc false + @spec normalize(String.t()) :: String.t() + def normalize(text) do + for <>, into: "", do: char(cp) + end + + defp char(cp) do + cond do + in?(cp, @remove) -> "" + in?(cp, @space) -> " " + in?(cp, @cjk) -> <> + cp >= 0xAC00 and cp <= 0xD7A3 -> hangul(cp) + true -> Map.get(@mapping, cp) || <> + end + end + + # The algorithmic decomposition of a Hangul syllable (Unicode chapter 3.12); the generator + # checked the reference produces exactly this for every syllable. + defp hangul(cp) do + s = cp - 0xAC00 + l = 0x1100 + div(s, 588) + v = 0x1161 + div(rem(s, 588), 28) + t = 0x11A7 + rem(s, 28) + if t == 0x11A7, do: <>, else: <> + end + + defp in?(cp, ranges), do: bsearch(ranges, cp, 0, tuple_size(ranges) - 1) + + defp bsearch(_t, _cp, lo, hi) when lo > hi, do: false + + defp bsearch(t, cp, lo, hi) do + mid = (lo + hi) >>> 1 + {a, b} = elem(t, mid) + + cond do + cp < a -> bsearch(t, cp, lo, mid - 1) + cp > b -> bsearch(t, cp, mid + 1, hi) + true -> true + end + end + + ## Pre-tokenisation: whitespace splits and is dropped, punctuation is its own piece + + @doc false + @spec pre_tokenize(String.t()) :: [String.t()] + def pre_tokenize(text) do + {words, current} = + for <>, reduce: {[], []} do + {words, current} -> + cond do + in?(cp, @ws) -> {flush(words, current), []} + in?(cp, @punct) -> {[<> | flush(words, current)], []} + true -> {words, [cp | current]} + end + end + + words |> flush(current) |> Enum.reverse() + end + + defp flush(words, []), do: words + defp flush(words, current), do: [current |> Enum.reverse() |> List.to_string() | words] + + ## WordPiece + + defp word(%__MODULE__{unk_id: unk} = tok, word) do + chars = String.codepoints(word) + + if length(chars) > @max_chars do + [unk] + else + case pieces(tok.vocab, chars, true, []) do + :bad -> [unk] + ids -> ids + end + end + end + + # Greedy longest match from the left: the longest prefix of what is left that is in the + # vocabulary (prefixed `##` after the first piece), then the rest; any remainder with no + # prefix in the vocabulary makes the whole word one [UNK]. + defp pieces(_vocab, [], _first?, acc), do: Enum.reverse(acc) + + defp pieces(vocab, chars, first?, acc) do + case longest(vocab, chars, length(chars), first?) do + nil -> :bad + {id, taken} -> pieces(vocab, Enum.drop(chars, taken), false, [id | acc]) + end + end + + defp longest(_vocab, _chars, 0, _first?), do: nil + + defp longest(vocab, chars, n, first?) do + candidate = chars |> Enum.take(n) |> Enum.join() + candidate = if first?, do: candidate, else: @prefix <> candidate + + case Map.fetch(vocab, candidate) do + {:ok, id} -> {id, n} + :error -> longest(vocab, chars, n - 1, first?) + end + end +end diff --git a/lib/trinity/memory/word_piece/tables.ex b/lib/trinity/memory/word_piece/tables.ex new file mode 100644 index 0000000..9d143c8 --- /dev/null +++ b/lib/trinity/memory/word_piece/tables.ex @@ -0,0 +1,3684 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +# GENERATED by scripts/static/wordpiece_tables.py from tokenizers 0.23.2. Do not edit. +defmodule Trinity.Memory.WordPiece.Tables do + @moduledoc """ + What the reference BERT normaliser and pre-tokeniser do to each code point (slice 133), + recorded by putting every code point through Hugging Face `tokenizers` 0.23.2 + one at a time (`scripts/static/wordpiece_tables.py`). Generated, not written from a Unicode + database: the reference classifies with its own, older, tables, and parity means matching + them rather than OTP's. + + Hangul syllables (U+AC00 to U+D7A3) are not listed: the generator checked that the reference + decomposes every one of them algorithmically, and `Trinity.Memory.WordPiece` does the same. + """ + + @remove [ + {0x0, 0x8}, + {0xB, 0xC}, + {0xE, 0x1F}, + {0x7F, 0x9F}, + {0xAD, 0xAD}, + {0x300, 0x36F}, + {0x483, 0x487}, + {0x591, 0x5BD}, + {0x5BF, 0x5BF}, + {0x5C1, 0x5C2}, + {0x5C4, 0x5C5}, + {0x5C7, 0x5C7}, + {0x600, 0x605}, + {0x610, 0x61A}, + {0x61C, 0x61C}, + {0x64B, 0x65F}, + {0x670, 0x670}, + {0x6D6, 0x6DD}, + {0x6DF, 0x6E4}, + {0x6E7, 0x6E8}, + {0x6EA, 0x6ED}, + {0x70F, 0x70F}, + {0x711, 0x711}, + {0x730, 0x74A}, + {0x7A6, 0x7B0}, + {0x7EB, 0x7F3}, + {0x816, 0x819}, + {0x81B, 0x823}, + {0x825, 0x827}, + {0x829, 0x82D}, + {0x859, 0x85B}, + {0x8E3, 0x902}, + {0x93A, 0x93A}, + {0x93C, 0x93C}, + {0x941, 0x948}, + {0x94D, 0x94D}, + {0x951, 0x957}, + {0x962, 0x963}, + {0x981, 0x981}, + {0x9BC, 0x9BC}, + {0x9C1, 0x9C4}, + {0x9CD, 0x9CD}, + {0x9E2, 0x9E3}, + {0xA01, 0xA02}, + {0xA3C, 0xA3C}, + {0xA41, 0xA42}, + {0xA47, 0xA48}, + {0xA4B, 0xA4D}, + {0xA51, 0xA51}, + {0xA70, 0xA71}, + {0xA75, 0xA75}, + {0xA81, 0xA82}, + {0xABC, 0xABC}, + {0xAC1, 0xAC5}, + {0xAC7, 0xAC8}, + {0xACD, 0xACD}, + {0xAE2, 0xAE3}, + {0xB01, 0xB01}, + {0xB3C, 0xB3C}, + {0xB3F, 0xB3F}, + {0xB41, 0xB44}, + {0xB4D, 0xB4D}, + {0xB56, 0xB56}, + {0xB62, 0xB63}, + {0xB82, 0xB82}, + {0xBC0, 0xBC0}, + {0xBCD, 0xBCD}, + {0xC00, 0xC00}, + {0xC3E, 0xC40}, + {0xC46, 0xC48}, + {0xC4A, 0xC4D}, + {0xC55, 0xC56}, + {0xC62, 0xC63}, + {0xC81, 0xC81}, + {0xCBC, 0xCBC}, + {0xCBF, 0xCBF}, + {0xCC6, 0xCC6}, + {0xCCC, 0xCCD}, + {0xCE2, 0xCE3}, + {0xD01, 0xD01}, + {0xD41, 0xD44}, + {0xD4D, 0xD4D}, + {0xD62, 0xD63}, + {0xDCA, 0xDCA}, + {0xDD2, 0xDD4}, + {0xDD6, 0xDD6}, + {0xE31, 0xE31}, + {0xE34, 0xE3A}, + {0xE47, 0xE4E}, + {0xEB1, 0xEB1}, + {0xEB4, 0xEB9}, + {0xEBB, 0xEBC}, + {0xEC8, 0xECD}, + {0xF18, 0xF19}, + {0xF35, 0xF35}, + {0xF37, 0xF37}, + {0xF39, 0xF39}, + {0xF71, 0xF7E}, + {0xF80, 0xF84}, + {0xF86, 0xF87}, + {0xF8D, 0xF97}, + {0xF99, 0xFBC}, + {0xFC6, 0xFC6}, + {0x102D, 0x1030}, + {0x1032, 0x1037}, + {0x1039, 0x103A}, + {0x103D, 0x103E}, + {0x1058, 0x1059}, + {0x105E, 0x1060}, + {0x1071, 0x1074}, + {0x1082, 0x1082}, + {0x1085, 0x1086}, + {0x108D, 0x108D}, + {0x109D, 0x109D}, + {0x135D, 0x135F}, + {0x1712, 0x1714}, + {0x1732, 0x1734}, + {0x1752, 0x1753}, + {0x1772, 0x1773}, + {0x17B4, 0x17B5}, + {0x17B7, 0x17BD}, + {0x17C6, 0x17C6}, + {0x17C9, 0x17D3}, + {0x17DD, 0x17DD}, + {0x180B, 0x180E}, + {0x18A9, 0x18A9}, + {0x1920, 0x1922}, + {0x1927, 0x1928}, + {0x1932, 0x1932}, + {0x1939, 0x193B}, + {0x1A17, 0x1A18}, + {0x1A1B, 0x1A1B}, + {0x1A56, 0x1A56}, + {0x1A58, 0x1A5E}, + {0x1A60, 0x1A60}, + {0x1A62, 0x1A62}, + {0x1A65, 0x1A6C}, + {0x1A73, 0x1A7C}, + {0x1A7F, 0x1A7F}, + {0x1AB0, 0x1ABD}, + {0x1B00, 0x1B03}, + {0x1B34, 0x1B34}, + {0x1B36, 0x1B3A}, + {0x1B3C, 0x1B3C}, + {0x1B42, 0x1B42}, + {0x1B6B, 0x1B73}, + {0x1B80, 0x1B81}, + {0x1BA2, 0x1BA5}, + {0x1BA8, 0x1BA9}, + {0x1BAB, 0x1BAD}, + {0x1BE6, 0x1BE6}, + {0x1BE8, 0x1BE9}, + {0x1BED, 0x1BED}, + {0x1BEF, 0x1BF1}, + {0x1C2C, 0x1C33}, + {0x1C36, 0x1C37}, + {0x1CD0, 0x1CD2}, + {0x1CD4, 0x1CE0}, + {0x1CE2, 0x1CE8}, + {0x1CED, 0x1CED}, + {0x1CF4, 0x1CF4}, + {0x1CF8, 0x1CF9}, + {0x1DC0, 0x1DF5}, + {0x1DFC, 0x1DFF}, + {0x200B, 0x200F}, + {0x202A, 0x202E}, + {0x2060, 0x2064}, + {0x2066, 0x206F}, + {0x20D0, 0x20DC}, + {0x20E1, 0x20E1}, + {0x20E5, 0x20F0}, + {0x2CEF, 0x2CF1}, + {0x2D7F, 0x2D7F}, + {0x2DE0, 0x2DFF}, + {0x302A, 0x302D}, + {0x3099, 0x309A}, + {0xA66F, 0xA66F}, + {0xA674, 0xA67D}, + {0xA69E, 0xA69F}, + {0xA6F0, 0xA6F1}, + {0xA802, 0xA802}, + {0xA806, 0xA806}, + {0xA80B, 0xA80B}, + {0xA825, 0xA826}, + {0xA8C4, 0xA8C4}, + {0xA8E0, 0xA8F1}, + {0xA926, 0xA92D}, + {0xA947, 0xA951}, + {0xA980, 0xA982}, + {0xA9B3, 0xA9B3}, + {0xA9B6, 0xA9B9}, + {0xA9BC, 0xA9BC}, + {0xA9E5, 0xA9E5}, + {0xAA29, 0xAA2E}, + {0xAA31, 0xAA32}, + {0xAA35, 0xAA36}, + {0xAA43, 0xAA43}, + {0xAA4C, 0xAA4C}, + {0xAA7C, 0xAA7C}, + {0xAAB0, 0xAAB0}, + {0xAAB2, 0xAAB4}, + {0xAAB7, 0xAAB8}, + {0xAABE, 0xAABF}, + {0xAAC1, 0xAAC1}, + {0xAAEC, 0xAAED}, + {0xAAF6, 0xAAF6}, + {0xABE5, 0xABE5}, + {0xABE8, 0xABE8}, + {0xABED, 0xABED}, + {0xE000, 0xF8FF}, + {0xFB1E, 0xFB1E}, + {0xFE00, 0xFE0F}, + {0xFE20, 0xFE2F}, + {0xFEFF, 0xFEFF}, + {0xFFF9, 0xFFFB}, + {0xFFFD, 0xFFFD}, + {0x101FD, 0x101FD}, + {0x102E0, 0x102E0}, + {0x10376, 0x1037A}, + {0x10A01, 0x10A03}, + {0x10A05, 0x10A06}, + {0x10A0C, 0x10A0F}, + {0x10A38, 0x10A3A}, + {0x10A3F, 0x10A3F}, + {0x10AE5, 0x10AE6}, + {0x11001, 0x11001}, + {0x11038, 0x11046}, + {0x1107F, 0x11081}, + {0x110B3, 0x110B6}, + {0x110B9, 0x110BA}, + {0x110BD, 0x110BD}, + {0x11100, 0x11102}, + {0x11127, 0x1112B}, + {0x1112D, 0x11134}, + {0x11173, 0x11173}, + {0x11180, 0x11181}, + {0x111B6, 0x111BE}, + {0x111CA, 0x111CC}, + {0x1122F, 0x11231}, + {0x11234, 0x11234}, + {0x11236, 0x11237}, + {0x112DF, 0x112DF}, + {0x112E3, 0x112EA}, + {0x11300, 0x11301}, + {0x1133C, 0x1133C}, + {0x11340, 0x11340}, + {0x11366, 0x1136C}, + {0x11370, 0x11374}, + {0x114B3, 0x114B8}, + {0x114BA, 0x114BA}, + {0x114BF, 0x114C0}, + {0x114C2, 0x114C3}, + {0x115B2, 0x115B5}, + {0x115BC, 0x115BD}, + {0x115BF, 0x115C0}, + {0x115DC, 0x115DD}, + {0x11633, 0x1163A}, + {0x1163D, 0x1163D}, + {0x1163F, 0x11640}, + {0x116AB, 0x116AB}, + {0x116AD, 0x116AD}, + {0x116B0, 0x116B5}, + {0x116B7, 0x116B7}, + {0x1171D, 0x1171F}, + {0x11722, 0x11725}, + {0x11727, 0x1172B}, + {0x16AF0, 0x16AF4}, + {0x16B30, 0x16B36}, + {0x16F8F, 0x16F92}, + {0x1BC9D, 0x1BC9E}, + {0x1BCA0, 0x1BCA3}, + {0x1D167, 0x1D169}, + {0x1D173, 0x1D182}, + {0x1D185, 0x1D18B}, + {0x1D1AA, 0x1D1AD}, + {0x1D242, 0x1D244}, + {0x1DA00, 0x1DA36}, + {0x1DA3B, 0x1DA6C}, + {0x1DA75, 0x1DA75}, + {0x1DA84, 0x1DA84}, + {0x1DA9B, 0x1DA9F}, + {0x1DAA1, 0x1DAAF}, + {0x1E8D0, 0x1E8D6}, + {0xE0001, 0xE0001}, + {0xE0020, 0xE007F}, + {0xE0100, 0xE01EF}, + {0xF0000, 0xFFFFD}, + {0x100000, 0x10FFFD} + ] + + @space [ + {0x9, 0xA}, + {0xD, 0xD}, + {0x20, 0x20}, + {0xA0, 0xA0}, + {0x1680, 0x1680}, + {0x2000, 0x200A}, + {0x2028, 0x2029}, + {0x202F, 0x202F}, + {0x205F, 0x205F}, + {0x3000, 0x3000} + ] + + @cjk [ + {0x3400, 0x4DBF}, + {0x4E00, 0x9FFF}, + {0xFA0E, 0xFA0F}, + {0xFA11, 0xFA11}, + {0xFA13, 0xFA14}, + {0xFA1F, 0xFA1F}, + {0xFA21, 0xFA21}, + {0xFA23, 0xFA24}, + {0xFA27, 0xFA29}, + {0xFA6E, 0xFA6F}, + {0xFADA, 0xFAFF}, + {0x20000, 0x2A6DF}, + {0x2A700, 0x2B81F}, + {0x2B920, 0x2CEAF}, + {0x2FA1E, 0x2FA1F} + ] + + @punct [ + {0x21, 0x2F}, + {0x3A, 0x40}, + {0x5B, 0x60}, + {0x7B, 0x7E}, + {0xA1, 0xA1}, + {0xA7, 0xA7}, + {0xAB, 0xAB}, + {0xB6, 0xB7}, + {0xBB, 0xBB}, + {0xBF, 0xBF}, + {0x37E, 0x37E}, + {0x387, 0x387}, + {0x55A, 0x55F}, + {0x589, 0x58A}, + {0x5BE, 0x5BE}, + {0x5C0, 0x5C0}, + {0x5C3, 0x5C3}, + {0x5C6, 0x5C6}, + {0x5F3, 0x5F4}, + {0x609, 0x60A}, + {0x60C, 0x60D}, + {0x61B, 0x61B}, + {0x61E, 0x61F}, + {0x66A, 0x66D}, + {0x6D4, 0x6D4}, + {0x700, 0x70D}, + {0x7F7, 0x7F9}, + {0x830, 0x83E}, + {0x85E, 0x85E}, + {0x964, 0x965}, + {0x970, 0x970}, + {0xAF0, 0xAF0}, + {0xDF4, 0xDF4}, + {0xE4F, 0xE4F}, + {0xE5A, 0xE5B}, + {0xF04, 0xF12}, + {0xF14, 0xF14}, + {0xF3A, 0xF3D}, + {0xF85, 0xF85}, + {0xFD0, 0xFD4}, + {0xFD9, 0xFDA}, + {0x104A, 0x104F}, + {0x10FB, 0x10FB}, + {0x1360, 0x1368}, + {0x1400, 0x1400}, + {0x166D, 0x166E}, + {0x169B, 0x169C}, + {0x16EB, 0x16ED}, + {0x1735, 0x1736}, + {0x17D4, 0x17D6}, + {0x17D8, 0x17DA}, + {0x1800, 0x180A}, + {0x1944, 0x1945}, + {0x1A1E, 0x1A1F}, + {0x1AA0, 0x1AA6}, + {0x1AA8, 0x1AAD}, + {0x1B5A, 0x1B60}, + {0x1BFC, 0x1BFF}, + {0x1C3B, 0x1C3F}, + {0x1C7E, 0x1C7F}, + {0x1CC0, 0x1CC7}, + {0x1CD3, 0x1CD3}, + {0x2010, 0x2027}, + {0x2030, 0x2043}, + {0x2045, 0x2051}, + {0x2053, 0x205E}, + {0x207D, 0x207E}, + {0x208D, 0x208E}, + {0x2308, 0x230B}, + {0x2329, 0x232A}, + {0x2768, 0x2775}, + {0x27C5, 0x27C6}, + {0x27E6, 0x27EF}, + {0x2983, 0x2998}, + {0x29D8, 0x29DB}, + {0x29FC, 0x29FD}, + {0x2CF9, 0x2CFC}, + {0x2CFE, 0x2CFF}, + {0x2D70, 0x2D70}, + {0x2E00, 0x2E2E}, + {0x2E30, 0x2E42}, + {0x3001, 0x3003}, + {0x3008, 0x3011}, + {0x3014, 0x301F}, + {0x3030, 0x3030}, + {0x303D, 0x303D}, + {0x30A0, 0x30A0}, + {0x30FB, 0x30FB}, + {0xA4FE, 0xA4FF}, + {0xA60D, 0xA60F}, + {0xA673, 0xA673}, + {0xA67E, 0xA67E}, + {0xA6F2, 0xA6F7}, + {0xA874, 0xA877}, + {0xA8CE, 0xA8CF}, + {0xA8F8, 0xA8FA}, + {0xA8FC, 0xA8FC}, + {0xA92E, 0xA92F}, + {0xA95F, 0xA95F}, + {0xA9C1, 0xA9CD}, + {0xA9DE, 0xA9DF}, + {0xAA5C, 0xAA5F}, + {0xAADE, 0xAADF}, + {0xAAF0, 0xAAF1}, + {0xABEB, 0xABEB}, + {0xFD3E, 0xFD3F}, + {0xFE10, 0xFE19}, + {0xFE30, 0xFE52}, + {0xFE54, 0xFE61}, + {0xFE63, 0xFE63}, + {0xFE68, 0xFE68}, + {0xFE6A, 0xFE6B}, + {0xFF01, 0xFF03}, + {0xFF05, 0xFF0A}, + {0xFF0C, 0xFF0F}, + {0xFF1A, 0xFF1B}, + {0xFF1F, 0xFF20}, + {0xFF3B, 0xFF3D}, + {0xFF3F, 0xFF3F}, + {0xFF5B, 0xFF5B}, + {0xFF5D, 0xFF5D}, + {0xFF5F, 0xFF65}, + {0x10100, 0x10102}, + {0x1039F, 0x1039F}, + {0x103D0, 0x103D0}, + {0x1056F, 0x1056F}, + {0x10857, 0x10857}, + {0x1091F, 0x1091F}, + {0x1093F, 0x1093F}, + {0x10A50, 0x10A58}, + {0x10A7F, 0x10A7F}, + {0x10AF0, 0x10AF6}, + {0x10B39, 0x10B3F}, + {0x10B99, 0x10B9C}, + {0x11047, 0x1104D}, + {0x110BB, 0x110BC}, + {0x110BE, 0x110C1}, + {0x11140, 0x11143}, + {0x11174, 0x11175}, + {0x111C5, 0x111C9}, + {0x111CD, 0x111CD}, + {0x111DB, 0x111DB}, + {0x111DD, 0x111DF}, + {0x11238, 0x1123D}, + {0x112A9, 0x112A9}, + {0x114C6, 0x114C6}, + {0x115C1, 0x115D7}, + {0x11641, 0x11643}, + {0x1173C, 0x1173E}, + {0x12470, 0x12474}, + {0x16A6E, 0x16A6F}, + {0x16AF5, 0x16AF5}, + {0x16B37, 0x16B3B}, + {0x16B44, 0x16B44}, + {0x1BC9F, 0x1BC9F}, + {0x1DA87, 0x1DA8B} + ] + + @ws [ + {0x9, 0xD}, + {0x20, 0x20}, + {0x85, 0x85}, + {0xA0, 0xA0}, + {0x1680, 0x1680}, + {0x2000, 0x200A}, + {0x2028, 0x2029}, + {0x202F, 0x202F}, + {0x205F, 0x205F}, + {0x3000, 0x3000} + ] + + @mapping %{ + 0x41 => "\u{61}", + 0x42 => "\u{62}", + 0x43 => "\u{63}", + 0x44 => "\u{64}", + 0x45 => "\u{65}", + 0x46 => "\u{66}", + 0x47 => "\u{67}", + 0x48 => "\u{68}", + 0x49 => "\u{69}", + 0x4A => "\u{6A}", + 0x4B => "\u{6B}", + 0x4C => "\u{6C}", + 0x4D => "\u{6D}", + 0x4E => "\u{6E}", + 0x4F => "\u{6F}", + 0x50 => "\u{70}", + 0x51 => "\u{71}", + 0x52 => "\u{72}", + 0x53 => "\u{73}", + 0x54 => "\u{74}", + 0x55 => "\u{75}", + 0x56 => "\u{76}", + 0x57 => "\u{77}", + 0x58 => "\u{78}", + 0x59 => "\u{79}", + 0x5A => "\u{7A}", + 0xC0 => "\u{61}", + 0xC1 => "\u{61}", + 0xC2 => "\u{61}", + 0xC3 => "\u{61}", + 0xC4 => "\u{61}", + 0xC5 => "\u{61}", + 0xC6 => "\u{E6}", + 0xC7 => "\u{63}", + 0xC8 => "\u{65}", + 0xC9 => "\u{65}", + 0xCA => "\u{65}", + 0xCB => "\u{65}", + 0xCC => "\u{69}", + 0xCD => "\u{69}", + 0xCE => "\u{69}", + 0xCF => "\u{69}", + 0xD0 => "\u{F0}", + 0xD1 => "\u{6E}", + 0xD2 => "\u{6F}", + 0xD3 => "\u{6F}", + 0xD4 => "\u{6F}", + 0xD5 => "\u{6F}", + 0xD6 => "\u{6F}", + 0xD8 => "\u{F8}", + 0xD9 => "\u{75}", + 0xDA => "\u{75}", + 0xDB => "\u{75}", + 0xDC => "\u{75}", + 0xDD => "\u{79}", + 0xDE => "\u{FE}", + 0xE0 => "\u{61}", + 0xE1 => "\u{61}", + 0xE2 => "\u{61}", + 0xE3 => "\u{61}", + 0xE4 => "\u{61}", + 0xE5 => "\u{61}", + 0xE7 => "\u{63}", + 0xE8 => "\u{65}", + 0xE9 => "\u{65}", + 0xEA => "\u{65}", + 0xEB => "\u{65}", + 0xEC => "\u{69}", + 0xED => "\u{69}", + 0xEE => "\u{69}", + 0xEF => "\u{69}", + 0xF1 => "\u{6E}", + 0xF2 => "\u{6F}", + 0xF3 => "\u{6F}", + 0xF4 => "\u{6F}", + 0xF5 => "\u{6F}", + 0xF6 => "\u{6F}", + 0xF9 => "\u{75}", + 0xFA => "\u{75}", + 0xFB => "\u{75}", + 0xFC => "\u{75}", + 0xFD => "\u{79}", + 0xFF => "\u{79}", + 0x100 => "\u{61}", + 0x101 => "\u{61}", + 0x102 => "\u{61}", + 0x103 => "\u{61}", + 0x104 => "\u{61}", + 0x105 => "\u{61}", + 0x106 => "\u{63}", + 0x107 => "\u{63}", + 0x108 => "\u{63}", + 0x109 => "\u{63}", + 0x10A => "\u{63}", + 0x10B => "\u{63}", + 0x10C => "\u{63}", + 0x10D => "\u{63}", + 0x10E => "\u{64}", + 0x10F => "\u{64}", + 0x110 => "\u{111}", + 0x112 => "\u{65}", + 0x113 => "\u{65}", + 0x114 => "\u{65}", + 0x115 => "\u{65}", + 0x116 => "\u{65}", + 0x117 => "\u{65}", + 0x118 => "\u{65}", + 0x119 => "\u{65}", + 0x11A => "\u{65}", + 0x11B => "\u{65}", + 0x11C => "\u{67}", + 0x11D => "\u{67}", + 0x11E => "\u{67}", + 0x11F => "\u{67}", + 0x120 => "\u{67}", + 0x121 => "\u{67}", + 0x122 => "\u{67}", + 0x123 => "\u{67}", + 0x124 => "\u{68}", + 0x125 => "\u{68}", + 0x126 => "\u{127}", + 0x128 => "\u{69}", + 0x129 => "\u{69}", + 0x12A => "\u{69}", + 0x12B => "\u{69}", + 0x12C => "\u{69}", + 0x12D => "\u{69}", + 0x12E => "\u{69}", + 0x12F => "\u{69}", + 0x130 => "\u{69}", + 0x132 => "\u{133}", + 0x134 => "\u{6A}", + 0x135 => "\u{6A}", + 0x136 => "\u{6B}", + 0x137 => "\u{6B}", + 0x139 => "\u{6C}", + 0x13A => "\u{6C}", + 0x13B => "\u{6C}", + 0x13C => "\u{6C}", + 0x13D => "\u{6C}", + 0x13E => "\u{6C}", + 0x13F => "\u{140}", + 0x141 => "\u{142}", + 0x143 => "\u{6E}", + 0x144 => "\u{6E}", + 0x145 => "\u{6E}", + 0x146 => "\u{6E}", + 0x147 => "\u{6E}", + 0x148 => "\u{6E}", + 0x14A => "\u{14B}", + 0x14C => "\u{6F}", + 0x14D => "\u{6F}", + 0x14E => "\u{6F}", + 0x14F => "\u{6F}", + 0x150 => "\u{6F}", + 0x151 => "\u{6F}", + 0x152 => "\u{153}", + 0x154 => "\u{72}", + 0x155 => "\u{72}", + 0x156 => "\u{72}", + 0x157 => "\u{72}", + 0x158 => "\u{72}", + 0x159 => "\u{72}", + 0x15A => "\u{73}", + 0x15B => "\u{73}", + 0x15C => "\u{73}", + 0x15D => "\u{73}", + 0x15E => "\u{73}", + 0x15F => "\u{73}", + 0x160 => "\u{73}", + 0x161 => "\u{73}", + 0x162 => "\u{74}", + 0x163 => "\u{74}", + 0x164 => "\u{74}", + 0x165 => "\u{74}", + 0x166 => "\u{167}", + 0x168 => "\u{75}", + 0x169 => "\u{75}", + 0x16A => "\u{75}", + 0x16B => "\u{75}", + 0x16C => "\u{75}", + 0x16D => "\u{75}", + 0x16E => "\u{75}", + 0x16F => "\u{75}", + 0x170 => "\u{75}", + 0x171 => "\u{75}", + 0x172 => "\u{75}", + 0x173 => "\u{75}", + 0x174 => "\u{77}", + 0x175 => "\u{77}", + 0x176 => "\u{79}", + 0x177 => "\u{79}", + 0x178 => "\u{79}", + 0x179 => "\u{7A}", + 0x17A => "\u{7A}", + 0x17B => "\u{7A}", + 0x17C => "\u{7A}", + 0x17D => "\u{7A}", + 0x17E => "\u{7A}", + 0x181 => "\u{253}", + 0x182 => "\u{183}", + 0x184 => "\u{185}", + 0x186 => "\u{254}", + 0x187 => "\u{188}", + 0x189 => "\u{256}", + 0x18A => "\u{257}", + 0x18B => "\u{18C}", + 0x18E => "\u{1DD}", + 0x18F => "\u{259}", + 0x190 => "\u{25B}", + 0x191 => "\u{192}", + 0x193 => "\u{260}", + 0x194 => "\u{263}", + 0x196 => "\u{269}", + 0x197 => "\u{268}", + 0x198 => "\u{199}", + 0x19C => "\u{26F}", + 0x19D => "\u{272}", + 0x19F => "\u{275}", + 0x1A0 => "\u{6F}", + 0x1A1 => "\u{6F}", + 0x1A2 => "\u{1A3}", + 0x1A4 => "\u{1A5}", + 0x1A6 => "\u{280}", + 0x1A7 => "\u{1A8}", + 0x1A9 => "\u{283}", + 0x1AC => "\u{1AD}", + 0x1AE => "\u{288}", + 0x1AF => "\u{75}", + 0x1B0 => "\u{75}", + 0x1B1 => "\u{28A}", + 0x1B2 => "\u{28B}", + 0x1B3 => "\u{1B4}", + 0x1B5 => "\u{1B6}", + 0x1B7 => "\u{292}", + 0x1B8 => "\u{1B9}", + 0x1BC => "\u{1BD}", + 0x1C4 => "\u{1C6}", + 0x1C5 => "\u{1C6}", + 0x1C7 => "\u{1C9}", + 0x1C8 => "\u{1C9}", + 0x1CA => "\u{1CC}", + 0x1CB => "\u{1CC}", + 0x1CD => "\u{61}", + 0x1CE => "\u{61}", + 0x1CF => "\u{69}", + 0x1D0 => "\u{69}", + 0x1D1 => "\u{6F}", + 0x1D2 => "\u{6F}", + 0x1D3 => "\u{75}", + 0x1D4 => "\u{75}", + 0x1D5 => "\u{75}", + 0x1D6 => "\u{75}", + 0x1D7 => "\u{75}", + 0x1D8 => "\u{75}", + 0x1D9 => "\u{75}", + 0x1DA => "\u{75}", + 0x1DB => "\u{75}", + 0x1DC => "\u{75}", + 0x1DE => "\u{61}", + 0x1DF => "\u{61}", + 0x1E0 => "\u{61}", + 0x1E1 => "\u{61}", + 0x1E2 => "\u{E6}", + 0x1E3 => "\u{E6}", + 0x1E4 => "\u{1E5}", + 0x1E6 => "\u{67}", + 0x1E7 => "\u{67}", + 0x1E8 => "\u{6B}", + 0x1E9 => "\u{6B}", + 0x1EA => "\u{6F}", + 0x1EB => "\u{6F}", + 0x1EC => "\u{6F}", + 0x1ED => "\u{6F}", + 0x1EE => "\u{292}", + 0x1EF => "\u{292}", + 0x1F0 => "\u{6A}", + 0x1F1 => "\u{1F3}", + 0x1F2 => "\u{1F3}", + 0x1F4 => "\u{67}", + 0x1F5 => "\u{67}", + 0x1F6 => "\u{195}", + 0x1F7 => "\u{1BF}", + 0x1F8 => "\u{6E}", + 0x1F9 => "\u{6E}", + 0x1FA => "\u{61}", + 0x1FB => "\u{61}", + 0x1FC => "\u{E6}", + 0x1FD => "\u{E6}", + 0x1FE => "\u{F8}", + 0x1FF => "\u{F8}", + 0x200 => "\u{61}", + 0x201 => "\u{61}", + 0x202 => "\u{61}", + 0x203 => "\u{61}", + 0x204 => "\u{65}", + 0x205 => "\u{65}", + 0x206 => "\u{65}", + 0x207 => "\u{65}", + 0x208 => "\u{69}", + 0x209 => "\u{69}", + 0x20A => "\u{69}", + 0x20B => "\u{69}", + 0x20C => "\u{6F}", + 0x20D => "\u{6F}", + 0x20E => "\u{6F}", + 0x20F => "\u{6F}", + 0x210 => "\u{72}", + 0x211 => "\u{72}", + 0x212 => "\u{72}", + 0x213 => "\u{72}", + 0x214 => "\u{75}", + 0x215 => "\u{75}", + 0x216 => "\u{75}", + 0x217 => "\u{75}", + 0x218 => "\u{73}", + 0x219 => "\u{73}", + 0x21A => "\u{74}", + 0x21B => "\u{74}", + 0x21C => "\u{21D}", + 0x21E => "\u{68}", + 0x21F => "\u{68}", + 0x220 => "\u{19E}", + 0x222 => "\u{223}", + 0x224 => "\u{225}", + 0x226 => "\u{61}", + 0x227 => "\u{61}", + 0x228 => "\u{65}", + 0x229 => "\u{65}", + 0x22A => "\u{6F}", + 0x22B => "\u{6F}", + 0x22C => "\u{6F}", + 0x22D => "\u{6F}", + 0x22E => "\u{6F}", + 0x22F => "\u{6F}", + 0x230 => "\u{6F}", + 0x231 => "\u{6F}", + 0x232 => "\u{79}", + 0x233 => "\u{79}", + 0x23A => "\u{2C65}", + 0x23B => "\u{23C}", + 0x23D => "\u{19A}", + 0x23E => "\u{2C66}", + 0x241 => "\u{242}", + 0x243 => "\u{180}", + 0x244 => "\u{289}", + 0x245 => "\u{28C}", + 0x246 => "\u{247}", + 0x248 => "\u{249}", + 0x24A => "\u{24B}", + 0x24C => "\u{24D}", + 0x24E => "\u{24F}", + 0x370 => "\u{371}", + 0x372 => "\u{373}", + 0x374 => "\u{2B9}", + 0x376 => "\u{377}", + 0x37E => "\u{3B}", + 0x37F => "\u{3F3}", + 0x385 => "\u{A8}", + 0x386 => "\u{3B1}", + 0x387 => "\u{B7}", + 0x388 => "\u{3B5}", + 0x389 => "\u{3B7}", + 0x38A => "\u{3B9}", + 0x38C => "\u{3BF}", + 0x38E => "\u{3C5}", + 0x38F => "\u{3C9}", + 0x390 => "\u{3B9}", + 0x391 => "\u{3B1}", + 0x392 => "\u{3B2}", + 0x393 => "\u{3B3}", + 0x394 => "\u{3B4}", + 0x395 => "\u{3B5}", + 0x396 => "\u{3B6}", + 0x397 => "\u{3B7}", + 0x398 => "\u{3B8}", + 0x399 => "\u{3B9}", + 0x39A => "\u{3BA}", + 0x39B => "\u{3BB}", + 0x39C => "\u{3BC}", + 0x39D => "\u{3BD}", + 0x39E => "\u{3BE}", + 0x39F => "\u{3BF}", + 0x3A0 => "\u{3C0}", + 0x3A1 => "\u{3C1}", + 0x3A3 => "\u{3C3}", + 0x3A4 => "\u{3C4}", + 0x3A5 => "\u{3C5}", + 0x3A6 => "\u{3C6}", + 0x3A7 => "\u{3C7}", + 0x3A8 => "\u{3C8}", + 0x3A9 => "\u{3C9}", + 0x3AA => "\u{3B9}", + 0x3AB => "\u{3C5}", + 0x3AC => "\u{3B1}", + 0x3AD => "\u{3B5}", + 0x3AE => "\u{3B7}", + 0x3AF => "\u{3B9}", + 0x3B0 => "\u{3C5}", + 0x3CA => "\u{3B9}", + 0x3CB => "\u{3C5}", + 0x3CC => "\u{3BF}", + 0x3CD => "\u{3C5}", + 0x3CE => "\u{3C9}", + 0x3CF => "\u{3D7}", + 0x3D3 => "\u{3D2}", + 0x3D4 => "\u{3D2}", + 0x3D8 => "\u{3D9}", + 0x3DA => "\u{3DB}", + 0x3DC => "\u{3DD}", + 0x3DE => "\u{3DF}", + 0x3E0 => "\u{3E1}", + 0x3E2 => "\u{3E3}", + 0x3E4 => "\u{3E5}", + 0x3E6 => "\u{3E7}", + 0x3E8 => "\u{3E9}", + 0x3EA => "\u{3EB}", + 0x3EC => "\u{3ED}", + 0x3EE => "\u{3EF}", + 0x3F4 => "\u{3B8}", + 0x3F7 => "\u{3F8}", + 0x3F9 => "\u{3F2}", + 0x3FA => "\u{3FB}", + 0x3FD => "\u{37B}", + 0x3FE => "\u{37C}", + 0x3FF => "\u{37D}", + 0x400 => "\u{435}", + 0x401 => "\u{435}", + 0x402 => "\u{452}", + 0x403 => "\u{433}", + 0x404 => "\u{454}", + 0x405 => "\u{455}", + 0x406 => "\u{456}", + 0x407 => "\u{456}", + 0x408 => "\u{458}", + 0x409 => "\u{459}", + 0x40A => "\u{45A}", + 0x40B => "\u{45B}", + 0x40C => "\u{43A}", + 0x40D => "\u{438}", + 0x40E => "\u{443}", + 0x40F => "\u{45F}", + 0x410 => "\u{430}", + 0x411 => "\u{431}", + 0x412 => "\u{432}", + 0x413 => "\u{433}", + 0x414 => "\u{434}", + 0x415 => "\u{435}", + 0x416 => "\u{436}", + 0x417 => "\u{437}", + 0x418 => "\u{438}", + 0x419 => "\u{438}", + 0x41A => "\u{43A}", + 0x41B => "\u{43B}", + 0x41C => "\u{43C}", + 0x41D => "\u{43D}", + 0x41E => "\u{43E}", + 0x41F => "\u{43F}", + 0x420 => "\u{440}", + 0x421 => "\u{441}", + 0x422 => "\u{442}", + 0x423 => "\u{443}", + 0x424 => "\u{444}", + 0x425 => "\u{445}", + 0x426 => "\u{446}", + 0x427 => "\u{447}", + 0x428 => "\u{448}", + 0x429 => "\u{449}", + 0x42A => "\u{44A}", + 0x42B => "\u{44B}", + 0x42C => "\u{44C}", + 0x42D => "\u{44D}", + 0x42E => "\u{44E}", + 0x42F => "\u{44F}", + 0x439 => "\u{438}", + 0x450 => "\u{435}", + 0x451 => "\u{435}", + 0x453 => "\u{433}", + 0x457 => "\u{456}", + 0x45C => "\u{43A}", + 0x45D => "\u{438}", + 0x45E => "\u{443}", + 0x460 => "\u{461}", + 0x462 => "\u{463}", + 0x464 => "\u{465}", + 0x466 => "\u{467}", + 0x468 => "\u{469}", + 0x46A => "\u{46B}", + 0x46C => "\u{46D}", + 0x46E => "\u{46F}", + 0x470 => "\u{471}", + 0x472 => "\u{473}", + 0x474 => "\u{475}", + 0x476 => "\u{475}", + 0x477 => "\u{475}", + 0x478 => "\u{479}", + 0x47A => "\u{47B}", + 0x47C => "\u{47D}", + 0x47E => "\u{47F}", + 0x480 => "\u{481}", + 0x48A => "\u{48B}", + 0x48C => "\u{48D}", + 0x48E => "\u{48F}", + 0x490 => "\u{491}", + 0x492 => "\u{493}", + 0x494 => "\u{495}", + 0x496 => "\u{497}", + 0x498 => "\u{499}", + 0x49A => "\u{49B}", + 0x49C => "\u{49D}", + 0x49E => "\u{49F}", + 0x4A0 => "\u{4A1}", + 0x4A2 => "\u{4A3}", + 0x4A4 => "\u{4A5}", + 0x4A6 => "\u{4A7}", + 0x4A8 => "\u{4A9}", + 0x4AA => "\u{4AB}", + 0x4AC => "\u{4AD}", + 0x4AE => "\u{4AF}", + 0x4B0 => "\u{4B1}", + 0x4B2 => "\u{4B3}", + 0x4B4 => "\u{4B5}", + 0x4B6 => "\u{4B7}", + 0x4B8 => "\u{4B9}", + 0x4BA => "\u{4BB}", + 0x4BC => "\u{4BD}", + 0x4BE => "\u{4BF}", + 0x4C0 => "\u{4CF}", + 0x4C1 => "\u{436}", + 0x4C2 => "\u{436}", + 0x4C3 => "\u{4C4}", + 0x4C5 => "\u{4C6}", + 0x4C7 => "\u{4C8}", + 0x4C9 => "\u{4CA}", + 0x4CB => "\u{4CC}", + 0x4CD => "\u{4CE}", + 0x4D0 => "\u{430}", + 0x4D1 => "\u{430}", + 0x4D2 => "\u{430}", + 0x4D3 => "\u{430}", + 0x4D4 => "\u{4D5}", + 0x4D6 => "\u{435}", + 0x4D7 => "\u{435}", + 0x4D8 => "\u{4D9}", + 0x4DA => "\u{4D9}", + 0x4DB => "\u{4D9}", + 0x4DC => "\u{436}", + 0x4DD => "\u{436}", + 0x4DE => "\u{437}", + 0x4DF => "\u{437}", + 0x4E0 => "\u{4E1}", + 0x4E2 => "\u{438}", + 0x4E3 => "\u{438}", + 0x4E4 => "\u{438}", + 0x4E5 => "\u{438}", + 0x4E6 => "\u{43E}", + 0x4E7 => "\u{43E}", + 0x4E8 => "\u{4E9}", + 0x4EA => "\u{4E9}", + 0x4EB => "\u{4E9}", + 0x4EC => "\u{44D}", + 0x4ED => "\u{44D}", + 0x4EE => "\u{443}", + 0x4EF => "\u{443}", + 0x4F0 => "\u{443}", + 0x4F1 => "\u{443}", + 0x4F2 => "\u{443}", + 0x4F3 => "\u{443}", + 0x4F4 => "\u{447}", + 0x4F5 => "\u{447}", + 0x4F6 => "\u{4F7}", + 0x4F8 => "\u{44B}", + 0x4F9 => "\u{44B}", + 0x4FA => "\u{4FB}", + 0x4FC => "\u{4FD}", + 0x4FE => "\u{4FF}", + 0x500 => "\u{501}", + 0x502 => "\u{503}", + 0x504 => "\u{505}", + 0x506 => "\u{507}", + 0x508 => "\u{509}", + 0x50A => "\u{50B}", + 0x50C => "\u{50D}", + 0x50E => "\u{50F}", + 0x510 => "\u{511}", + 0x512 => "\u{513}", + 0x514 => "\u{515}", + 0x516 => "\u{517}", + 0x518 => "\u{519}", + 0x51A => "\u{51B}", + 0x51C => "\u{51D}", + 0x51E => "\u{51F}", + 0x520 => "\u{521}", + 0x522 => "\u{523}", + 0x524 => "\u{525}", + 0x526 => "\u{527}", + 0x528 => "\u{529}", + 0x52A => "\u{52B}", + 0x52C => "\u{52D}", + 0x52E => "\u{52F}", + 0x531 => "\u{561}", + 0x532 => "\u{562}", + 0x533 => "\u{563}", + 0x534 => "\u{564}", + 0x535 => "\u{565}", + 0x536 => "\u{566}", + 0x537 => "\u{567}", + 0x538 => "\u{568}", + 0x539 => "\u{569}", + 0x53A => "\u{56A}", + 0x53B => "\u{56B}", + 0x53C => "\u{56C}", + 0x53D => "\u{56D}", + 0x53E => "\u{56E}", + 0x53F => "\u{56F}", + 0x540 => "\u{570}", + 0x541 => "\u{571}", + 0x542 => "\u{572}", + 0x543 => "\u{573}", + 0x544 => "\u{574}", + 0x545 => "\u{575}", + 0x546 => "\u{576}", + 0x547 => "\u{577}", + 0x548 => "\u{578}", + 0x549 => "\u{579}", + 0x54A => "\u{57A}", + 0x54B => "\u{57B}", + 0x54C => "\u{57C}", + 0x54D => "\u{57D}", + 0x54E => "\u{57E}", + 0x54F => "\u{57F}", + 0x550 => "\u{580}", + 0x551 => "\u{581}", + 0x552 => "\u{582}", + 0x553 => "\u{583}", + 0x554 => "\u{584}", + 0x555 => "\u{585}", + 0x556 => "\u{586}", + 0x622 => "\u{627}", + 0x623 => "\u{627}", + 0x624 => "\u{648}", + 0x625 => "\u{627}", + 0x626 => "\u{64A}", + 0x6C0 => "\u{6D5}", + 0x6C2 => "\u{6C1}", + 0x6D3 => "\u{6D2}", + 0x929 => "\u{928}", + 0x931 => "\u{930}", + 0x934 => "\u{933}", + 0x958 => "\u{915}", + 0x959 => "\u{916}", + 0x95A => "\u{917}", + 0x95B => "\u{91C}", + 0x95C => "\u{921}", + 0x95D => "\u{922}", + 0x95E => "\u{92B}", + 0x95F => "\u{92F}", + 0x9CB => "\u{9C7}\u{9BE}", + 0x9CC => "\u{9C7}\u{9D7}", + 0x9DC => "\u{9A1}", + 0x9DD => "\u{9A2}", + 0x9DF => "\u{9AF}", + 0xA33 => "\u{A32}", + 0xA36 => "\u{A38}", + 0xA59 => "\u{A16}", + 0xA5A => "\u{A17}", + 0xA5B => "\u{A1C}", + 0xA5E => "\u{A2B}", + 0xB48 => "\u{B47}", + 0xB4B => "\u{B47}\u{B3E}", + 0xB4C => "\u{B47}\u{B57}", + 0xB5C => "\u{B21}", + 0xB5D => "\u{B22}", + 0xB94 => "\u{B92}\u{BD7}", + 0xBCA => "\u{BC6}\u{BBE}", + 0xBCB => "\u{BC7}\u{BBE}", + 0xBCC => "\u{BC6}\u{BD7}", + 0xCC0 => "\u{CD5}", + 0xCC7 => "\u{CD5}", + 0xCC8 => "\u{CD6}", + 0xCCA => "\u{CC2}", + 0xCCB => "\u{CC2}\u{CD5}", + 0xD4A => "\u{D46}\u{D3E}", + 0xD4B => "\u{D47}\u{D3E}", + 0xD4C => "\u{D46}\u{D57}", + 0xDDA => "\u{DD9}", + 0xDDC => "\u{DD9}\u{DCF}", + 0xDDD => "\u{DD9}\u{DCF}", + 0xDDE => "\u{DD9}\u{DDF}", + 0xF43 => "\u{F42}", + 0xF4D => "\u{F4C}", + 0xF52 => "\u{F51}", + 0xF57 => "\u{F56}", + 0xF5C => "\u{F5B}", + 0xF69 => "\u{F40}", + 0x1026 => "\u{1025}", + 0x10A0 => "\u{2D00}", + 0x10A1 => "\u{2D01}", + 0x10A2 => "\u{2D02}", + 0x10A3 => "\u{2D03}", + 0x10A4 => "\u{2D04}", + 0x10A5 => "\u{2D05}", + 0x10A6 => "\u{2D06}", + 0x10A7 => "\u{2D07}", + 0x10A8 => "\u{2D08}", + 0x10A9 => "\u{2D09}", + 0x10AA => "\u{2D0A}", + 0x10AB => "\u{2D0B}", + 0x10AC => "\u{2D0C}", + 0x10AD => "\u{2D0D}", + 0x10AE => "\u{2D0E}", + 0x10AF => "\u{2D0F}", + 0x10B0 => "\u{2D10}", + 0x10B1 => "\u{2D11}", + 0x10B2 => "\u{2D12}", + 0x10B3 => "\u{2D13}", + 0x10B4 => "\u{2D14}", + 0x10B5 => "\u{2D15}", + 0x10B6 => "\u{2D16}", + 0x10B7 => "\u{2D17}", + 0x10B8 => "\u{2D18}", + 0x10B9 => "\u{2D19}", + 0x10BA => "\u{2D1A}", + 0x10BB => "\u{2D1B}", + 0x10BC => "\u{2D1C}", + 0x10BD => "\u{2D1D}", + 0x10BE => "\u{2D1E}", + 0x10BF => "\u{2D1F}", + 0x10C0 => "\u{2D20}", + 0x10C1 => "\u{2D21}", + 0x10C2 => "\u{2D22}", + 0x10C3 => "\u{2D23}", + 0x10C4 => "\u{2D24}", + 0x10C5 => "\u{2D25}", + 0x10C7 => "\u{2D27}", + 0x10CD => "\u{2D2D}", + 0x13A0 => "\u{AB70}", + 0x13A1 => "\u{AB71}", + 0x13A2 => "\u{AB72}", + 0x13A3 => "\u{AB73}", + 0x13A4 => "\u{AB74}", + 0x13A5 => "\u{AB75}", + 0x13A6 => "\u{AB76}", + 0x13A7 => "\u{AB77}", + 0x13A8 => "\u{AB78}", + 0x13A9 => "\u{AB79}", + 0x13AA => "\u{AB7A}", + 0x13AB => "\u{AB7B}", + 0x13AC => "\u{AB7C}", + 0x13AD => "\u{AB7D}", + 0x13AE => "\u{AB7E}", + 0x13AF => "\u{AB7F}", + 0x13B0 => "\u{AB80}", + 0x13B1 => "\u{AB81}", + 0x13B2 => "\u{AB82}", + 0x13B3 => "\u{AB83}", + 0x13B4 => "\u{AB84}", + 0x13B5 => "\u{AB85}", + 0x13B6 => "\u{AB86}", + 0x13B7 => "\u{AB87}", + 0x13B8 => "\u{AB88}", + 0x13B9 => "\u{AB89}", + 0x13BA => "\u{AB8A}", + 0x13BB => "\u{AB8B}", + 0x13BC => "\u{AB8C}", + 0x13BD => "\u{AB8D}", + 0x13BE => "\u{AB8E}", + 0x13BF => "\u{AB8F}", + 0x13C0 => "\u{AB90}", + 0x13C1 => "\u{AB91}", + 0x13C2 => "\u{AB92}", + 0x13C3 => "\u{AB93}", + 0x13C4 => "\u{AB94}", + 0x13C5 => "\u{AB95}", + 0x13C6 => "\u{AB96}", + 0x13C7 => "\u{AB97}", + 0x13C8 => "\u{AB98}", + 0x13C9 => "\u{AB99}", + 0x13CA => "\u{AB9A}", + 0x13CB => "\u{AB9B}", + 0x13CC => "\u{AB9C}", + 0x13CD => "\u{AB9D}", + 0x13CE => "\u{AB9E}", + 0x13CF => "\u{AB9F}", + 0x13D0 => "\u{ABA0}", + 0x13D1 => "\u{ABA1}", + 0x13D2 => "\u{ABA2}", + 0x13D3 => "\u{ABA3}", + 0x13D4 => "\u{ABA4}", + 0x13D5 => "\u{ABA5}", + 0x13D6 => "\u{ABA6}", + 0x13D7 => "\u{ABA7}", + 0x13D8 => "\u{ABA8}", + 0x13D9 => "\u{ABA9}", + 0x13DA => "\u{ABAA}", + 0x13DB => "\u{ABAB}", + 0x13DC => "\u{ABAC}", + 0x13DD => "\u{ABAD}", + 0x13DE => "\u{ABAE}", + 0x13DF => "\u{ABAF}", + 0x13E0 => "\u{ABB0}", + 0x13E1 => "\u{ABB1}", + 0x13E2 => "\u{ABB2}", + 0x13E3 => "\u{ABB3}", + 0x13E4 => "\u{ABB4}", + 0x13E5 => "\u{ABB5}", + 0x13E6 => "\u{ABB6}", + 0x13E7 => "\u{ABB7}", + 0x13E8 => "\u{ABB8}", + 0x13E9 => "\u{ABB9}", + 0x13EA => "\u{ABBA}", + 0x13EB => "\u{ABBB}", + 0x13EC => "\u{ABBC}", + 0x13ED => "\u{ABBD}", + 0x13EE => "\u{ABBE}", + 0x13EF => "\u{ABBF}", + 0x13F0 => "\u{13F8}", + 0x13F1 => "\u{13F9}", + 0x13F2 => "\u{13FA}", + 0x13F3 => "\u{13FB}", + 0x13F4 => "\u{13FC}", + 0x13F5 => "\u{13FD}", + 0x1B06 => "\u{1B05}\u{1B35}", + 0x1B08 => "\u{1B07}\u{1B35}", + 0x1B0A => "\u{1B09}\u{1B35}", + 0x1B0C => "\u{1B0B}\u{1B35}", + 0x1B0E => "\u{1B0D}\u{1B35}", + 0x1B12 => "\u{1B11}\u{1B35}", + 0x1B3B => "\u{1B35}", + 0x1B3D => "\u{1B35}", + 0x1B40 => "\u{1B3E}\u{1B35}", + 0x1B41 => "\u{1B3F}\u{1B35}", + 0x1B43 => "\u{1B35}", + 0x1C89 => "\u{1C8A}", + 0x1C90 => "\u{10D0}", + 0x1C91 => "\u{10D1}", + 0x1C92 => "\u{10D2}", + 0x1C93 => "\u{10D3}", + 0x1C94 => "\u{10D4}", + 0x1C95 => "\u{10D5}", + 0x1C96 => "\u{10D6}", + 0x1C97 => "\u{10D7}", + 0x1C98 => "\u{10D8}", + 0x1C99 => "\u{10D9}", + 0x1C9A => "\u{10DA}", + 0x1C9B => "\u{10DB}", + 0x1C9C => "\u{10DC}", + 0x1C9D => "\u{10DD}", + 0x1C9E => "\u{10DE}", + 0x1C9F => "\u{10DF}", + 0x1CA0 => "\u{10E0}", + 0x1CA1 => "\u{10E1}", + 0x1CA2 => "\u{10E2}", + 0x1CA3 => "\u{10E3}", + 0x1CA4 => "\u{10E4}", + 0x1CA5 => "\u{10E5}", + 0x1CA6 => "\u{10E6}", + 0x1CA7 => "\u{10E7}", + 0x1CA8 => "\u{10E8}", + 0x1CA9 => "\u{10E9}", + 0x1CAA => "\u{10EA}", + 0x1CAB => "\u{10EB}", + 0x1CAC => "\u{10EC}", + 0x1CAD => "\u{10ED}", + 0x1CAE => "\u{10EE}", + 0x1CAF => "\u{10EF}", + 0x1CB0 => "\u{10F0}", + 0x1CB1 => "\u{10F1}", + 0x1CB2 => "\u{10F2}", + 0x1CB3 => "\u{10F3}", + 0x1CB4 => "\u{10F4}", + 0x1CB5 => "\u{10F5}", + 0x1CB6 => "\u{10F6}", + 0x1CB7 => "\u{10F7}", + 0x1CB8 => "\u{10F8}", + 0x1CB9 => "\u{10F9}", + 0x1CBA => "\u{10FA}", + 0x1CBD => "\u{10FD}", + 0x1CBE => "\u{10FE}", + 0x1CBF => "\u{10FF}", + 0x1E00 => "\u{61}", + 0x1E01 => "\u{61}", + 0x1E02 => "\u{62}", + 0x1E03 => "\u{62}", + 0x1E04 => "\u{62}", + 0x1E05 => "\u{62}", + 0x1E06 => "\u{62}", + 0x1E07 => "\u{62}", + 0x1E08 => "\u{63}", + 0x1E09 => "\u{63}", + 0x1E0A => "\u{64}", + 0x1E0B => "\u{64}", + 0x1E0C => "\u{64}", + 0x1E0D => "\u{64}", + 0x1E0E => "\u{64}", + 0x1E0F => "\u{64}", + 0x1E10 => "\u{64}", + 0x1E11 => "\u{64}", + 0x1E12 => "\u{64}", + 0x1E13 => "\u{64}", + 0x1E14 => "\u{65}", + 0x1E15 => "\u{65}", + 0x1E16 => "\u{65}", + 0x1E17 => "\u{65}", + 0x1E18 => "\u{65}", + 0x1E19 => "\u{65}", + 0x1E1A => "\u{65}", + 0x1E1B => "\u{65}", + 0x1E1C => "\u{65}", + 0x1E1D => "\u{65}", + 0x1E1E => "\u{66}", + 0x1E1F => "\u{66}", + 0x1E20 => "\u{67}", + 0x1E21 => "\u{67}", + 0x1E22 => "\u{68}", + 0x1E23 => "\u{68}", + 0x1E24 => "\u{68}", + 0x1E25 => "\u{68}", + 0x1E26 => "\u{68}", + 0x1E27 => "\u{68}", + 0x1E28 => "\u{68}", + 0x1E29 => "\u{68}", + 0x1E2A => "\u{68}", + 0x1E2B => "\u{68}", + 0x1E2C => "\u{69}", + 0x1E2D => "\u{69}", + 0x1E2E => "\u{69}", + 0x1E2F => "\u{69}", + 0x1E30 => "\u{6B}", + 0x1E31 => "\u{6B}", + 0x1E32 => "\u{6B}", + 0x1E33 => "\u{6B}", + 0x1E34 => "\u{6B}", + 0x1E35 => "\u{6B}", + 0x1E36 => "\u{6C}", + 0x1E37 => "\u{6C}", + 0x1E38 => "\u{6C}", + 0x1E39 => "\u{6C}", + 0x1E3A => "\u{6C}", + 0x1E3B => "\u{6C}", + 0x1E3C => "\u{6C}", + 0x1E3D => "\u{6C}", + 0x1E3E => "\u{6D}", + 0x1E3F => "\u{6D}", + 0x1E40 => "\u{6D}", + 0x1E41 => "\u{6D}", + 0x1E42 => "\u{6D}", + 0x1E43 => "\u{6D}", + 0x1E44 => "\u{6E}", + 0x1E45 => "\u{6E}", + 0x1E46 => "\u{6E}", + 0x1E47 => "\u{6E}", + 0x1E48 => "\u{6E}", + 0x1E49 => "\u{6E}", + 0x1E4A => "\u{6E}", + 0x1E4B => "\u{6E}", + 0x1E4C => "\u{6F}", + 0x1E4D => "\u{6F}", + 0x1E4E => "\u{6F}", + 0x1E4F => "\u{6F}", + 0x1E50 => "\u{6F}", + 0x1E51 => "\u{6F}", + 0x1E52 => "\u{6F}", + 0x1E53 => "\u{6F}", + 0x1E54 => "\u{70}", + 0x1E55 => "\u{70}", + 0x1E56 => "\u{70}", + 0x1E57 => "\u{70}", + 0x1E58 => "\u{72}", + 0x1E59 => "\u{72}", + 0x1E5A => "\u{72}", + 0x1E5B => "\u{72}", + 0x1E5C => "\u{72}", + 0x1E5D => "\u{72}", + 0x1E5E => "\u{72}", + 0x1E5F => "\u{72}", + 0x1E60 => "\u{73}", + 0x1E61 => "\u{73}", + 0x1E62 => "\u{73}", + 0x1E63 => "\u{73}", + 0x1E64 => "\u{73}", + 0x1E65 => "\u{73}", + 0x1E66 => "\u{73}", + 0x1E67 => "\u{73}", + 0x1E68 => "\u{73}", + 0x1E69 => "\u{73}", + 0x1E6A => "\u{74}", + 0x1E6B => "\u{74}", + 0x1E6C => "\u{74}", + 0x1E6D => "\u{74}", + 0x1E6E => "\u{74}", + 0x1E6F => "\u{74}", + 0x1E70 => "\u{74}", + 0x1E71 => "\u{74}", + 0x1E72 => "\u{75}", + 0x1E73 => "\u{75}", + 0x1E74 => "\u{75}", + 0x1E75 => "\u{75}", + 0x1E76 => "\u{75}", + 0x1E77 => "\u{75}", + 0x1E78 => "\u{75}", + 0x1E79 => "\u{75}", + 0x1E7A => "\u{75}", + 0x1E7B => "\u{75}", + 0x1E7C => "\u{76}", + 0x1E7D => "\u{76}", + 0x1E7E => "\u{76}", + 0x1E7F => "\u{76}", + 0x1E80 => "\u{77}", + 0x1E81 => "\u{77}", + 0x1E82 => "\u{77}", + 0x1E83 => "\u{77}", + 0x1E84 => "\u{77}", + 0x1E85 => "\u{77}", + 0x1E86 => "\u{77}", + 0x1E87 => "\u{77}", + 0x1E88 => "\u{77}", + 0x1E89 => "\u{77}", + 0x1E8A => "\u{78}", + 0x1E8B => "\u{78}", + 0x1E8C => "\u{78}", + 0x1E8D => "\u{78}", + 0x1E8E => "\u{79}", + 0x1E8F => "\u{79}", + 0x1E90 => "\u{7A}", + 0x1E91 => "\u{7A}", + 0x1E92 => "\u{7A}", + 0x1E93 => "\u{7A}", + 0x1E94 => "\u{7A}", + 0x1E95 => "\u{7A}", + 0x1E96 => "\u{68}", + 0x1E97 => "\u{74}", + 0x1E98 => "\u{77}", + 0x1E99 => "\u{79}", + 0x1E9B => "\u{17F}", + 0x1E9E => "\u{DF}", + 0x1EA0 => "\u{61}", + 0x1EA1 => "\u{61}", + 0x1EA2 => "\u{61}", + 0x1EA3 => "\u{61}", + 0x1EA4 => "\u{61}", + 0x1EA5 => "\u{61}", + 0x1EA6 => "\u{61}", + 0x1EA7 => "\u{61}", + 0x1EA8 => "\u{61}", + 0x1EA9 => "\u{61}", + 0x1EAA => "\u{61}", + 0x1EAB => "\u{61}", + 0x1EAC => "\u{61}", + 0x1EAD => "\u{61}", + 0x1EAE => "\u{61}", + 0x1EAF => "\u{61}", + 0x1EB0 => "\u{61}", + 0x1EB1 => "\u{61}", + 0x1EB2 => "\u{61}", + 0x1EB3 => "\u{61}", + 0x1EB4 => "\u{61}", + 0x1EB5 => "\u{61}", + 0x1EB6 => "\u{61}", + 0x1EB7 => "\u{61}", + 0x1EB8 => "\u{65}", + 0x1EB9 => "\u{65}", + 0x1EBA => "\u{65}", + 0x1EBB => "\u{65}", + 0x1EBC => "\u{65}", + 0x1EBD => "\u{65}", + 0x1EBE => "\u{65}", + 0x1EBF => "\u{65}", + 0x1EC0 => "\u{65}", + 0x1EC1 => "\u{65}", + 0x1EC2 => "\u{65}", + 0x1EC3 => "\u{65}", + 0x1EC4 => "\u{65}", + 0x1EC5 => "\u{65}", + 0x1EC6 => "\u{65}", + 0x1EC7 => "\u{65}", + 0x1EC8 => "\u{69}", + 0x1EC9 => "\u{69}", + 0x1ECA => "\u{69}", + 0x1ECB => "\u{69}", + 0x1ECC => "\u{6F}", + 0x1ECD => "\u{6F}", + 0x1ECE => "\u{6F}", + 0x1ECF => "\u{6F}", + 0x1ED0 => "\u{6F}", + 0x1ED1 => "\u{6F}", + 0x1ED2 => "\u{6F}", + 0x1ED3 => "\u{6F}", + 0x1ED4 => "\u{6F}", + 0x1ED5 => "\u{6F}", + 0x1ED6 => "\u{6F}", + 0x1ED7 => "\u{6F}", + 0x1ED8 => "\u{6F}", + 0x1ED9 => "\u{6F}", + 0x1EDA => "\u{6F}", + 0x1EDB => "\u{6F}", + 0x1EDC => "\u{6F}", + 0x1EDD => "\u{6F}", + 0x1EDE => "\u{6F}", + 0x1EDF => "\u{6F}", + 0x1EE0 => "\u{6F}", + 0x1EE1 => "\u{6F}", + 0x1EE2 => "\u{6F}", + 0x1EE3 => "\u{6F}", + 0x1EE4 => "\u{75}", + 0x1EE5 => "\u{75}", + 0x1EE6 => "\u{75}", + 0x1EE7 => "\u{75}", + 0x1EE8 => "\u{75}", + 0x1EE9 => "\u{75}", + 0x1EEA => "\u{75}", + 0x1EEB => "\u{75}", + 0x1EEC => "\u{75}", + 0x1EED => "\u{75}", + 0x1EEE => "\u{75}", + 0x1EEF => "\u{75}", + 0x1EF0 => "\u{75}", + 0x1EF1 => "\u{75}", + 0x1EF2 => "\u{79}", + 0x1EF3 => "\u{79}", + 0x1EF4 => "\u{79}", + 0x1EF5 => "\u{79}", + 0x1EF6 => "\u{79}", + 0x1EF7 => "\u{79}", + 0x1EF8 => "\u{79}", + 0x1EF9 => "\u{79}", + 0x1EFA => "\u{1EFB}", + 0x1EFC => "\u{1EFD}", + 0x1EFE => "\u{1EFF}", + 0x1F00 => "\u{3B1}", + 0x1F01 => "\u{3B1}", + 0x1F02 => "\u{3B1}", + 0x1F03 => "\u{3B1}", + 0x1F04 => "\u{3B1}", + 0x1F05 => "\u{3B1}", + 0x1F06 => "\u{3B1}", + 0x1F07 => "\u{3B1}", + 0x1F08 => "\u{3B1}", + 0x1F09 => "\u{3B1}", + 0x1F0A => "\u{3B1}", + 0x1F0B => "\u{3B1}", + 0x1F0C => "\u{3B1}", + 0x1F0D => "\u{3B1}", + 0x1F0E => "\u{3B1}", + 0x1F0F => "\u{3B1}", + 0x1F10 => "\u{3B5}", + 0x1F11 => "\u{3B5}", + 0x1F12 => "\u{3B5}", + 0x1F13 => "\u{3B5}", + 0x1F14 => "\u{3B5}", + 0x1F15 => "\u{3B5}", + 0x1F18 => "\u{3B5}", + 0x1F19 => "\u{3B5}", + 0x1F1A => "\u{3B5}", + 0x1F1B => "\u{3B5}", + 0x1F1C => "\u{3B5}", + 0x1F1D => "\u{3B5}", + 0x1F20 => "\u{3B7}", + 0x1F21 => "\u{3B7}", + 0x1F22 => "\u{3B7}", + 0x1F23 => "\u{3B7}", + 0x1F24 => "\u{3B7}", + 0x1F25 => "\u{3B7}", + 0x1F26 => "\u{3B7}", + 0x1F27 => "\u{3B7}", + 0x1F28 => "\u{3B7}", + 0x1F29 => "\u{3B7}", + 0x1F2A => "\u{3B7}", + 0x1F2B => "\u{3B7}", + 0x1F2C => "\u{3B7}", + 0x1F2D => "\u{3B7}", + 0x1F2E => "\u{3B7}", + 0x1F2F => "\u{3B7}", + 0x1F30 => "\u{3B9}", + 0x1F31 => "\u{3B9}", + 0x1F32 => "\u{3B9}", + 0x1F33 => "\u{3B9}", + 0x1F34 => "\u{3B9}", + 0x1F35 => "\u{3B9}", + 0x1F36 => "\u{3B9}", + 0x1F37 => "\u{3B9}", + 0x1F38 => "\u{3B9}", + 0x1F39 => "\u{3B9}", + 0x1F3A => "\u{3B9}", + 0x1F3B => "\u{3B9}", + 0x1F3C => "\u{3B9}", + 0x1F3D => "\u{3B9}", + 0x1F3E => "\u{3B9}", + 0x1F3F => "\u{3B9}", + 0x1F40 => "\u{3BF}", + 0x1F41 => "\u{3BF}", + 0x1F42 => "\u{3BF}", + 0x1F43 => "\u{3BF}", + 0x1F44 => "\u{3BF}", + 0x1F45 => "\u{3BF}", + 0x1F48 => "\u{3BF}", + 0x1F49 => "\u{3BF}", + 0x1F4A => "\u{3BF}", + 0x1F4B => "\u{3BF}", + 0x1F4C => "\u{3BF}", + 0x1F4D => "\u{3BF}", + 0x1F50 => "\u{3C5}", + 0x1F51 => "\u{3C5}", + 0x1F52 => "\u{3C5}", + 0x1F53 => "\u{3C5}", + 0x1F54 => "\u{3C5}", + 0x1F55 => "\u{3C5}", + 0x1F56 => "\u{3C5}", + 0x1F57 => "\u{3C5}", + 0x1F59 => "\u{3C5}", + 0x1F5B => "\u{3C5}", + 0x1F5D => "\u{3C5}", + 0x1F5F => "\u{3C5}", + 0x1F60 => "\u{3C9}", + 0x1F61 => "\u{3C9}", + 0x1F62 => "\u{3C9}", + 0x1F63 => "\u{3C9}", + 0x1F64 => "\u{3C9}", + 0x1F65 => "\u{3C9}", + 0x1F66 => "\u{3C9}", + 0x1F67 => "\u{3C9}", + 0x1F68 => "\u{3C9}", + 0x1F69 => "\u{3C9}", + 0x1F6A => "\u{3C9}", + 0x1F6B => "\u{3C9}", + 0x1F6C => "\u{3C9}", + 0x1F6D => "\u{3C9}", + 0x1F6E => "\u{3C9}", + 0x1F6F => "\u{3C9}", + 0x1F70 => "\u{3B1}", + 0x1F71 => "\u{3B1}", + 0x1F72 => "\u{3B5}", + 0x1F73 => "\u{3B5}", + 0x1F74 => "\u{3B7}", + 0x1F75 => "\u{3B7}", + 0x1F76 => "\u{3B9}", + 0x1F77 => "\u{3B9}", + 0x1F78 => "\u{3BF}", + 0x1F79 => "\u{3BF}", + 0x1F7A => "\u{3C5}", + 0x1F7B => "\u{3C5}", + 0x1F7C => "\u{3C9}", + 0x1F7D => "\u{3C9}", + 0x1F80 => "\u{3B1}", + 0x1F81 => "\u{3B1}", + 0x1F82 => "\u{3B1}", + 0x1F83 => "\u{3B1}", + 0x1F84 => "\u{3B1}", + 0x1F85 => "\u{3B1}", + 0x1F86 => "\u{3B1}", + 0x1F87 => "\u{3B1}", + 0x1F88 => "\u{3B1}", + 0x1F89 => "\u{3B1}", + 0x1F8A => "\u{3B1}", + 0x1F8B => "\u{3B1}", + 0x1F8C => "\u{3B1}", + 0x1F8D => "\u{3B1}", + 0x1F8E => "\u{3B1}", + 0x1F8F => "\u{3B1}", + 0x1F90 => "\u{3B7}", + 0x1F91 => "\u{3B7}", + 0x1F92 => "\u{3B7}", + 0x1F93 => "\u{3B7}", + 0x1F94 => "\u{3B7}", + 0x1F95 => "\u{3B7}", + 0x1F96 => "\u{3B7}", + 0x1F97 => "\u{3B7}", + 0x1F98 => "\u{3B7}", + 0x1F99 => "\u{3B7}", + 0x1F9A => "\u{3B7}", + 0x1F9B => "\u{3B7}", + 0x1F9C => "\u{3B7}", + 0x1F9D => "\u{3B7}", + 0x1F9E => "\u{3B7}", + 0x1F9F => "\u{3B7}", + 0x1FA0 => "\u{3C9}", + 0x1FA1 => "\u{3C9}", + 0x1FA2 => "\u{3C9}", + 0x1FA3 => "\u{3C9}", + 0x1FA4 => "\u{3C9}", + 0x1FA5 => "\u{3C9}", + 0x1FA6 => "\u{3C9}", + 0x1FA7 => "\u{3C9}", + 0x1FA8 => "\u{3C9}", + 0x1FA9 => "\u{3C9}", + 0x1FAA => "\u{3C9}", + 0x1FAB => "\u{3C9}", + 0x1FAC => "\u{3C9}", + 0x1FAD => "\u{3C9}", + 0x1FAE => "\u{3C9}", + 0x1FAF => "\u{3C9}", + 0x1FB0 => "\u{3B1}", + 0x1FB1 => "\u{3B1}", + 0x1FB2 => "\u{3B1}", + 0x1FB3 => "\u{3B1}", + 0x1FB4 => "\u{3B1}", + 0x1FB6 => "\u{3B1}", + 0x1FB7 => "\u{3B1}", + 0x1FB8 => "\u{3B1}", + 0x1FB9 => "\u{3B1}", + 0x1FBA => "\u{3B1}", + 0x1FBB => "\u{3B1}", + 0x1FBC => "\u{3B1}", + 0x1FBE => "\u{3B9}", + 0x1FC1 => "\u{A8}", + 0x1FC2 => "\u{3B7}", + 0x1FC3 => "\u{3B7}", + 0x1FC4 => "\u{3B7}", + 0x1FC6 => "\u{3B7}", + 0x1FC7 => "\u{3B7}", + 0x1FC8 => "\u{3B5}", + 0x1FC9 => "\u{3B5}", + 0x1FCA => "\u{3B7}", + 0x1FCB => "\u{3B7}", + 0x1FCC => "\u{3B7}", + 0x1FCD => "\u{1FBF}", + 0x1FCE => "\u{1FBF}", + 0x1FCF => "\u{1FBF}", + 0x1FD0 => "\u{3B9}", + 0x1FD1 => "\u{3B9}", + 0x1FD2 => "\u{3B9}", + 0x1FD3 => "\u{3B9}", + 0x1FD6 => "\u{3B9}", + 0x1FD7 => "\u{3B9}", + 0x1FD8 => "\u{3B9}", + 0x1FD9 => "\u{3B9}", + 0x1FDA => "\u{3B9}", + 0x1FDB => "\u{3B9}", + 0x1FDD => "\u{1FFE}", + 0x1FDE => "\u{1FFE}", + 0x1FDF => "\u{1FFE}", + 0x1FE0 => "\u{3C5}", + 0x1FE1 => "\u{3C5}", + 0x1FE2 => "\u{3C5}", + 0x1FE3 => "\u{3C5}", + 0x1FE4 => "\u{3C1}", + 0x1FE5 => "\u{3C1}", + 0x1FE6 => "\u{3C5}", + 0x1FE7 => "\u{3C5}", + 0x1FE8 => "\u{3C5}", + 0x1FE9 => "\u{3C5}", + 0x1FEA => "\u{3C5}", + 0x1FEB => "\u{3C5}", + 0x1FEC => "\u{3C1}", + 0x1FED => "\u{A8}", + 0x1FEE => "\u{A8}", + 0x1FEF => "\u{60}", + 0x1FF2 => "\u{3C9}", + 0x1FF3 => "\u{3C9}", + 0x1FF4 => "\u{3C9}", + 0x1FF6 => "\u{3C9}", + 0x1FF7 => "\u{3C9}", + 0x1FF8 => "\u{3BF}", + 0x1FF9 => "\u{3BF}", + 0x1FFA => "\u{3C9}", + 0x1FFB => "\u{3C9}", + 0x1FFC => "\u{3C9}", + 0x1FFD => "\u{B4}", + 0x2126 => "\u{3C9}", + 0x212A => "\u{6B}", + 0x212B => "\u{61}", + 0x2132 => "\u{214E}", + 0x2160 => "\u{2170}", + 0x2161 => "\u{2171}", + 0x2162 => "\u{2172}", + 0x2163 => "\u{2173}", + 0x2164 => "\u{2174}", + 0x2165 => "\u{2175}", + 0x2166 => "\u{2176}", + 0x2167 => "\u{2177}", + 0x2168 => "\u{2178}", + 0x2169 => "\u{2179}", + 0x216A => "\u{217A}", + 0x216B => "\u{217B}", + 0x216C => "\u{217C}", + 0x216D => "\u{217D}", + 0x216E => "\u{217E}", + 0x216F => "\u{217F}", + 0x2183 => "\u{2184}", + 0x219A => "\u{2190}", + 0x219B => "\u{2192}", + 0x21AE => "\u{2194}", + 0x21CD => "\u{21D0}", + 0x21CE => "\u{21D4}", + 0x21CF => "\u{21D2}", + 0x2204 => "\u{2203}", + 0x2209 => "\u{2208}", + 0x220C => "\u{220B}", + 0x2224 => "\u{2223}", + 0x2226 => "\u{2225}", + 0x2241 => "\u{223C}", + 0x2244 => "\u{2243}", + 0x2247 => "\u{2245}", + 0x2249 => "\u{2248}", + 0x2260 => "\u{3D}", + 0x2262 => "\u{2261}", + 0x226D => "\u{224D}", + 0x226E => "\u{3C}", + 0x226F => "\u{3E}", + 0x2270 => "\u{2264}", + 0x2271 => "\u{2265}", + 0x2274 => "\u{2272}", + 0x2275 => "\u{2273}", + 0x2278 => "\u{2276}", + 0x2279 => "\u{2277}", + 0x2280 => "\u{227A}", + 0x2281 => "\u{227B}", + 0x2284 => "\u{2282}", + 0x2285 => "\u{2283}", + 0x2288 => "\u{2286}", + 0x2289 => "\u{2287}", + 0x22AC => "\u{22A2}", + 0x22AD => "\u{22A8}", + 0x22AE => "\u{22A9}", + 0x22AF => "\u{22AB}", + 0x22E0 => "\u{227C}", + 0x22E1 => "\u{227D}", + 0x22E2 => "\u{2291}", + 0x22E3 => "\u{2292}", + 0x22EA => "\u{22B2}", + 0x22EB => "\u{22B3}", + 0x22EC => "\u{22B4}", + 0x22ED => "\u{22B5}", + 0x2329 => "\u{3008}", + 0x232A => "\u{3009}", + 0x24B6 => "\u{24D0}", + 0x24B7 => "\u{24D1}", + 0x24B8 => "\u{24D2}", + 0x24B9 => "\u{24D3}", + 0x24BA => "\u{24D4}", + 0x24BB => "\u{24D5}", + 0x24BC => "\u{24D6}", + 0x24BD => "\u{24D7}", + 0x24BE => "\u{24D8}", + 0x24BF => "\u{24D9}", + 0x24C0 => "\u{24DA}", + 0x24C1 => "\u{24DB}", + 0x24C2 => "\u{24DC}", + 0x24C3 => "\u{24DD}", + 0x24C4 => "\u{24DE}", + 0x24C5 => "\u{24DF}", + 0x24C6 => "\u{24E0}", + 0x24C7 => "\u{24E1}", + 0x24C8 => "\u{24E2}", + 0x24C9 => "\u{24E3}", + 0x24CA => "\u{24E4}", + 0x24CB => "\u{24E5}", + 0x24CC => "\u{24E6}", + 0x24CD => "\u{24E7}", + 0x24CE => "\u{24E8}", + 0x24CF => "\u{24E9}", + 0x2ADC => "\u{2ADD}", + 0x2C00 => "\u{2C30}", + 0x2C01 => "\u{2C31}", + 0x2C02 => "\u{2C32}", + 0x2C03 => "\u{2C33}", + 0x2C04 => "\u{2C34}", + 0x2C05 => "\u{2C35}", + 0x2C06 => "\u{2C36}", + 0x2C07 => "\u{2C37}", + 0x2C08 => "\u{2C38}", + 0x2C09 => "\u{2C39}", + 0x2C0A => "\u{2C3A}", + 0x2C0B => "\u{2C3B}", + 0x2C0C => "\u{2C3C}", + 0x2C0D => "\u{2C3D}", + 0x2C0E => "\u{2C3E}", + 0x2C0F => "\u{2C3F}", + 0x2C10 => "\u{2C40}", + 0x2C11 => "\u{2C41}", + 0x2C12 => "\u{2C42}", + 0x2C13 => "\u{2C43}", + 0x2C14 => "\u{2C44}", + 0x2C15 => "\u{2C45}", + 0x2C16 => "\u{2C46}", + 0x2C17 => "\u{2C47}", + 0x2C18 => "\u{2C48}", + 0x2C19 => "\u{2C49}", + 0x2C1A => "\u{2C4A}", + 0x2C1B => "\u{2C4B}", + 0x2C1C => "\u{2C4C}", + 0x2C1D => "\u{2C4D}", + 0x2C1E => "\u{2C4E}", + 0x2C1F => "\u{2C4F}", + 0x2C20 => "\u{2C50}", + 0x2C21 => "\u{2C51}", + 0x2C22 => "\u{2C52}", + 0x2C23 => "\u{2C53}", + 0x2C24 => "\u{2C54}", + 0x2C25 => "\u{2C55}", + 0x2C26 => "\u{2C56}", + 0x2C27 => "\u{2C57}", + 0x2C28 => "\u{2C58}", + 0x2C29 => "\u{2C59}", + 0x2C2A => "\u{2C5A}", + 0x2C2B => "\u{2C5B}", + 0x2C2C => "\u{2C5C}", + 0x2C2D => "\u{2C5D}", + 0x2C2E => "\u{2C5E}", + 0x2C2F => "\u{2C5F}", + 0x2C60 => "\u{2C61}", + 0x2C62 => "\u{26B}", + 0x2C63 => "\u{1D7D}", + 0x2C64 => "\u{27D}", + 0x2C67 => "\u{2C68}", + 0x2C69 => "\u{2C6A}", + 0x2C6B => "\u{2C6C}", + 0x2C6D => "\u{251}", + 0x2C6E => "\u{271}", + 0x2C6F => "\u{250}", + 0x2C70 => "\u{252}", + 0x2C72 => "\u{2C73}", + 0x2C75 => "\u{2C76}", + 0x2C7E => "\u{23F}", + 0x2C7F => "\u{240}", + 0x2C80 => "\u{2C81}", + 0x2C82 => "\u{2C83}", + 0x2C84 => "\u{2C85}", + 0x2C86 => "\u{2C87}", + 0x2C88 => "\u{2C89}", + 0x2C8A => "\u{2C8B}", + 0x2C8C => "\u{2C8D}", + 0x2C8E => "\u{2C8F}", + 0x2C90 => "\u{2C91}", + 0x2C92 => "\u{2C93}", + 0x2C94 => "\u{2C95}", + 0x2C96 => "\u{2C97}", + 0x2C98 => "\u{2C99}", + 0x2C9A => "\u{2C9B}", + 0x2C9C => "\u{2C9D}", + 0x2C9E => "\u{2C9F}", + 0x2CA0 => "\u{2CA1}", + 0x2CA2 => "\u{2CA3}", + 0x2CA4 => "\u{2CA5}", + 0x2CA6 => "\u{2CA7}", + 0x2CA8 => "\u{2CA9}", + 0x2CAA => "\u{2CAB}", + 0x2CAC => "\u{2CAD}", + 0x2CAE => "\u{2CAF}", + 0x2CB0 => "\u{2CB1}", + 0x2CB2 => "\u{2CB3}", + 0x2CB4 => "\u{2CB5}", + 0x2CB6 => "\u{2CB7}", + 0x2CB8 => "\u{2CB9}", + 0x2CBA => "\u{2CBB}", + 0x2CBC => "\u{2CBD}", + 0x2CBE => "\u{2CBF}", + 0x2CC0 => "\u{2CC1}", + 0x2CC2 => "\u{2CC3}", + 0x2CC4 => "\u{2CC5}", + 0x2CC6 => "\u{2CC7}", + 0x2CC8 => "\u{2CC9}", + 0x2CCA => "\u{2CCB}", + 0x2CCC => "\u{2CCD}", + 0x2CCE => "\u{2CCF}", + 0x2CD0 => "\u{2CD1}", + 0x2CD2 => "\u{2CD3}", + 0x2CD4 => "\u{2CD5}", + 0x2CD6 => "\u{2CD7}", + 0x2CD8 => "\u{2CD9}", + 0x2CDA => "\u{2CDB}", + 0x2CDC => "\u{2CDD}", + 0x2CDE => "\u{2CDF}", + 0x2CE0 => "\u{2CE1}", + 0x2CE2 => "\u{2CE3}", + 0x2CEB => "\u{2CEC}", + 0x2CED => "\u{2CEE}", + 0x2CF2 => "\u{2CF3}", + 0x304C => "\u{304B}", + 0x304E => "\u{304D}", + 0x3050 => "\u{304F}", + 0x3052 => "\u{3051}", + 0x3054 => "\u{3053}", + 0x3056 => "\u{3055}", + 0x3058 => "\u{3057}", + 0x305A => "\u{3059}", + 0x305C => "\u{305B}", + 0x305E => "\u{305D}", + 0x3060 => "\u{305F}", + 0x3062 => "\u{3061}", + 0x3065 => "\u{3064}", + 0x3067 => "\u{3066}", + 0x3069 => "\u{3068}", + 0x3070 => "\u{306F}", + 0x3071 => "\u{306F}", + 0x3073 => "\u{3072}", + 0x3074 => "\u{3072}", + 0x3076 => "\u{3075}", + 0x3077 => "\u{3075}", + 0x3079 => "\u{3078}", + 0x307A => "\u{3078}", + 0x307C => "\u{307B}", + 0x307D => "\u{307B}", + 0x3094 => "\u{3046}", + 0x309E => "\u{309D}", + 0x30AC => "\u{30AB}", + 0x30AE => "\u{30AD}", + 0x30B0 => "\u{30AF}", + 0x30B2 => "\u{30B1}", + 0x30B4 => "\u{30B3}", + 0x30B6 => "\u{30B5}", + 0x30B8 => "\u{30B7}", + 0x30BA => "\u{30B9}", + 0x30BC => "\u{30BB}", + 0x30BE => "\u{30BD}", + 0x30C0 => "\u{30BF}", + 0x30C2 => "\u{30C1}", + 0x30C5 => "\u{30C4}", + 0x30C7 => "\u{30C6}", + 0x30C9 => "\u{30C8}", + 0x30D0 => "\u{30CF}", + 0x30D1 => "\u{30CF}", + 0x30D3 => "\u{30D2}", + 0x30D4 => "\u{30D2}", + 0x30D6 => "\u{30D5}", + 0x30D7 => "\u{30D5}", + 0x30D9 => "\u{30D8}", + 0x30DA => "\u{30D8}", + 0x30DC => "\u{30DB}", + 0x30DD => "\u{30DB}", + 0x30F4 => "\u{30A6}", + 0x30F7 => "\u{30EF}", + 0x30F8 => "\u{30F0}", + 0x30F9 => "\u{30F1}", + 0x30FA => "\u{30F2}", + 0x30FE => "\u{30FD}", + 0xA640 => "\u{A641}", + 0xA642 => "\u{A643}", + 0xA644 => "\u{A645}", + 0xA646 => "\u{A647}", + 0xA648 => "\u{A649}", + 0xA64A => "\u{A64B}", + 0xA64C => "\u{A64D}", + 0xA64E => "\u{A64F}", + 0xA650 => "\u{A651}", + 0xA652 => "\u{A653}", + 0xA654 => "\u{A655}", + 0xA656 => "\u{A657}", + 0xA658 => "\u{A659}", + 0xA65A => "\u{A65B}", + 0xA65C => "\u{A65D}", + 0xA65E => "\u{A65F}", + 0xA660 => "\u{A661}", + 0xA662 => "\u{A663}", + 0xA664 => "\u{A665}", + 0xA666 => "\u{A667}", + 0xA668 => "\u{A669}", + 0xA66A => "\u{A66B}", + 0xA66C => "\u{A66D}", + 0xA680 => "\u{A681}", + 0xA682 => "\u{A683}", + 0xA684 => "\u{A685}", + 0xA686 => "\u{A687}", + 0xA688 => "\u{A689}", + 0xA68A => "\u{A68B}", + 0xA68C => "\u{A68D}", + 0xA68E => "\u{A68F}", + 0xA690 => "\u{A691}", + 0xA692 => "\u{A693}", + 0xA694 => "\u{A695}", + 0xA696 => "\u{A697}", + 0xA698 => "\u{A699}", + 0xA69A => "\u{A69B}", + 0xA722 => "\u{A723}", + 0xA724 => "\u{A725}", + 0xA726 => "\u{A727}", + 0xA728 => "\u{A729}", + 0xA72A => "\u{A72B}", + 0xA72C => "\u{A72D}", + 0xA72E => "\u{A72F}", + 0xA732 => "\u{A733}", + 0xA734 => "\u{A735}", + 0xA736 => "\u{A737}", + 0xA738 => "\u{A739}", + 0xA73A => "\u{A73B}", + 0xA73C => "\u{A73D}", + 0xA73E => "\u{A73F}", + 0xA740 => "\u{A741}", + 0xA742 => "\u{A743}", + 0xA744 => "\u{A745}", + 0xA746 => "\u{A747}", + 0xA748 => "\u{A749}", + 0xA74A => "\u{A74B}", + 0xA74C => "\u{A74D}", + 0xA74E => "\u{A74F}", + 0xA750 => "\u{A751}", + 0xA752 => "\u{A753}", + 0xA754 => "\u{A755}", + 0xA756 => "\u{A757}", + 0xA758 => "\u{A759}", + 0xA75A => "\u{A75B}", + 0xA75C => "\u{A75D}", + 0xA75E => "\u{A75F}", + 0xA760 => "\u{A761}", + 0xA762 => "\u{A763}", + 0xA764 => "\u{A765}", + 0xA766 => "\u{A767}", + 0xA768 => "\u{A769}", + 0xA76A => "\u{A76B}", + 0xA76C => "\u{A76D}", + 0xA76E => "\u{A76F}", + 0xA779 => "\u{A77A}", + 0xA77B => "\u{A77C}", + 0xA77D => "\u{1D79}", + 0xA77E => "\u{A77F}", + 0xA780 => "\u{A781}", + 0xA782 => "\u{A783}", + 0xA784 => "\u{A785}", + 0xA786 => "\u{A787}", + 0xA78B => "\u{A78C}", + 0xA78D => "\u{265}", + 0xA790 => "\u{A791}", + 0xA792 => "\u{A793}", + 0xA796 => "\u{A797}", + 0xA798 => "\u{A799}", + 0xA79A => "\u{A79B}", + 0xA79C => "\u{A79D}", + 0xA79E => "\u{A79F}", + 0xA7A0 => "\u{A7A1}", + 0xA7A2 => "\u{A7A3}", + 0xA7A4 => "\u{A7A5}", + 0xA7A6 => "\u{A7A7}", + 0xA7A8 => "\u{A7A9}", + 0xA7AA => "\u{266}", + 0xA7AB => "\u{25C}", + 0xA7AC => "\u{261}", + 0xA7AD => "\u{26C}", + 0xA7AE => "\u{26A}", + 0xA7B0 => "\u{29E}", + 0xA7B1 => "\u{287}", + 0xA7B2 => "\u{29D}", + 0xA7B3 => "\u{AB53}", + 0xA7B4 => "\u{A7B5}", + 0xA7B6 => "\u{A7B7}", + 0xA7B8 => "\u{A7B9}", + 0xA7BA => "\u{A7BB}", + 0xA7BC => "\u{A7BD}", + 0xA7BE => "\u{A7BF}", + 0xA7C0 => "\u{A7C1}", + 0xA7C2 => "\u{A7C3}", + 0xA7C4 => "\u{A794}", + 0xA7C5 => "\u{282}", + 0xA7C6 => "\u{1D8E}", + 0xA7C7 => "\u{A7C8}", + 0xA7C9 => "\u{A7CA}", + 0xA7CB => "\u{264}", + 0xA7CC => "\u{A7CD}", + 0xA7CE => "\u{A7CF}", + 0xA7D0 => "\u{A7D1}", + 0xA7D2 => "\u{A7D3}", + 0xA7D4 => "\u{A7D5}", + 0xA7D6 => "\u{A7D7}", + 0xA7D8 => "\u{A7D9}", + 0xA7DA => "\u{A7DB}", + 0xA7DC => "\u{19B}", + 0xA7F5 => "\u{A7F6}", + 0xF900 => "\u{20}\u{8C48}\u{20}", + 0xF901 => "\u{20}\u{66F4}\u{20}", + 0xF902 => "\u{20}\u{8ECA}\u{20}", + 0xF903 => "\u{20}\u{8CC8}\u{20}", + 0xF904 => "\u{20}\u{6ED1}\u{20}", + 0xF905 => "\u{20}\u{4E32}\u{20}", + 0xF906 => "\u{20}\u{53E5}\u{20}", + 0xF907 => "\u{20}\u{9F9C}\u{20}", + 0xF908 => "\u{20}\u{9F9C}\u{20}", + 0xF909 => "\u{20}\u{5951}\u{20}", + 0xF90A => "\u{20}\u{91D1}\u{20}", + 0xF90B => "\u{20}\u{5587}\u{20}", + 0xF90C => "\u{20}\u{5948}\u{20}", + 0xF90D => "\u{20}\u{61F6}\u{20}", + 0xF90E => "\u{20}\u{7669}\u{20}", + 0xF90F => "\u{20}\u{7F85}\u{20}", + 0xF910 => "\u{20}\u{863F}\u{20}", + 0xF911 => "\u{20}\u{87BA}\u{20}", + 0xF912 => "\u{20}\u{88F8}\u{20}", + 0xF913 => "\u{20}\u{908F}\u{20}", + 0xF914 => "\u{20}\u{6A02}\u{20}", + 0xF915 => "\u{20}\u{6D1B}\u{20}", + 0xF916 => "\u{20}\u{70D9}\u{20}", + 0xF917 => "\u{20}\u{73DE}\u{20}", + 0xF918 => "\u{20}\u{843D}\u{20}", + 0xF919 => "\u{20}\u{916A}\u{20}", + 0xF91A => "\u{20}\u{99F1}\u{20}", + 0xF91B => "\u{20}\u{4E82}\u{20}", + 0xF91C => "\u{20}\u{5375}\u{20}", + 0xF91D => "\u{20}\u{6B04}\u{20}", + 0xF91E => "\u{20}\u{721B}\u{20}", + 0xF91F => "\u{20}\u{862D}\u{20}", + 0xF920 => "\u{20}\u{9E1E}\u{20}", + 0xF921 => "\u{20}\u{5D50}\u{20}", + 0xF922 => "\u{20}\u{6FEB}\u{20}", + 0xF923 => "\u{20}\u{85CD}\u{20}", + 0xF924 => "\u{20}\u{8964}\u{20}", + 0xF925 => "\u{20}\u{62C9}\u{20}", + 0xF926 => "\u{20}\u{81D8}\u{20}", + 0xF927 => "\u{20}\u{881F}\u{20}", + 0xF928 => "\u{20}\u{5ECA}\u{20}", + 0xF929 => "\u{20}\u{6717}\u{20}", + 0xF92A => "\u{20}\u{6D6A}\u{20}", + 0xF92B => "\u{20}\u{72FC}\u{20}", + 0xF92C => "\u{20}\u{90CE}\u{20}", + 0xF92D => "\u{20}\u{4F86}\u{20}", + 0xF92E => "\u{20}\u{51B7}\u{20}", + 0xF92F => "\u{20}\u{52DE}\u{20}", + 0xF930 => "\u{20}\u{64C4}\u{20}", + 0xF931 => "\u{20}\u{6AD3}\u{20}", + 0xF932 => "\u{20}\u{7210}\u{20}", + 0xF933 => "\u{20}\u{76E7}\u{20}", + 0xF934 => "\u{20}\u{8001}\u{20}", + 0xF935 => "\u{20}\u{8606}\u{20}", + 0xF936 => "\u{20}\u{865C}\u{20}", + 0xF937 => "\u{20}\u{8DEF}\u{20}", + 0xF938 => "\u{20}\u{9732}\u{20}", + 0xF939 => "\u{20}\u{9B6F}\u{20}", + 0xF93A => "\u{20}\u{9DFA}\u{20}", + 0xF93B => "\u{20}\u{788C}\u{20}", + 0xF93C => "\u{20}\u{797F}\u{20}", + 0xF93D => "\u{20}\u{7DA0}\u{20}", + 0xF93E => "\u{20}\u{83C9}\u{20}", + 0xF93F => "\u{20}\u{9304}\u{20}", + 0xF940 => "\u{20}\u{9E7F}\u{20}", + 0xF941 => "\u{20}\u{8AD6}\u{20}", + 0xF942 => "\u{20}\u{58DF}\u{20}", + 0xF943 => "\u{20}\u{5F04}\u{20}", + 0xF944 => "\u{20}\u{7C60}\u{20}", + 0xF945 => "\u{20}\u{807E}\u{20}", + 0xF946 => "\u{20}\u{7262}\u{20}", + 0xF947 => "\u{20}\u{78CA}\u{20}", + 0xF948 => "\u{20}\u{8CC2}\u{20}", + 0xF949 => "\u{20}\u{96F7}\u{20}", + 0xF94A => "\u{20}\u{58D8}\u{20}", + 0xF94B => "\u{20}\u{5C62}\u{20}", + 0xF94C => "\u{20}\u{6A13}\u{20}", + 0xF94D => "\u{20}\u{6DDA}\u{20}", + 0xF94E => "\u{20}\u{6F0F}\u{20}", + 0xF94F => "\u{20}\u{7D2F}\u{20}", + 0xF950 => "\u{20}\u{7E37}\u{20}", + 0xF951 => "\u{20}\u{964B}\u{20}", + 0xF952 => "\u{20}\u{52D2}\u{20}", + 0xF953 => "\u{20}\u{808B}\u{20}", + 0xF954 => "\u{20}\u{51DC}\u{20}", + 0xF955 => "\u{20}\u{51CC}\u{20}", + 0xF956 => "\u{20}\u{7A1C}\u{20}", + 0xF957 => "\u{20}\u{7DBE}\u{20}", + 0xF958 => "\u{20}\u{83F1}\u{20}", + 0xF959 => "\u{20}\u{9675}\u{20}", + 0xF95A => "\u{20}\u{8B80}\u{20}", + 0xF95B => "\u{20}\u{62CF}\u{20}", + 0xF95C => "\u{20}\u{6A02}\u{20}", + 0xF95D => "\u{20}\u{8AFE}\u{20}", + 0xF95E => "\u{20}\u{4E39}\u{20}", + 0xF95F => "\u{20}\u{5BE7}\u{20}", + 0xF960 => "\u{20}\u{6012}\u{20}", + 0xF961 => "\u{20}\u{7387}\u{20}", + 0xF962 => "\u{20}\u{7570}\u{20}", + 0xF963 => "\u{20}\u{5317}\u{20}", + 0xF964 => "\u{20}\u{78FB}\u{20}", + 0xF965 => "\u{20}\u{4FBF}\u{20}", + 0xF966 => "\u{20}\u{5FA9}\u{20}", + 0xF967 => "\u{20}\u{4E0D}\u{20}", + 0xF968 => "\u{20}\u{6CCC}\u{20}", + 0xF969 => "\u{20}\u{6578}\u{20}", + 0xF96A => "\u{20}\u{7D22}\u{20}", + 0xF96B => "\u{20}\u{53C3}\u{20}", + 0xF96C => "\u{20}\u{585E}\u{20}", + 0xF96D => "\u{20}\u{7701}\u{20}", + 0xF96E => "\u{20}\u{8449}\u{20}", + 0xF96F => "\u{20}\u{8AAA}\u{20}", + 0xF970 => "\u{20}\u{6BBA}\u{20}", + 0xF971 => "\u{20}\u{8FB0}\u{20}", + 0xF972 => "\u{20}\u{6C88}\u{20}", + 0xF973 => "\u{20}\u{62FE}\u{20}", + 0xF974 => "\u{20}\u{82E5}\u{20}", + 0xF975 => "\u{20}\u{63A0}\u{20}", + 0xF976 => "\u{20}\u{7565}\u{20}", + 0xF977 => "\u{20}\u{4EAE}\u{20}", + 0xF978 => "\u{20}\u{5169}\u{20}", + 0xF979 => "\u{20}\u{51C9}\u{20}", + 0xF97A => "\u{20}\u{6881}\u{20}", + 0xF97B => "\u{20}\u{7CE7}\u{20}", + 0xF97C => "\u{20}\u{826F}\u{20}", + 0xF97D => "\u{20}\u{8AD2}\u{20}", + 0xF97E => "\u{20}\u{91CF}\u{20}", + 0xF97F => "\u{20}\u{52F5}\u{20}", + 0xF980 => "\u{20}\u{5442}\u{20}", + 0xF981 => "\u{20}\u{5973}\u{20}", + 0xF982 => "\u{20}\u{5EEC}\u{20}", + 0xF983 => "\u{20}\u{65C5}\u{20}", + 0xF984 => "\u{20}\u{6FFE}\u{20}", + 0xF985 => "\u{20}\u{792A}\u{20}", + 0xF986 => "\u{20}\u{95AD}\u{20}", + 0xF987 => "\u{20}\u{9A6A}\u{20}", + 0xF988 => "\u{20}\u{9E97}\u{20}", + 0xF989 => "\u{20}\u{9ECE}\u{20}", + 0xF98A => "\u{20}\u{529B}\u{20}", + 0xF98B => "\u{20}\u{66C6}\u{20}", + 0xF98C => "\u{20}\u{6B77}\u{20}", + 0xF98D => "\u{20}\u{8F62}\u{20}", + 0xF98E => "\u{20}\u{5E74}\u{20}", + 0xF98F => "\u{20}\u{6190}\u{20}", + 0xF990 => "\u{20}\u{6200}\u{20}", + 0xF991 => "\u{20}\u{649A}\u{20}", + 0xF992 => "\u{20}\u{6F23}\u{20}", + 0xF993 => "\u{20}\u{7149}\u{20}", + 0xF994 => "\u{20}\u{7489}\u{20}", + 0xF995 => "\u{20}\u{79CA}\u{20}", + 0xF996 => "\u{20}\u{7DF4}\u{20}", + 0xF997 => "\u{20}\u{806F}\u{20}", + 0xF998 => "\u{20}\u{8F26}\u{20}", + 0xF999 => "\u{20}\u{84EE}\u{20}", + 0xF99A => "\u{20}\u{9023}\u{20}", + 0xF99B => "\u{20}\u{934A}\u{20}", + 0xF99C => "\u{20}\u{5217}\u{20}", + 0xF99D => "\u{20}\u{52A3}\u{20}", + 0xF99E => "\u{20}\u{54BD}\u{20}", + 0xF99F => "\u{20}\u{70C8}\u{20}", + 0xF9A0 => "\u{20}\u{88C2}\u{20}", + 0xF9A1 => "\u{20}\u{8AAA}\u{20}", + 0xF9A2 => "\u{20}\u{5EC9}\u{20}", + 0xF9A3 => "\u{20}\u{5FF5}\u{20}", + 0xF9A4 => "\u{20}\u{637B}\u{20}", + 0xF9A5 => "\u{20}\u{6BAE}\u{20}", + 0xF9A6 => "\u{20}\u{7C3E}\u{20}", + 0xF9A7 => "\u{20}\u{7375}\u{20}", + 0xF9A8 => "\u{20}\u{4EE4}\u{20}", + 0xF9A9 => "\u{20}\u{56F9}\u{20}", + 0xF9AA => "\u{20}\u{5BE7}\u{20}", + 0xF9AB => "\u{20}\u{5DBA}\u{20}", + 0xF9AC => "\u{20}\u{601C}\u{20}", + 0xF9AD => "\u{20}\u{73B2}\u{20}", + 0xF9AE => "\u{20}\u{7469}\u{20}", + 0xF9AF => "\u{20}\u{7F9A}\u{20}", + 0xF9B0 => "\u{20}\u{8046}\u{20}", + 0xF9B1 => "\u{20}\u{9234}\u{20}", + 0xF9B2 => "\u{20}\u{96F6}\u{20}", + 0xF9B3 => "\u{20}\u{9748}\u{20}", + 0xF9B4 => "\u{20}\u{9818}\u{20}", + 0xF9B5 => "\u{20}\u{4F8B}\u{20}", + 0xF9B6 => "\u{20}\u{79AE}\u{20}", + 0xF9B7 => "\u{20}\u{91B4}\u{20}", + 0xF9B8 => "\u{20}\u{96B8}\u{20}", + 0xF9B9 => "\u{20}\u{60E1}\u{20}", + 0xF9BA => "\u{20}\u{4E86}\u{20}", + 0xF9BB => "\u{20}\u{50DA}\u{20}", + 0xF9BC => "\u{20}\u{5BEE}\u{20}", + 0xF9BD => "\u{20}\u{5C3F}\u{20}", + 0xF9BE => "\u{20}\u{6599}\u{20}", + 0xF9BF => "\u{20}\u{6A02}\u{20}", + 0xF9C0 => "\u{20}\u{71CE}\u{20}", + 0xF9C1 => "\u{20}\u{7642}\u{20}", + 0xF9C2 => "\u{20}\u{84FC}\u{20}", + 0xF9C3 => "\u{20}\u{907C}\u{20}", + 0xF9C4 => "\u{20}\u{9F8D}\u{20}", + 0xF9C5 => "\u{20}\u{6688}\u{20}", + 0xF9C6 => "\u{20}\u{962E}\u{20}", + 0xF9C7 => "\u{20}\u{5289}\u{20}", + 0xF9C8 => "\u{20}\u{677B}\u{20}", + 0xF9C9 => "\u{20}\u{67F3}\u{20}", + 0xF9CA => "\u{20}\u{6D41}\u{20}", + 0xF9CB => "\u{20}\u{6E9C}\u{20}", + 0xF9CC => "\u{20}\u{7409}\u{20}", + 0xF9CD => "\u{20}\u{7559}\u{20}", + 0xF9CE => "\u{20}\u{786B}\u{20}", + 0xF9CF => "\u{20}\u{7D10}\u{20}", + 0xF9D0 => "\u{20}\u{985E}\u{20}", + 0xF9D1 => "\u{20}\u{516D}\u{20}", + 0xF9D2 => "\u{20}\u{622E}\u{20}", + 0xF9D3 => "\u{20}\u{9678}\u{20}", + 0xF9D4 => "\u{20}\u{502B}\u{20}", + 0xF9D5 => "\u{20}\u{5D19}\u{20}", + 0xF9D6 => "\u{20}\u{6DEA}\u{20}", + 0xF9D7 => "\u{20}\u{8F2A}\u{20}", + 0xF9D8 => "\u{20}\u{5F8B}\u{20}", + 0xF9D9 => "\u{20}\u{6144}\u{20}", + 0xF9DA => "\u{20}\u{6817}\u{20}", + 0xF9DB => "\u{20}\u{7387}\u{20}", + 0xF9DC => "\u{20}\u{9686}\u{20}", + 0xF9DD => "\u{20}\u{5229}\u{20}", + 0xF9DE => "\u{20}\u{540F}\u{20}", + 0xF9DF => "\u{20}\u{5C65}\u{20}", + 0xF9E0 => "\u{20}\u{6613}\u{20}", + 0xF9E1 => "\u{20}\u{674E}\u{20}", + 0xF9E2 => "\u{20}\u{68A8}\u{20}", + 0xF9E3 => "\u{20}\u{6CE5}\u{20}", + 0xF9E4 => "\u{20}\u{7406}\u{20}", + 0xF9E5 => "\u{20}\u{75E2}\u{20}", + 0xF9E6 => "\u{20}\u{7F79}\u{20}", + 0xF9E7 => "\u{20}\u{88CF}\u{20}", + 0xF9E8 => "\u{20}\u{88E1}\u{20}", + 0xF9E9 => "\u{20}\u{91CC}\u{20}", + 0xF9EA => "\u{20}\u{96E2}\u{20}", + 0xF9EB => "\u{20}\u{533F}\u{20}", + 0xF9EC => "\u{20}\u{6EBA}\u{20}", + 0xF9ED => "\u{20}\u{541D}\u{20}", + 0xF9EE => "\u{20}\u{71D0}\u{20}", + 0xF9EF => "\u{20}\u{7498}\u{20}", + 0xF9F0 => "\u{20}\u{85FA}\u{20}", + 0xF9F1 => "\u{20}\u{96A3}\u{20}", + 0xF9F2 => "\u{20}\u{9C57}\u{20}", + 0xF9F3 => "\u{20}\u{9E9F}\u{20}", + 0xF9F4 => "\u{20}\u{6797}\u{20}", + 0xF9F5 => "\u{20}\u{6DCB}\u{20}", + 0xF9F6 => "\u{20}\u{81E8}\u{20}", + 0xF9F7 => "\u{20}\u{7ACB}\u{20}", + 0xF9F8 => "\u{20}\u{7B20}\u{20}", + 0xF9F9 => "\u{20}\u{7C92}\u{20}", + 0xF9FA => "\u{20}\u{72C0}\u{20}", + 0xF9FB => "\u{20}\u{7099}\u{20}", + 0xF9FC => "\u{20}\u{8B58}\u{20}", + 0xF9FD => "\u{20}\u{4EC0}\u{20}", + 0xF9FE => "\u{20}\u{8336}\u{20}", + 0xF9FF => "\u{20}\u{523A}\u{20}", + 0xFA00 => "\u{20}\u{5207}\u{20}", + 0xFA01 => "\u{20}\u{5EA6}\u{20}", + 0xFA02 => "\u{20}\u{62D3}\u{20}", + 0xFA03 => "\u{20}\u{7CD6}\u{20}", + 0xFA04 => "\u{20}\u{5B85}\u{20}", + 0xFA05 => "\u{20}\u{6D1E}\u{20}", + 0xFA06 => "\u{20}\u{66B4}\u{20}", + 0xFA07 => "\u{20}\u{8F3B}\u{20}", + 0xFA08 => "\u{20}\u{884C}\u{20}", + 0xFA09 => "\u{20}\u{964D}\u{20}", + 0xFA0A => "\u{20}\u{898B}\u{20}", + 0xFA0B => "\u{20}\u{5ED3}\u{20}", + 0xFA0C => "\u{20}\u{5140}\u{20}", + 0xFA0D => "\u{20}\u{55C0}\u{20}", + 0xFA10 => "\u{20}\u{585A}\u{20}", + 0xFA12 => "\u{20}\u{6674}\u{20}", + 0xFA15 => "\u{20}\u{51DE}\u{20}", + 0xFA16 => "\u{20}\u{732A}\u{20}", + 0xFA17 => "\u{20}\u{76CA}\u{20}", + 0xFA18 => "\u{20}\u{793C}\u{20}", + 0xFA19 => "\u{20}\u{795E}\u{20}", + 0xFA1A => "\u{20}\u{7965}\u{20}", + 0xFA1B => "\u{20}\u{798F}\u{20}", + 0xFA1C => "\u{20}\u{9756}\u{20}", + 0xFA1D => "\u{20}\u{7CBE}\u{20}", + 0xFA1E => "\u{20}\u{7FBD}\u{20}", + 0xFA20 => "\u{20}\u{8612}\u{20}", + 0xFA22 => "\u{20}\u{8AF8}\u{20}", + 0xFA25 => "\u{20}\u{9038}\u{20}", + 0xFA26 => "\u{20}\u{90FD}\u{20}", + 0xFA2A => "\u{20}\u{98EF}\u{20}", + 0xFA2B => "\u{20}\u{98FC}\u{20}", + 0xFA2C => "\u{20}\u{9928}\u{20}", + 0xFA2D => "\u{20}\u{9DB4}\u{20}", + 0xFA2E => "\u{20}\u{90DE}\u{20}", + 0xFA2F => "\u{20}\u{96B7}\u{20}", + 0xFA30 => "\u{20}\u{4FAE}\u{20}", + 0xFA31 => "\u{20}\u{50E7}\u{20}", + 0xFA32 => "\u{20}\u{514D}\u{20}", + 0xFA33 => "\u{20}\u{52C9}\u{20}", + 0xFA34 => "\u{20}\u{52E4}\u{20}", + 0xFA35 => "\u{20}\u{5351}\u{20}", + 0xFA36 => "\u{20}\u{559D}\u{20}", + 0xFA37 => "\u{20}\u{5606}\u{20}", + 0xFA38 => "\u{20}\u{5668}\u{20}", + 0xFA39 => "\u{20}\u{5840}\u{20}", + 0xFA3A => "\u{20}\u{58A8}\u{20}", + 0xFA3B => "\u{20}\u{5C64}\u{20}", + 0xFA3C => "\u{20}\u{5C6E}\u{20}", + 0xFA3D => "\u{20}\u{6094}\u{20}", + 0xFA3E => "\u{20}\u{6168}\u{20}", + 0xFA3F => "\u{20}\u{618E}\u{20}", + 0xFA40 => "\u{20}\u{61F2}\u{20}", + 0xFA41 => "\u{20}\u{654F}\u{20}", + 0xFA42 => "\u{20}\u{65E2}\u{20}", + 0xFA43 => "\u{20}\u{6691}\u{20}", + 0xFA44 => "\u{20}\u{6885}\u{20}", + 0xFA45 => "\u{20}\u{6D77}\u{20}", + 0xFA46 => "\u{20}\u{6E1A}\u{20}", + 0xFA47 => "\u{20}\u{6F22}\u{20}", + 0xFA48 => "\u{20}\u{716E}\u{20}", + 0xFA49 => "\u{20}\u{722B}\u{20}", + 0xFA4A => "\u{20}\u{7422}\u{20}", + 0xFA4B => "\u{20}\u{7891}\u{20}", + 0xFA4C => "\u{20}\u{793E}\u{20}", + 0xFA4D => "\u{20}\u{7949}\u{20}", + 0xFA4E => "\u{20}\u{7948}\u{20}", + 0xFA4F => "\u{20}\u{7950}\u{20}", + 0xFA50 => "\u{20}\u{7956}\u{20}", + 0xFA51 => "\u{20}\u{795D}\u{20}", + 0xFA52 => "\u{20}\u{798D}\u{20}", + 0xFA53 => "\u{20}\u{798E}\u{20}", + 0xFA54 => "\u{20}\u{7A40}\u{20}", + 0xFA55 => "\u{20}\u{7A81}\u{20}", + 0xFA56 => "\u{20}\u{7BC0}\u{20}", + 0xFA57 => "\u{20}\u{7DF4}\u{20}", + 0xFA58 => "\u{20}\u{7E09}\u{20}", + 0xFA59 => "\u{20}\u{7E41}\u{20}", + 0xFA5A => "\u{20}\u{7F72}\u{20}", + 0xFA5B => "\u{20}\u{8005}\u{20}", + 0xFA5C => "\u{20}\u{81ED}\u{20}", + 0xFA5D => "\u{20}\u{8279}\u{20}", + 0xFA5E => "\u{20}\u{8279}\u{20}", + 0xFA5F => "\u{20}\u{8457}\u{20}", + 0xFA60 => "\u{20}\u{8910}\u{20}", + 0xFA61 => "\u{20}\u{8996}\u{20}", + 0xFA62 => "\u{20}\u{8B01}\u{20}", + 0xFA63 => "\u{20}\u{8B39}\u{20}", + 0xFA64 => "\u{20}\u{8CD3}\u{20}", + 0xFA65 => "\u{20}\u{8D08}\u{20}", + 0xFA66 => "\u{20}\u{8FB6}\u{20}", + 0xFA67 => "\u{20}\u{9038}\u{20}", + 0xFA68 => "\u{20}\u{96E3}\u{20}", + 0xFA69 => "\u{20}\u{97FF}\u{20}", + 0xFA6A => "\u{20}\u{983B}\u{20}", + 0xFA6B => "\u{20}\u{6075}\u{20}", + 0xFA6C => "\u{20}\u{242EE}\u{20}", + 0xFA6D => "\u{20}\u{8218}\u{20}", + 0xFA70 => "\u{20}\u{4E26}\u{20}", + 0xFA71 => "\u{20}\u{51B5}\u{20}", + 0xFA72 => "\u{20}\u{5168}\u{20}", + 0xFA73 => "\u{20}\u{4F80}\u{20}", + 0xFA74 => "\u{20}\u{5145}\u{20}", + 0xFA75 => "\u{20}\u{5180}\u{20}", + 0xFA76 => "\u{20}\u{52C7}\u{20}", + 0xFA77 => "\u{20}\u{52FA}\u{20}", + 0xFA78 => "\u{20}\u{559D}\u{20}", + 0xFA79 => "\u{20}\u{5555}\u{20}", + 0xFA7A => "\u{20}\u{5599}\u{20}", + 0xFA7B => "\u{20}\u{55E2}\u{20}", + 0xFA7C => "\u{20}\u{585A}\u{20}", + 0xFA7D => "\u{20}\u{58B3}\u{20}", + 0xFA7E => "\u{20}\u{5944}\u{20}", + 0xFA7F => "\u{20}\u{5954}\u{20}", + 0xFA80 => "\u{20}\u{5A62}\u{20}", + 0xFA81 => "\u{20}\u{5B28}\u{20}", + 0xFA82 => "\u{20}\u{5ED2}\u{20}", + 0xFA83 => "\u{20}\u{5ED9}\u{20}", + 0xFA84 => "\u{20}\u{5F69}\u{20}", + 0xFA85 => "\u{20}\u{5FAD}\u{20}", + 0xFA86 => "\u{20}\u{60D8}\u{20}", + 0xFA87 => "\u{20}\u{614E}\u{20}", + 0xFA88 => "\u{20}\u{6108}\u{20}", + 0xFA89 => "\u{20}\u{618E}\u{20}", + 0xFA8A => "\u{20}\u{6160}\u{20}", + 0xFA8B => "\u{20}\u{61F2}\u{20}", + 0xFA8C => "\u{20}\u{6234}\u{20}", + 0xFA8D => "\u{20}\u{63C4}\u{20}", + 0xFA8E => "\u{20}\u{641C}\u{20}", + 0xFA8F => "\u{20}\u{6452}\u{20}", + 0xFA90 => "\u{20}\u{6556}\u{20}", + 0xFA91 => "\u{20}\u{6674}\u{20}", + 0xFA92 => "\u{20}\u{6717}\u{20}", + 0xFA93 => "\u{20}\u{671B}\u{20}", + 0xFA94 => "\u{20}\u{6756}\u{20}", + 0xFA95 => "\u{20}\u{6B79}\u{20}", + 0xFA96 => "\u{20}\u{6BBA}\u{20}", + 0xFA97 => "\u{20}\u{6D41}\u{20}", + 0xFA98 => "\u{20}\u{6EDB}\u{20}", + 0xFA99 => "\u{20}\u{6ECB}\u{20}", + 0xFA9A => "\u{20}\u{6F22}\u{20}", + 0xFA9B => "\u{20}\u{701E}\u{20}", + 0xFA9C => "\u{20}\u{716E}\u{20}", + 0xFA9D => "\u{20}\u{77A7}\u{20}", + 0xFA9E => "\u{20}\u{7235}\u{20}", + 0xFA9F => "\u{20}\u{72AF}\u{20}", + 0xFAA0 => "\u{20}\u{732A}\u{20}", + 0xFAA1 => "\u{20}\u{7471}\u{20}", + 0xFAA2 => "\u{20}\u{7506}\u{20}", + 0xFAA3 => "\u{20}\u{753B}\u{20}", + 0xFAA4 => "\u{20}\u{761D}\u{20}", + 0xFAA5 => "\u{20}\u{761F}\u{20}", + 0xFAA6 => "\u{20}\u{76CA}\u{20}", + 0xFAA7 => "\u{20}\u{76DB}\u{20}", + 0xFAA8 => "\u{20}\u{76F4}\u{20}", + 0xFAA9 => "\u{20}\u{774A}\u{20}", + 0xFAAA => "\u{20}\u{7740}\u{20}", + 0xFAAB => "\u{20}\u{78CC}\u{20}", + 0xFAAC => "\u{20}\u{7AB1}\u{20}", + 0xFAAD => "\u{20}\u{7BC0}\u{20}", + 0xFAAE => "\u{20}\u{7C7B}\u{20}", + 0xFAAF => "\u{20}\u{7D5B}\u{20}", + 0xFAB0 => "\u{20}\u{7DF4}\u{20}", + 0xFAB1 => "\u{20}\u{7F3E}\u{20}", + 0xFAB2 => "\u{20}\u{8005}\u{20}", + 0xFAB3 => "\u{20}\u{8352}\u{20}", + 0xFAB4 => "\u{20}\u{83EF}\u{20}", + 0xFAB5 => "\u{20}\u{8779}\u{20}", + 0xFAB6 => "\u{20}\u{8941}\u{20}", + 0xFAB7 => "\u{20}\u{8986}\u{20}", + 0xFAB8 => "\u{20}\u{8996}\u{20}", + 0xFAB9 => "\u{20}\u{8ABF}\u{20}", + 0xFABA => "\u{20}\u{8AF8}\u{20}", + 0xFABB => "\u{20}\u{8ACB}\u{20}", + 0xFABC => "\u{20}\u{8B01}\u{20}", + 0xFABD => "\u{20}\u{8AFE}\u{20}", + 0xFABE => "\u{20}\u{8AED}\u{20}", + 0xFABF => "\u{20}\u{8B39}\u{20}", + 0xFAC0 => "\u{20}\u{8B8A}\u{20}", + 0xFAC1 => "\u{20}\u{8D08}\u{20}", + 0xFAC2 => "\u{20}\u{8F38}\u{20}", + 0xFAC3 => "\u{20}\u{9072}\u{20}", + 0xFAC4 => "\u{20}\u{9199}\u{20}", + 0xFAC5 => "\u{20}\u{9276}\u{20}", + 0xFAC6 => "\u{20}\u{967C}\u{20}", + 0xFAC7 => "\u{20}\u{96E3}\u{20}", + 0xFAC8 => "\u{20}\u{9756}\u{20}", + 0xFAC9 => "\u{20}\u{97DB}\u{20}", + 0xFACA => "\u{20}\u{97FF}\u{20}", + 0xFACB => "\u{20}\u{980B}\u{20}", + 0xFACC => "\u{20}\u{983B}\u{20}", + 0xFACD => "\u{20}\u{9B12}\u{20}", + 0xFACE => "\u{20}\u{9F9C}\u{20}", + 0xFACF => "\u{20}\u{2284A}\u{20}", + 0xFAD0 => "\u{20}\u{22844}\u{20}", + 0xFAD1 => "\u{20}\u{233D5}\u{20}", + 0xFAD2 => "\u{20}\u{3B9D}\u{20}", + 0xFAD3 => "\u{20}\u{4018}\u{20}", + 0xFAD4 => "\u{20}\u{4039}\u{20}", + 0xFAD5 => "\u{20}\u{25249}\u{20}", + 0xFAD6 => "\u{20}\u{25CD0}\u{20}", + 0xFAD7 => "\u{20}\u{27ED3}\u{20}", + 0xFAD8 => "\u{20}\u{9F43}\u{20}", + 0xFAD9 => "\u{20}\u{9F8E}\u{20}", + 0xFB1D => "\u{5D9}", + 0xFB1F => "\u{5F2}", + 0xFB2A => "\u{5E9}", + 0xFB2B => "\u{5E9}", + 0xFB2C => "\u{5E9}", + 0xFB2D => "\u{5E9}", + 0xFB2E => "\u{5D0}", + 0xFB2F => "\u{5D0}", + 0xFB30 => "\u{5D0}", + 0xFB31 => "\u{5D1}", + 0xFB32 => "\u{5D2}", + 0xFB33 => "\u{5D3}", + 0xFB34 => "\u{5D4}", + 0xFB35 => "\u{5D5}", + 0xFB36 => "\u{5D6}", + 0xFB38 => "\u{5D8}", + 0xFB39 => "\u{5D9}", + 0xFB3A => "\u{5DA}", + 0xFB3B => "\u{5DB}", + 0xFB3C => "\u{5DC}", + 0xFB3E => "\u{5DE}", + 0xFB40 => "\u{5E0}", + 0xFB41 => "\u{5E1}", + 0xFB43 => "\u{5E3}", + 0xFB44 => "\u{5E4}", + 0xFB46 => "\u{5E6}", + 0xFB47 => "\u{5E7}", + 0xFB48 => "\u{5E8}", + 0xFB49 => "\u{5E9}", + 0xFB4A => "\u{5EA}", + 0xFB4B => "\u{5D5}", + 0xFB4C => "\u{5D1}", + 0xFB4D => "\u{5DB}", + 0xFB4E => "\u{5E4}", + 0xFF21 => "\u{FF41}", + 0xFF22 => "\u{FF42}", + 0xFF23 => "\u{FF43}", + 0xFF24 => "\u{FF44}", + 0xFF25 => "\u{FF45}", + 0xFF26 => "\u{FF46}", + 0xFF27 => "\u{FF47}", + 0xFF28 => "\u{FF48}", + 0xFF29 => "\u{FF49}", + 0xFF2A => "\u{FF4A}", + 0xFF2B => "\u{FF4B}", + 0xFF2C => "\u{FF4C}", + 0xFF2D => "\u{FF4D}", + 0xFF2E => "\u{FF4E}", + 0xFF2F => "\u{FF4F}", + 0xFF30 => "\u{FF50}", + 0xFF31 => "\u{FF51}", + 0xFF32 => "\u{FF52}", + 0xFF33 => "\u{FF53}", + 0xFF34 => "\u{FF54}", + 0xFF35 => "\u{FF55}", + 0xFF36 => "\u{FF56}", + 0xFF37 => "\u{FF57}", + 0xFF38 => "\u{FF58}", + 0xFF39 => "\u{FF59}", + 0xFF3A => "\u{FF5A}", + 0x10400 => "\u{10428}", + 0x10401 => "\u{10429}", + 0x10402 => "\u{1042A}", + 0x10403 => "\u{1042B}", + 0x10404 => "\u{1042C}", + 0x10405 => "\u{1042D}", + 0x10406 => "\u{1042E}", + 0x10407 => "\u{1042F}", + 0x10408 => "\u{10430}", + 0x10409 => "\u{10431}", + 0x1040A => "\u{10432}", + 0x1040B => "\u{10433}", + 0x1040C => "\u{10434}", + 0x1040D => "\u{10435}", + 0x1040E => "\u{10436}", + 0x1040F => "\u{10437}", + 0x10410 => "\u{10438}", + 0x10411 => "\u{10439}", + 0x10412 => "\u{1043A}", + 0x10413 => "\u{1043B}", + 0x10414 => "\u{1043C}", + 0x10415 => "\u{1043D}", + 0x10416 => "\u{1043E}", + 0x10417 => "\u{1043F}", + 0x10418 => "\u{10440}", + 0x10419 => "\u{10441}", + 0x1041A => "\u{10442}", + 0x1041B => "\u{10443}", + 0x1041C => "\u{10444}", + 0x1041D => "\u{10445}", + 0x1041E => "\u{10446}", + 0x1041F => "\u{10447}", + 0x10420 => "\u{10448}", + 0x10421 => "\u{10449}", + 0x10422 => "\u{1044A}", + 0x10423 => "\u{1044B}", + 0x10424 => "\u{1044C}", + 0x10425 => "\u{1044D}", + 0x10426 => "\u{1044E}", + 0x10427 => "\u{1044F}", + 0x104B0 => "\u{104D8}", + 0x104B1 => "\u{104D9}", + 0x104B2 => "\u{104DA}", + 0x104B3 => "\u{104DB}", + 0x104B4 => "\u{104DC}", + 0x104B5 => "\u{104DD}", + 0x104B6 => "\u{104DE}", + 0x104B7 => "\u{104DF}", + 0x104B8 => "\u{104E0}", + 0x104B9 => "\u{104E1}", + 0x104BA => "\u{104E2}", + 0x104BB => "\u{104E3}", + 0x104BC => "\u{104E4}", + 0x104BD => "\u{104E5}", + 0x104BE => "\u{104E6}", + 0x104BF => "\u{104E7}", + 0x104C0 => "\u{104E8}", + 0x104C1 => "\u{104E9}", + 0x104C2 => "\u{104EA}", + 0x104C3 => "\u{104EB}", + 0x104C4 => "\u{104EC}", + 0x104C5 => "\u{104ED}", + 0x104C6 => "\u{104EE}", + 0x104C7 => "\u{104EF}", + 0x104C8 => "\u{104F0}", + 0x104C9 => "\u{104F1}", + 0x104CA => "\u{104F2}", + 0x104CB => "\u{104F3}", + 0x104CC => "\u{104F4}", + 0x104CD => "\u{104F5}", + 0x104CE => "\u{104F6}", + 0x104CF => "\u{104F7}", + 0x104D0 => "\u{104F8}", + 0x104D1 => "\u{104F9}", + 0x104D2 => "\u{104FA}", + 0x104D3 => "\u{104FB}", + 0x10570 => "\u{10597}", + 0x10571 => "\u{10598}", + 0x10572 => "\u{10599}", + 0x10573 => "\u{1059A}", + 0x10574 => "\u{1059B}", + 0x10575 => "\u{1059C}", + 0x10576 => "\u{1059D}", + 0x10577 => "\u{1059E}", + 0x10578 => "\u{1059F}", + 0x10579 => "\u{105A0}", + 0x1057A => "\u{105A1}", + 0x1057C => "\u{105A3}", + 0x1057D => "\u{105A4}", + 0x1057E => "\u{105A5}", + 0x1057F => "\u{105A6}", + 0x10580 => "\u{105A7}", + 0x10581 => "\u{105A8}", + 0x10582 => "\u{105A9}", + 0x10583 => "\u{105AA}", + 0x10584 => "\u{105AB}", + 0x10585 => "\u{105AC}", + 0x10586 => "\u{105AD}", + 0x10587 => "\u{105AE}", + 0x10588 => "\u{105AF}", + 0x10589 => "\u{105B0}", + 0x1058A => "\u{105B1}", + 0x1058C => "\u{105B3}", + 0x1058D => "\u{105B4}", + 0x1058E => "\u{105B5}", + 0x1058F => "\u{105B6}", + 0x10590 => "\u{105B7}", + 0x10591 => "\u{105B8}", + 0x10592 => "\u{105B9}", + 0x10594 => "\u{105BB}", + 0x10595 => "\u{105BC}", + 0x10C80 => "\u{10CC0}", + 0x10C81 => "\u{10CC1}", + 0x10C82 => "\u{10CC2}", + 0x10C83 => "\u{10CC3}", + 0x10C84 => "\u{10CC4}", + 0x10C85 => "\u{10CC5}", + 0x10C86 => "\u{10CC6}", + 0x10C87 => "\u{10CC7}", + 0x10C88 => "\u{10CC8}", + 0x10C89 => "\u{10CC9}", + 0x10C8A => "\u{10CCA}", + 0x10C8B => "\u{10CCB}", + 0x10C8C => "\u{10CCC}", + 0x10C8D => "\u{10CCD}", + 0x10C8E => "\u{10CCE}", + 0x10C8F => "\u{10CCF}", + 0x10C90 => "\u{10CD0}", + 0x10C91 => "\u{10CD1}", + 0x10C92 => "\u{10CD2}", + 0x10C93 => "\u{10CD3}", + 0x10C94 => "\u{10CD4}", + 0x10C95 => "\u{10CD5}", + 0x10C96 => "\u{10CD6}", + 0x10C97 => "\u{10CD7}", + 0x10C98 => "\u{10CD8}", + 0x10C99 => "\u{10CD9}", + 0x10C9A => "\u{10CDA}", + 0x10C9B => "\u{10CDB}", + 0x10C9C => "\u{10CDC}", + 0x10C9D => "\u{10CDD}", + 0x10C9E => "\u{10CDE}", + 0x10C9F => "\u{10CDF}", + 0x10CA0 => "\u{10CE0}", + 0x10CA1 => "\u{10CE1}", + 0x10CA2 => "\u{10CE2}", + 0x10CA3 => "\u{10CE3}", + 0x10CA4 => "\u{10CE4}", + 0x10CA5 => "\u{10CE5}", + 0x10CA6 => "\u{10CE6}", + 0x10CA7 => "\u{10CE7}", + 0x10CA8 => "\u{10CE8}", + 0x10CA9 => "\u{10CE9}", + 0x10CAA => "\u{10CEA}", + 0x10CAB => "\u{10CEB}", + 0x10CAC => "\u{10CEC}", + 0x10CAD => "\u{10CED}", + 0x10CAE => "\u{10CEE}", + 0x10CAF => "\u{10CEF}", + 0x10CB0 => "\u{10CF0}", + 0x10CB1 => "\u{10CF1}", + 0x10CB2 => "\u{10CF2}", + 0x10D50 => "\u{10D70}", + 0x10D51 => "\u{10D71}", + 0x10D52 => "\u{10D72}", + 0x10D53 => "\u{10D73}", + 0x10D54 => "\u{10D74}", + 0x10D55 => "\u{10D75}", + 0x10D56 => "\u{10D76}", + 0x10D57 => "\u{10D77}", + 0x10D58 => "\u{10D78}", + 0x10D59 => "\u{10D79}", + 0x10D5A => "\u{10D7A}", + 0x10D5B => "\u{10D7B}", + 0x10D5C => "\u{10D7C}", + 0x10D5D => "\u{10D7D}", + 0x10D5E => "\u{10D7E}", + 0x10D5F => "\u{10D7F}", + 0x10D60 => "\u{10D80}", + 0x10D61 => "\u{10D81}", + 0x10D62 => "\u{10D82}", + 0x10D63 => "\u{10D83}", + 0x10D64 => "\u{10D84}", + 0x10D65 => "\u{10D85}", + 0x1109A => "\u{11099}", + 0x1109C => "\u{1109B}", + 0x110AB => "\u{110A5}", + 0x1134B => "\u{11347}\u{1133E}", + 0x1134C => "\u{11347}\u{11357}", + 0x114BB => "\u{114B9}", + 0x114BC => "\u{114B9}\u{114B0}", + 0x114BE => "\u{114B9}\u{114BD}", + 0x115BA => "\u{115B8}\u{115AF}", + 0x115BB => "\u{115B9}\u{115AF}", + 0x118A0 => "\u{118C0}", + 0x118A1 => "\u{118C1}", + 0x118A2 => "\u{118C2}", + 0x118A3 => "\u{118C3}", + 0x118A4 => "\u{118C4}", + 0x118A5 => "\u{118C5}", + 0x118A6 => "\u{118C6}", + 0x118A7 => "\u{118C7}", + 0x118A8 => "\u{118C8}", + 0x118A9 => "\u{118C9}", + 0x118AA => "\u{118CA}", + 0x118AB => "\u{118CB}", + 0x118AC => "\u{118CC}", + 0x118AD => "\u{118CD}", + 0x118AE => "\u{118CE}", + 0x118AF => "\u{118CF}", + 0x118B0 => "\u{118D0}", + 0x118B1 => "\u{118D1}", + 0x118B2 => "\u{118D2}", + 0x118B3 => "\u{118D3}", + 0x118B4 => "\u{118D4}", + 0x118B5 => "\u{118D5}", + 0x118B6 => "\u{118D6}", + 0x118B7 => "\u{118D7}", + 0x118B8 => "\u{118D8}", + 0x118B9 => "\u{118D9}", + 0x118BA => "\u{118DA}", + 0x118BB => "\u{118DB}", + 0x118BC => "\u{118DC}", + 0x118BD => "\u{118DD}", + 0x118BE => "\u{118DE}", + 0x118BF => "\u{118DF}", + 0x16E40 => "\u{16E60}", + 0x16E41 => "\u{16E61}", + 0x16E42 => "\u{16E62}", + 0x16E43 => "\u{16E63}", + 0x16E44 => "\u{16E64}", + 0x16E45 => "\u{16E65}", + 0x16E46 => "\u{16E66}", + 0x16E47 => "\u{16E67}", + 0x16E48 => "\u{16E68}", + 0x16E49 => "\u{16E69}", + 0x16E4A => "\u{16E6A}", + 0x16E4B => "\u{16E6B}", + 0x16E4C => "\u{16E6C}", + 0x16E4D => "\u{16E6D}", + 0x16E4E => "\u{16E6E}", + 0x16E4F => "\u{16E6F}", + 0x16E50 => "\u{16E70}", + 0x16E51 => "\u{16E71}", + 0x16E52 => "\u{16E72}", + 0x16E53 => "\u{16E73}", + 0x16E54 => "\u{16E74}", + 0x16E55 => "\u{16E75}", + 0x16E56 => "\u{16E76}", + 0x16E57 => "\u{16E77}", + 0x16E58 => "\u{16E78}", + 0x16E59 => "\u{16E79}", + 0x16E5A => "\u{16E7A}", + 0x16E5B => "\u{16E7B}", + 0x16E5C => "\u{16E7C}", + 0x16E5D => "\u{16E7D}", + 0x16E5E => "\u{16E7E}", + 0x16E5F => "\u{16E7F}", + 0x16EA0 => "\u{16EBB}", + 0x16EA1 => "\u{16EBC}", + 0x16EA2 => "\u{16EBD}", + 0x16EA3 => "\u{16EBE}", + 0x16EA4 => "\u{16EBF}", + 0x16EA5 => "\u{16EC0}", + 0x16EA6 => "\u{16EC1}", + 0x16EA7 => "\u{16EC2}", + 0x16EA8 => "\u{16EC3}", + 0x16EA9 => "\u{16EC4}", + 0x16EAA => "\u{16EC5}", + 0x16EAB => "\u{16EC6}", + 0x16EAC => "\u{16EC7}", + 0x16EAD => "\u{16EC8}", + 0x16EAE => "\u{16EC9}", + 0x16EAF => "\u{16ECA}", + 0x16EB0 => "\u{16ECB}", + 0x16EB1 => "\u{16ECC}", + 0x16EB2 => "\u{16ECD}", + 0x16EB3 => "\u{16ECE}", + 0x16EB4 => "\u{16ECF}", + 0x16EB5 => "\u{16ED0}", + 0x16EB6 => "\u{16ED1}", + 0x16EB7 => "\u{16ED2}", + 0x16EB8 => "\u{16ED3}", + 0x1D15E => "\u{1D157}\u{1D165}", + 0x1D15F => "\u{1D158}\u{1D165}", + 0x1D160 => "\u{1D158}\u{1D165}\u{1D16E}", + 0x1D161 => "\u{1D158}\u{1D165}\u{1D16F}", + 0x1D162 => "\u{1D158}\u{1D165}\u{1D170}", + 0x1D163 => "\u{1D158}\u{1D165}\u{1D171}", + 0x1D164 => "\u{1D158}\u{1D165}\u{1D172}", + 0x1D1BB => "\u{1D1B9}\u{1D165}", + 0x1D1BC => "\u{1D1BA}\u{1D165}", + 0x1D1BD => "\u{1D1B9}\u{1D165}\u{1D16E}", + 0x1D1BE => "\u{1D1BA}\u{1D165}\u{1D16E}", + 0x1D1BF => "\u{1D1B9}\u{1D165}\u{1D16F}", + 0x1D1C0 => "\u{1D1BA}\u{1D165}\u{1D16F}", + 0x1E900 => "\u{1E922}", + 0x1E901 => "\u{1E923}", + 0x1E902 => "\u{1E924}", + 0x1E903 => "\u{1E925}", + 0x1E904 => "\u{1E926}", + 0x1E905 => "\u{1E927}", + 0x1E906 => "\u{1E928}", + 0x1E907 => "\u{1E929}", + 0x1E908 => "\u{1E92A}", + 0x1E909 => "\u{1E92B}", + 0x1E90A => "\u{1E92C}", + 0x1E90B => "\u{1E92D}", + 0x1E90C => "\u{1E92E}", + 0x1E90D => "\u{1E92F}", + 0x1E90E => "\u{1E930}", + 0x1E90F => "\u{1E931}", + 0x1E910 => "\u{1E932}", + 0x1E911 => "\u{1E933}", + 0x1E912 => "\u{1E934}", + 0x1E913 => "\u{1E935}", + 0x1E914 => "\u{1E936}", + 0x1E915 => "\u{1E937}", + 0x1E916 => "\u{1E938}", + 0x1E917 => "\u{1E939}", + 0x1E918 => "\u{1E93A}", + 0x1E919 => "\u{1E93B}", + 0x1E91A => "\u{1E93C}", + 0x1E91B => "\u{1E93D}", + 0x1E91C => "\u{1E93E}", + 0x1E91D => "\u{1E93F}", + 0x1E91E => "\u{1E940}", + 0x1E91F => "\u{1E941}", + 0x1E920 => "\u{1E942}", + 0x1E921 => "\u{1E943}", + 0x2F800 => "\u{20}\u{4E3D}\u{20}", + 0x2F801 => "\u{20}\u{4E38}\u{20}", + 0x2F802 => "\u{20}\u{4E41}\u{20}", + 0x2F803 => "\u{20}\u{20122}\u{20}", + 0x2F804 => "\u{20}\u{4F60}\u{20}", + 0x2F805 => "\u{20}\u{4FAE}\u{20}", + 0x2F806 => "\u{20}\u{4FBB}\u{20}", + 0x2F807 => "\u{20}\u{5002}\u{20}", + 0x2F808 => "\u{20}\u{507A}\u{20}", + 0x2F809 => "\u{20}\u{5099}\u{20}", + 0x2F80A => "\u{20}\u{50E7}\u{20}", + 0x2F80B => "\u{20}\u{50CF}\u{20}", + 0x2F80C => "\u{20}\u{349E}\u{20}", + 0x2F80D => "\u{20}\u{2063A}\u{20}", + 0x2F80E => "\u{20}\u{514D}\u{20}", + 0x2F80F => "\u{20}\u{5154}\u{20}", + 0x2F810 => "\u{20}\u{5164}\u{20}", + 0x2F811 => "\u{20}\u{5177}\u{20}", + 0x2F812 => "\u{20}\u{2051C}\u{20}", + 0x2F813 => "\u{20}\u{34B9}\u{20}", + 0x2F814 => "\u{20}\u{5167}\u{20}", + 0x2F815 => "\u{20}\u{518D}\u{20}", + 0x2F816 => "\u{20}\u{2054B}\u{20}", + 0x2F817 => "\u{20}\u{5197}\u{20}", + 0x2F818 => "\u{20}\u{51A4}\u{20}", + 0x2F819 => "\u{20}\u{4ECC}\u{20}", + 0x2F81A => "\u{20}\u{51AC}\u{20}", + 0x2F81B => "\u{20}\u{51B5}\u{20}", + 0x2F81C => "\u{20}\u{291DF}\u{20}", + 0x2F81D => "\u{20}\u{51F5}\u{20}", + 0x2F81E => "\u{20}\u{5203}\u{20}", + 0x2F81F => "\u{20}\u{34DF}\u{20}", + 0x2F820 => "\u{20}\u{523B}\u{20}", + 0x2F821 => "\u{20}\u{5246}\u{20}", + 0x2F822 => "\u{20}\u{5272}\u{20}", + 0x2F823 => "\u{20}\u{5277}\u{20}", + 0x2F824 => "\u{20}\u{3515}\u{20}", + 0x2F825 => "\u{20}\u{52C7}\u{20}", + 0x2F826 => "\u{20}\u{52C9}\u{20}", + 0x2F827 => "\u{20}\u{52E4}\u{20}", + 0x2F828 => "\u{20}\u{52FA}\u{20}", + 0x2F829 => "\u{20}\u{5305}\u{20}", + 0x2F82A => "\u{20}\u{5306}\u{20}", + 0x2F82B => "\u{20}\u{5317}\u{20}", + 0x2F82C => "\u{20}\u{5349}\u{20}", + 0x2F82D => "\u{20}\u{5351}\u{20}", + 0x2F82E => "\u{20}\u{535A}\u{20}", + 0x2F82F => "\u{20}\u{5373}\u{20}", + 0x2F830 => "\u{20}\u{537D}\u{20}", + 0x2F831 => "\u{20}\u{537F}\u{20}", + 0x2F832 => "\u{20}\u{537F}\u{20}", + 0x2F833 => "\u{20}\u{537F}\u{20}", + 0x2F834 => "\u{20}\u{20A2C}\u{20}", + 0x2F835 => "\u{20}\u{7070}\u{20}", + 0x2F836 => "\u{20}\u{53CA}\u{20}", + 0x2F837 => "\u{20}\u{53DF}\u{20}", + 0x2F838 => "\u{20}\u{20B63}\u{20}", + 0x2F839 => "\u{20}\u{53EB}\u{20}", + 0x2F83A => "\u{20}\u{53F1}\u{20}", + 0x2F83B => "\u{20}\u{5406}\u{20}", + 0x2F83C => "\u{20}\u{549E}\u{20}", + 0x2F83D => "\u{20}\u{5438}\u{20}", + 0x2F83E => "\u{20}\u{5448}\u{20}", + 0x2F83F => "\u{20}\u{5468}\u{20}", + 0x2F840 => "\u{20}\u{54A2}\u{20}", + 0x2F841 => "\u{20}\u{54F6}\u{20}", + 0x2F842 => "\u{20}\u{5510}\u{20}", + 0x2F843 => "\u{20}\u{5553}\u{20}", + 0x2F844 => "\u{20}\u{5563}\u{20}", + 0x2F845 => "\u{20}\u{5584}\u{20}", + 0x2F846 => "\u{20}\u{5584}\u{20}", + 0x2F847 => "\u{20}\u{5599}\u{20}", + 0x2F848 => "\u{20}\u{55AB}\u{20}", + 0x2F849 => "\u{20}\u{55B3}\u{20}", + 0x2F84A => "\u{20}\u{55C2}\u{20}", + 0x2F84B => "\u{20}\u{5716}\u{20}", + 0x2F84C => "\u{20}\u{5606}\u{20}", + 0x2F84D => "\u{20}\u{5717}\u{20}", + 0x2F84E => "\u{20}\u{5651}\u{20}", + 0x2F84F => "\u{20}\u{5674}\u{20}", + 0x2F850 => "\u{20}\u{5207}\u{20}", + 0x2F851 => "\u{20}\u{58EE}\u{20}", + 0x2F852 => "\u{20}\u{57CE}\u{20}", + 0x2F853 => "\u{20}\u{57F4}\u{20}", + 0x2F854 => "\u{20}\u{580D}\u{20}", + 0x2F855 => "\u{20}\u{578B}\u{20}", + 0x2F856 => "\u{20}\u{5832}\u{20}", + 0x2F857 => "\u{20}\u{5831}\u{20}", + 0x2F858 => "\u{20}\u{58AC}\u{20}", + 0x2F859 => "\u{20}\u{214E4}\u{20}", + 0x2F85A => "\u{20}\u{58F2}\u{20}", + 0x2F85B => "\u{20}\u{58F7}\u{20}", + 0x2F85C => "\u{20}\u{5906}\u{20}", + 0x2F85D => "\u{20}\u{591A}\u{20}", + 0x2F85E => "\u{20}\u{5922}\u{20}", + 0x2F85F => "\u{20}\u{5962}\u{20}", + 0x2F860 => "\u{20}\u{216A8}\u{20}", + 0x2F861 => "\u{20}\u{216EA}\u{20}", + 0x2F862 => "\u{20}\u{59EC}\u{20}", + 0x2F863 => "\u{20}\u{5A1B}\u{20}", + 0x2F864 => "\u{20}\u{5A27}\u{20}", + 0x2F865 => "\u{20}\u{59D8}\u{20}", + 0x2F866 => "\u{20}\u{5A66}\u{20}", + 0x2F867 => "\u{20}\u{36EE}\u{20}", + 0x2F868 => "\u{20}\u{36FC}\u{20}", + 0x2F869 => "\u{20}\u{5B08}\u{20}", + 0x2F86A => "\u{20}\u{5B3E}\u{20}", + 0x2F86B => "\u{20}\u{5B3E}\u{20}", + 0x2F86C => "\u{20}\u{219C8}\u{20}", + 0x2F86D => "\u{20}\u{5BC3}\u{20}", + 0x2F86E => "\u{20}\u{5BD8}\u{20}", + 0x2F86F => "\u{20}\u{5BE7}\u{20}", + 0x2F870 => "\u{20}\u{5BF3}\u{20}", + 0x2F871 => "\u{20}\u{21B18}\u{20}", + 0x2F872 => "\u{20}\u{5BFF}\u{20}", + 0x2F873 => "\u{20}\u{5C06}\u{20}", + 0x2F874 => "\u{20}\u{5F53}\u{20}", + 0x2F875 => "\u{20}\u{5C22}\u{20}", + 0x2F876 => "\u{20}\u{3781}\u{20}", + 0x2F877 => "\u{20}\u{5C60}\u{20}", + 0x2F878 => "\u{20}\u{5C6E}\u{20}", + 0x2F879 => "\u{20}\u{5CC0}\u{20}", + 0x2F87A => "\u{20}\u{5C8D}\u{20}", + 0x2F87B => "\u{20}\u{21DE4}\u{20}", + 0x2F87C => "\u{20}\u{5D43}\u{20}", + 0x2F87D => "\u{20}\u{21DE6}\u{20}", + 0x2F87E => "\u{20}\u{5D6E}\u{20}", + 0x2F87F => "\u{20}\u{5D6B}\u{20}", + 0x2F880 => "\u{20}\u{5D7C}\u{20}", + 0x2F881 => "\u{20}\u{5DE1}\u{20}", + 0x2F882 => "\u{20}\u{5DE2}\u{20}", + 0x2F883 => "\u{20}\u{382F}\u{20}", + 0x2F884 => "\u{20}\u{5DFD}\u{20}", + 0x2F885 => "\u{20}\u{5E28}\u{20}", + 0x2F886 => "\u{20}\u{5E3D}\u{20}", + 0x2F887 => "\u{20}\u{5E69}\u{20}", + 0x2F888 => "\u{20}\u{3862}\u{20}", + 0x2F889 => "\u{20}\u{22183}\u{20}", + 0x2F88A => "\u{20}\u{387C}\u{20}", + 0x2F88B => "\u{20}\u{5EB0}\u{20}", + 0x2F88C => "\u{20}\u{5EB3}\u{20}", + 0x2F88D => "\u{20}\u{5EB6}\u{20}", + 0x2F88E => "\u{20}\u{5ECA}\u{20}", + 0x2F88F => "\u{20}\u{2A392}\u{20}", + 0x2F890 => "\u{20}\u{5EFE}\u{20}", + 0x2F891 => "\u{20}\u{22331}\u{20}", + 0x2F892 => "\u{20}\u{22331}\u{20}", + 0x2F893 => "\u{20}\u{8201}\u{20}", + 0x2F894 => "\u{20}\u{5F22}\u{20}", + 0x2F895 => "\u{20}\u{5F22}\u{20}", + 0x2F896 => "\u{20}\u{38C7}\u{20}", + 0x2F897 => "\u{20}\u{232B8}\u{20}", + 0x2F898 => "\u{20}\u{261DA}\u{20}", + 0x2F899 => "\u{20}\u{5F62}\u{20}", + 0x2F89A => "\u{20}\u{5F6B}\u{20}", + 0x2F89B => "\u{20}\u{38E3}\u{20}", + 0x2F89C => "\u{20}\u{5F9A}\u{20}", + 0x2F89D => "\u{20}\u{5FCD}\u{20}", + 0x2F89E => "\u{20}\u{5FD7}\u{20}", + 0x2F89F => "\u{20}\u{5FF9}\u{20}", + 0x2F8A0 => "\u{20}\u{6081}\u{20}", + 0x2F8A1 => "\u{20}\u{393A}\u{20}", + 0x2F8A2 => "\u{20}\u{391C}\u{20}", + 0x2F8A3 => "\u{20}\u{6094}\u{20}", + 0x2F8A4 => "\u{20}\u{226D4}\u{20}", + 0x2F8A5 => "\u{20}\u{60C7}\u{20}", + 0x2F8A6 => "\u{20}\u{6148}\u{20}", + 0x2F8A7 => "\u{20}\u{614C}\u{20}", + 0x2F8A8 => "\u{20}\u{614E}\u{20}", + 0x2F8A9 => "\u{20}\u{614C}\u{20}", + 0x2F8AA => "\u{20}\u{617A}\u{20}", + 0x2F8AB => "\u{20}\u{618E}\u{20}", + 0x2F8AC => "\u{20}\u{61B2}\u{20}", + 0x2F8AD => "\u{20}\u{61A4}\u{20}", + 0x2F8AE => "\u{20}\u{61AF}\u{20}", + 0x2F8AF => "\u{20}\u{61DE}\u{20}", + 0x2F8B0 => "\u{20}\u{61F2}\u{20}", + 0x2F8B1 => "\u{20}\u{61F6}\u{20}", + 0x2F8B2 => "\u{20}\u{6210}\u{20}", + 0x2F8B3 => "\u{20}\u{621B}\u{20}", + 0x2F8B4 => "\u{20}\u{625D}\u{20}", + 0x2F8B5 => "\u{20}\u{62B1}\u{20}", + 0x2F8B6 => "\u{20}\u{62D4}\u{20}", + 0x2F8B7 => "\u{20}\u{6350}\u{20}", + 0x2F8B8 => "\u{20}\u{22B0C}\u{20}", + 0x2F8B9 => "\u{20}\u{633D}\u{20}", + 0x2F8BA => "\u{20}\u{62FC}\u{20}", + 0x2F8BB => "\u{20}\u{6368}\u{20}", + 0x2F8BC => "\u{20}\u{6383}\u{20}", + 0x2F8BD => "\u{20}\u{63E4}\u{20}", + 0x2F8BE => "\u{20}\u{22BF1}\u{20}", + 0x2F8BF => "\u{20}\u{6422}\u{20}", + 0x2F8C0 => "\u{20}\u{63C5}\u{20}", + 0x2F8C1 => "\u{20}\u{63A9}\u{20}", + 0x2F8C2 => "\u{20}\u{3A2E}\u{20}", + 0x2F8C3 => "\u{20}\u{6469}\u{20}", + 0x2F8C4 => "\u{20}\u{647E}\u{20}", + 0x2F8C5 => "\u{20}\u{649D}\u{20}", + 0x2F8C6 => "\u{20}\u{6477}\u{20}", + 0x2F8C7 => "\u{20}\u{3A6C}\u{20}", + 0x2F8C8 => "\u{20}\u{654F}\u{20}", + 0x2F8C9 => "\u{20}\u{656C}\u{20}", + 0x2F8CA => "\u{20}\u{2300A}\u{20}", + 0x2F8CB => "\u{20}\u{65E3}\u{20}", + 0x2F8CC => "\u{20}\u{66F8}\u{20}", + 0x2F8CD => "\u{20}\u{6649}\u{20}", + 0x2F8CE => "\u{20}\u{3B19}\u{20}", + 0x2F8CF => "\u{20}\u{6691}\u{20}", + 0x2F8D0 => "\u{20}\u{3B08}\u{20}", + 0x2F8D1 => "\u{20}\u{3AE4}\u{20}", + 0x2F8D2 => "\u{20}\u{5192}\u{20}", + 0x2F8D3 => "\u{20}\u{5195}\u{20}", + 0x2F8D4 => "\u{20}\u{6700}\u{20}", + 0x2F8D5 => "\u{20}\u{669C}\u{20}", + 0x2F8D6 => "\u{20}\u{80AD}\u{20}", + 0x2F8D7 => "\u{20}\u{43D9}\u{20}", + 0x2F8D8 => "\u{20}\u{6717}\u{20}", + 0x2F8D9 => "\u{20}\u{671B}\u{20}", + 0x2F8DA => "\u{20}\u{6721}\u{20}", + 0x2F8DB => "\u{20}\u{675E}\u{20}", + 0x2F8DC => "\u{20}\u{6753}\u{20}", + 0x2F8DD => "\u{20}\u{233C3}\u{20}", + 0x2F8DE => "\u{20}\u{3B49}\u{20}", + 0x2F8DF => "\u{20}\u{67FA}\u{20}", + 0x2F8E0 => "\u{20}\u{6785}\u{20}", + 0x2F8E1 => "\u{20}\u{6852}\u{20}", + 0x2F8E2 => "\u{20}\u{6885}\u{20}", + 0x2F8E3 => "\u{20}\u{2346D}\u{20}", + 0x2F8E4 => "\u{20}\u{688E}\u{20}", + 0x2F8E5 => "\u{20}\u{681F}\u{20}", + 0x2F8E6 => "\u{20}\u{6914}\u{20}", + 0x2F8E7 => "\u{20}\u{3B9D}\u{20}", + 0x2F8E8 => "\u{20}\u{6942}\u{20}", + 0x2F8E9 => "\u{20}\u{69A3}\u{20}", + 0x2F8EA => "\u{20}\u{69EA}\u{20}", + 0x2F8EB => "\u{20}\u{6AA8}\u{20}", + 0x2F8EC => "\u{20}\u{236A3}\u{20}", + 0x2F8ED => "\u{20}\u{6ADB}\u{20}", + 0x2F8EE => "\u{20}\u{3C18}\u{20}", + 0x2F8EF => "\u{20}\u{6B21}\u{20}", + 0x2F8F0 => "\u{20}\u{238A7}\u{20}", + 0x2F8F1 => "\u{20}\u{6B54}\u{20}", + 0x2F8F2 => "\u{20}\u{3C4E}\u{20}", + 0x2F8F3 => "\u{20}\u{6B72}\u{20}", + 0x2F8F4 => "\u{20}\u{6B9F}\u{20}", + 0x2F8F5 => "\u{20}\u{6BBA}\u{20}", + 0x2F8F6 => "\u{20}\u{6BBB}\u{20}", + 0x2F8F7 => "\u{20}\u{23A8D}\u{20}", + 0x2F8F8 => "\u{20}\u{21D0B}\u{20}", + 0x2F8F9 => "\u{20}\u{23AFA}\u{20}", + 0x2F8FA => "\u{20}\u{6C4E}\u{20}", + 0x2F8FB => "\u{20}\u{23CBC}\u{20}", + 0x2F8FC => "\u{20}\u{6CBF}\u{20}", + 0x2F8FD => "\u{20}\u{6CCD}\u{20}", + 0x2F8FE => "\u{20}\u{6C67}\u{20}", + 0x2F8FF => "\u{20}\u{6D16}\u{20}", + 0x2F900 => "\u{20}\u{6D3E}\u{20}", + 0x2F901 => "\u{20}\u{6D77}\u{20}", + 0x2F902 => "\u{20}\u{6D41}\u{20}", + 0x2F903 => "\u{20}\u{6D69}\u{20}", + 0x2F904 => "\u{20}\u{6D78}\u{20}", + 0x2F905 => "\u{20}\u{6D85}\u{20}", + 0x2F906 => "\u{20}\u{23D1E}\u{20}", + 0x2F907 => "\u{20}\u{6D34}\u{20}", + 0x2F908 => "\u{20}\u{6E2F}\u{20}", + 0x2F909 => "\u{20}\u{6E6E}\u{20}", + 0x2F90A => "\u{20}\u{3D33}\u{20}", + 0x2F90B => "\u{20}\u{6ECB}\u{20}", + 0x2F90C => "\u{20}\u{6EC7}\u{20}", + 0x2F90D => "\u{20}\u{23ED1}\u{20}", + 0x2F90E => "\u{20}\u{6DF9}\u{20}", + 0x2F90F => "\u{20}\u{6F6E}\u{20}", + 0x2F910 => "\u{20}\u{23F5E}\u{20}", + 0x2F911 => "\u{20}\u{23F8E}\u{20}", + 0x2F912 => "\u{20}\u{6FC6}\u{20}", + 0x2F913 => "\u{20}\u{7039}\u{20}", + 0x2F914 => "\u{20}\u{701E}\u{20}", + 0x2F915 => "\u{20}\u{701B}\u{20}", + 0x2F916 => "\u{20}\u{3D96}\u{20}", + 0x2F917 => "\u{20}\u{704A}\u{20}", + 0x2F918 => "\u{20}\u{707D}\u{20}", + 0x2F919 => "\u{20}\u{7077}\u{20}", + 0x2F91A => "\u{20}\u{70AD}\u{20}", + 0x2F91B => "\u{20}\u{20525}\u{20}", + 0x2F91C => "\u{20}\u{7145}\u{20}", + 0x2F91D => "\u{20}\u{24263}\u{20}", + 0x2F91E => "\u{20}\u{719C}\u{20}", + 0x2F91F => "\u{20}\u{243AB}\u{20}", + 0x2F920 => "\u{20}\u{7228}\u{20}", + 0x2F921 => "\u{20}\u{7235}\u{20}", + 0x2F922 => "\u{20}\u{7250}\u{20}", + 0x2F923 => "\u{20}\u{24608}\u{20}", + 0x2F924 => "\u{20}\u{7280}\u{20}", + 0x2F925 => "\u{20}\u{7295}\u{20}", + 0x2F926 => "\u{20}\u{24735}\u{20}", + 0x2F927 => "\u{20}\u{24814}\u{20}", + 0x2F928 => "\u{20}\u{737A}\u{20}", + 0x2F929 => "\u{20}\u{738B}\u{20}", + 0x2F92A => "\u{20}\u{3EAC}\u{20}", + 0x2F92B => "\u{20}\u{73A5}\u{20}", + 0x2F92C => "\u{20}\u{3EB8}\u{20}", + 0x2F92D => "\u{20}\u{3EB8}\u{20}", + 0x2F92E => "\u{20}\u{7447}\u{20}", + 0x2F92F => "\u{20}\u{745C}\u{20}", + 0x2F930 => "\u{20}\u{7471}\u{20}", + 0x2F931 => "\u{20}\u{7485}\u{20}", + 0x2F932 => "\u{20}\u{74CA}\u{20}", + 0x2F933 => "\u{20}\u{3F1B}\u{20}", + 0x2F934 => "\u{20}\u{7524}\u{20}", + 0x2F935 => "\u{20}\u{24C36}\u{20}", + 0x2F936 => "\u{20}\u{753E}\u{20}", + 0x2F937 => "\u{20}\u{24C92}\u{20}", + 0x2F938 => "\u{20}\u{7570}\u{20}", + 0x2F939 => "\u{20}\u{2219F}\u{20}", + 0x2F93A => "\u{20}\u{7610}\u{20}", + 0x2F93B => "\u{20}\u{24FA1}\u{20}", + 0x2F93C => "\u{20}\u{24FB8}\u{20}", + 0x2F93D => "\u{20}\u{25044}\u{20}", + 0x2F93E => "\u{20}\u{3FFC}\u{20}", + 0x2F93F => "\u{20}\u{4008}\u{20}", + 0x2F940 => "\u{20}\u{76F4}\u{20}", + 0x2F941 => "\u{20}\u{250F3}\u{20}", + 0x2F942 => "\u{20}\u{250F2}\u{20}", + 0x2F943 => "\u{20}\u{25119}\u{20}", + 0x2F944 => "\u{20}\u{25133}\u{20}", + 0x2F945 => "\u{20}\u{771E}\u{20}", + 0x2F946 => "\u{20}\u{771F}\u{20}", + 0x2F947 => "\u{20}\u{771F}\u{20}", + 0x2F948 => "\u{20}\u{774A}\u{20}", + 0x2F949 => "\u{20}\u{4039}\u{20}", + 0x2F94A => "\u{20}\u{778B}\u{20}", + 0x2F94B => "\u{20}\u{4046}\u{20}", + 0x2F94C => "\u{20}\u{4096}\u{20}", + 0x2F94D => "\u{20}\u{2541D}\u{20}", + 0x2F94E => "\u{20}\u{784E}\u{20}", + 0x2F94F => "\u{20}\u{788C}\u{20}", + 0x2F950 => "\u{20}\u{78CC}\u{20}", + 0x2F951 => "\u{20}\u{40E3}\u{20}", + 0x2F952 => "\u{20}\u{25626}\u{20}", + 0x2F953 => "\u{20}\u{7956}\u{20}", + 0x2F954 => "\u{20}\u{2569A}\u{20}", + 0x2F955 => "\u{20}\u{256C5}\u{20}", + 0x2F956 => "\u{20}\u{798F}\u{20}", + 0x2F957 => "\u{20}\u{79EB}\u{20}", + 0x2F958 => "\u{20}\u{412F}\u{20}", + 0x2F959 => "\u{20}\u{7A40}\u{20}", + 0x2F95A => "\u{20}\u{7A4A}\u{20}", + 0x2F95B => "\u{20}\u{7A4F}\u{20}", + 0x2F95C => "\u{20}\u{2597C}\u{20}", + 0x2F95D => "\u{20}\u{25AA7}\u{20}", + 0x2F95E => "\u{20}\u{25AA7}\u{20}", + 0x2F95F => "\u{20}\u{7AEE}\u{20}", + 0x2F960 => "\u{20}\u{4202}\u{20}", + 0x2F961 => "\u{20}\u{25BAB}\u{20}", + 0x2F962 => "\u{20}\u{7BC6}\u{20}", + 0x2F963 => "\u{20}\u{7BC9}\u{20}", + 0x2F964 => "\u{20}\u{4227}\u{20}", + 0x2F965 => "\u{20}\u{25C80}\u{20}", + 0x2F966 => "\u{20}\u{7CD2}\u{20}", + 0x2F967 => "\u{20}\u{42A0}\u{20}", + 0x2F968 => "\u{20}\u{7CE8}\u{20}", + 0x2F969 => "\u{20}\u{7CE3}\u{20}", + 0x2F96A => "\u{20}\u{7D00}\u{20}", + 0x2F96B => "\u{20}\u{25F86}\u{20}", + 0x2F96C => "\u{20}\u{7D63}\u{20}", + 0x2F96D => "\u{20}\u{4301}\u{20}", + 0x2F96E => "\u{20}\u{7DC7}\u{20}", + 0x2F96F => "\u{20}\u{7E02}\u{20}", + 0x2F970 => "\u{20}\u{7E45}\u{20}", + 0x2F971 => "\u{20}\u{4334}\u{20}", + 0x2F972 => "\u{20}\u{26228}\u{20}", + 0x2F973 => "\u{20}\u{26247}\u{20}", + 0x2F974 => "\u{20}\u{4359}\u{20}", + 0x2F975 => "\u{20}\u{262D9}\u{20}", + 0x2F976 => "\u{20}\u{7F7A}\u{20}", + 0x2F977 => "\u{20}\u{2633E}\u{20}", + 0x2F978 => "\u{20}\u{7F95}\u{20}", + 0x2F979 => "\u{20}\u{7FFA}\u{20}", + 0x2F97A => "\u{20}\u{8005}\u{20}", + 0x2F97B => "\u{20}\u{264DA}\u{20}", + 0x2F97C => "\u{20}\u{26523}\u{20}", + 0x2F97D => "\u{20}\u{8060}\u{20}", + 0x2F97E => "\u{20}\u{265A8}\u{20}", + 0x2F97F => "\u{20}\u{8070}\u{20}", + 0x2F980 => "\u{20}\u{2335F}\u{20}", + 0x2F981 => "\u{20}\u{43D5}\u{20}", + 0x2F982 => "\u{20}\u{80B2}\u{20}", + 0x2F983 => "\u{20}\u{8103}\u{20}", + 0x2F984 => "\u{20}\u{440B}\u{20}", + 0x2F985 => "\u{20}\u{813E}\u{20}", + 0x2F986 => "\u{20}\u{5AB5}\u{20}", + 0x2F987 => "\u{20}\u{267A7}\u{20}", + 0x2F988 => "\u{20}\u{267B5}\u{20}", + 0x2F989 => "\u{20}\u{23393}\u{20}", + 0x2F98A => "\u{20}\u{2339C}\u{20}", + 0x2F98B => "\u{20}\u{8201}\u{20}", + 0x2F98C => "\u{20}\u{8204}\u{20}", + 0x2F98D => "\u{20}\u{8F9E}\u{20}", + 0x2F98E => "\u{20}\u{446B}\u{20}", + 0x2F98F => "\u{20}\u{8291}\u{20}", + 0x2F990 => "\u{20}\u{828B}\u{20}", + 0x2F991 => "\u{20}\u{829D}\u{20}", + 0x2F992 => "\u{20}\u{52B3}\u{20}", + 0x2F993 => "\u{20}\u{82B1}\u{20}", + 0x2F994 => "\u{20}\u{82B3}\u{20}", + 0x2F995 => "\u{20}\u{82BD}\u{20}", + 0x2F996 => "\u{20}\u{82E6}\u{20}", + 0x2F997 => "\u{20}\u{26B3C}\u{20}", + 0x2F998 => "\u{20}\u{82E5}\u{20}", + 0x2F999 => "\u{20}\u{831D}\u{20}", + 0x2F99A => "\u{20}\u{8363}\u{20}", + 0x2F99B => "\u{20}\u{83AD}\u{20}", + 0x2F99C => "\u{20}\u{8323}\u{20}", + 0x2F99D => "\u{20}\u{83BD}\u{20}", + 0x2F99E => "\u{20}\u{83E7}\u{20}", + 0x2F99F => "\u{20}\u{8457}\u{20}", + 0x2F9A0 => "\u{20}\u{8353}\u{20}", + 0x2F9A1 => "\u{20}\u{83CA}\u{20}", + 0x2F9A2 => "\u{20}\u{83CC}\u{20}", + 0x2F9A3 => "\u{20}\u{83DC}\u{20}", + 0x2F9A4 => "\u{20}\u{26C36}\u{20}", + 0x2F9A5 => "\u{20}\u{26D6B}\u{20}", + 0x2F9A6 => "\u{20}\u{26CD5}\u{20}", + 0x2F9A7 => "\u{20}\u{452B}\u{20}", + 0x2F9A8 => "\u{20}\u{84F1}\u{20}", + 0x2F9A9 => "\u{20}\u{84F3}\u{20}", + 0x2F9AA => "\u{20}\u{8516}\u{20}", + 0x2F9AB => "\u{20}\u{273CA}\u{20}", + 0x2F9AC => "\u{20}\u{8564}\u{20}", + 0x2F9AD => "\u{20}\u{26F2C}\u{20}", + 0x2F9AE => "\u{20}\u{455D}\u{20}", + 0x2F9AF => "\u{20}\u{4561}\u{20}", + 0x2F9B0 => "\u{20}\u{26FB1}\u{20}", + 0x2F9B1 => "\u{20}\u{270D2}\u{20}", + 0x2F9B2 => "\u{20}\u{456B}\u{20}", + 0x2F9B3 => "\u{20}\u{8650}\u{20}", + 0x2F9B4 => "\u{20}\u{865C}\u{20}", + 0x2F9B5 => "\u{20}\u{8667}\u{20}", + 0x2F9B6 => "\u{20}\u{8669}\u{20}", + 0x2F9B7 => "\u{20}\u{86A9}\u{20}", + 0x2F9B8 => "\u{20}\u{8688}\u{20}", + 0x2F9B9 => "\u{20}\u{870E}\u{20}", + 0x2F9BA => "\u{20}\u{86E2}\u{20}", + 0x2F9BB => "\u{20}\u{8779}\u{20}", + 0x2F9BC => "\u{20}\u{8728}\u{20}", + 0x2F9BD => "\u{20}\u{876B}\u{20}", + 0x2F9BE => "\u{20}\u{8786}\u{20}", + 0x2F9BF => "\u{20}\u{45D7}\u{20}", + 0x2F9C0 => "\u{20}\u{87E1}\u{20}", + 0x2F9C1 => "\u{20}\u{8801}\u{20}", + 0x2F9C2 => "\u{20}\u{45F9}\u{20}", + 0x2F9C3 => "\u{20}\u{8860}\u{20}", + 0x2F9C4 => "\u{20}\u{8863}\u{20}", + 0x2F9C5 => "\u{20}\u{27667}\u{20}", + 0x2F9C6 => "\u{20}\u{88D7}\u{20}", + 0x2F9C7 => "\u{20}\u{88DE}\u{20}", + 0x2F9C8 => "\u{20}\u{4635}\u{20}", + 0x2F9C9 => "\u{20}\u{88FA}\u{20}", + 0x2F9CA => "\u{20}\u{34BB}\u{20}", + 0x2F9CB => "\u{20}\u{278AE}\u{20}", + 0x2F9CC => "\u{20}\u{27966}\u{20}", + 0x2F9CD => "\u{20}\u{46BE}\u{20}", + 0x2F9CE => "\u{20}\u{46C7}\u{20}", + 0x2F9CF => "\u{20}\u{8AA0}\u{20}", + 0x2F9D0 => "\u{20}\u{8AED}\u{20}", + 0x2F9D1 => "\u{20}\u{8B8A}\u{20}", + 0x2F9D2 => "\u{20}\u{8C55}\u{20}", + 0x2F9D3 => "\u{20}\u{27CA8}\u{20}", + 0x2F9D4 => "\u{20}\u{8CAB}\u{20}", + 0x2F9D5 => "\u{20}\u{8CC1}\u{20}", + 0x2F9D6 => "\u{20}\u{8D1B}\u{20}", + 0x2F9D7 => "\u{20}\u{8D77}\u{20}", + 0x2F9D8 => "\u{20}\u{27F2F}\u{20}", + 0x2F9D9 => "\u{20}\u{20804}\u{20}", + 0x2F9DA => "\u{20}\u{8DCB}\u{20}", + 0x2F9DB => "\u{20}\u{8DBC}\u{20}", + 0x2F9DC => "\u{20}\u{8DF0}\u{20}", + 0x2F9DD => "\u{20}\u{208DE}\u{20}", + 0x2F9DE => "\u{20}\u{8ED4}\u{20}", + 0x2F9DF => "\u{20}\u{8F38}\u{20}", + 0x2F9E0 => "\u{20}\u{285D2}\u{20}", + 0x2F9E1 => "\u{20}\u{285ED}\u{20}", + 0x2F9E2 => "\u{20}\u{9094}\u{20}", + 0x2F9E3 => "\u{20}\u{90F1}\u{20}", + 0x2F9E4 => "\u{20}\u{9111}\u{20}", + 0x2F9E5 => "\u{20}\u{2872E}\u{20}", + 0x2F9E6 => "\u{20}\u{911B}\u{20}", + 0x2F9E7 => "\u{20}\u{9238}\u{20}", + 0x2F9E8 => "\u{20}\u{92D7}\u{20}", + 0x2F9E9 => "\u{20}\u{92D8}\u{20}", + 0x2F9EA => "\u{20}\u{927C}\u{20}", + 0x2F9EB => "\u{20}\u{93F9}\u{20}", + 0x2F9EC => "\u{20}\u{9415}\u{20}", + 0x2F9ED => "\u{20}\u{28BFA}\u{20}", + 0x2F9EE => "\u{20}\u{958B}\u{20}", + 0x2F9EF => "\u{20}\u{4995}\u{20}", + 0x2F9F0 => "\u{20}\u{95B7}\u{20}", + 0x2F9F1 => "\u{20}\u{28D77}\u{20}", + 0x2F9F2 => "\u{20}\u{49E6}\u{20}", + 0x2F9F3 => "\u{20}\u{96C3}\u{20}", + 0x2F9F4 => "\u{20}\u{5DB2}\u{20}", + 0x2F9F5 => "\u{20}\u{9723}\u{20}", + 0x2F9F6 => "\u{20}\u{29145}\u{20}", + 0x2F9F7 => "\u{20}\u{2921A}\u{20}", + 0x2F9F8 => "\u{20}\u{4A6E}\u{20}", + 0x2F9F9 => "\u{20}\u{4A76}\u{20}", + 0x2F9FA => "\u{20}\u{97E0}\u{20}", + 0x2F9FB => "\u{20}\u{2940A}\u{20}", + 0x2F9FC => "\u{20}\u{4AB2}\u{20}", + 0x2F9FD => "\u{20}\u{29496}\u{20}", + 0x2F9FE => "\u{20}\u{980B}\u{20}", + 0x2F9FF => "\u{20}\u{980B}\u{20}", + 0x2FA00 => "\u{20}\u{9829}\u{20}", + 0x2FA01 => "\u{20}\u{295B6}\u{20}", + 0x2FA02 => "\u{20}\u{98E2}\u{20}", + 0x2FA03 => "\u{20}\u{4B33}\u{20}", + 0x2FA04 => "\u{20}\u{9929}\u{20}", + 0x2FA05 => "\u{20}\u{99A7}\u{20}", + 0x2FA06 => "\u{20}\u{99C2}\u{20}", + 0x2FA07 => "\u{20}\u{99FE}\u{20}", + 0x2FA08 => "\u{20}\u{4BCE}\u{20}", + 0x2FA09 => "\u{20}\u{29B30}\u{20}", + 0x2FA0A => "\u{20}\u{9B12}\u{20}", + 0x2FA0B => "\u{20}\u{9C40}\u{20}", + 0x2FA0C => "\u{20}\u{9CFD}\u{20}", + 0x2FA0D => "\u{20}\u{4CCE}\u{20}", + 0x2FA0E => "\u{20}\u{4CED}\u{20}", + 0x2FA0F => "\u{20}\u{9D67}\u{20}", + 0x2FA10 => "\u{20}\u{2A0CE}\u{20}", + 0x2FA11 => "\u{20}\u{4CF8}\u{20}", + 0x2FA12 => "\u{20}\u{2A105}\u{20}", + 0x2FA13 => "\u{20}\u{2A20E}\u{20}", + 0x2FA14 => "\u{20}\u{2A291}\u{20}", + 0x2FA15 => "\u{20}\u{9EBB}\u{20}", + 0x2FA16 => "\u{20}\u{4D56}\u{20}", + 0x2FA17 => "\u{20}\u{9EF9}\u{20}", + 0x2FA18 => "\u{20}\u{9EFE}\u{20}", + 0x2FA19 => "\u{20}\u{9F05}\u{20}", + 0x2FA1A => "\u{20}\u{9F0F}\u{20}", + 0x2FA1B => "\u{20}\u{9F16}\u{20}", + 0x2FA1C => "\u{20}\u{9F3B}\u{20}", + 0x2FA1D => "\u{20}\u{2A600}\u{20}" + } + + @doc "Code points the cleaning step removes (control, format, private use, NUL, U+FFFD)." + @spec remove() :: [{non_neg_integer(), non_neg_integer()}] + def remove, do: @remove + + @doc "Code points the cleaning step turns into a space." + @spec space() :: [{non_neg_integer(), non_neg_integer()}] + def space, do: @space + + @doc "Code points padded with spaces as CJK ideographs." + @spec cjk() :: [{non_neg_integer(), non_neg_integer()}] + def cjk, do: @cjk + + @doc "Code points the pre-tokeniser isolates as punctuation." + @spec punct() :: [{non_neg_integer(), non_neg_integer()}] + def punct, do: @punct + + @doc "Code points the pre-tokeniser splits on as whitespace." + @spec ws() :: [{non_neg_integer(), non_neg_integer()}] + def ws, do: @ws + + @doc "Every other code point whose normalised form is not itself: NFD, marks stripped, lowercased." + @spec mapping() :: %{non_neg_integer() => String.t()} + def mapping, do: @mapping +end diff --git a/lib/trinity/smoke.ex b/lib/trinity/smoke.ex index 77fd5c1..8932eb3 100644 --- a/lib/trinity/smoke.ex +++ b/lib/trinity/smoke.ex @@ -29,6 +29,10 @@ defmodule Trinity.Smoke do use Boundary, top_level?: true, deps: [Trinity, TrinityWeb], exports: [] + # Slice 133: `exla_line/0` names these only after `Code.ensure_loaded?(EXLA)`; a build without + # the neural group (`TRINITY_WITHOUT_ML=1`) compiles this module with none of them present. + @compile {:no_warn_undefined, [Nx, EXLA.Backend]} + @flag "--smoke" @type halt_fun :: (non_neg_integer() -> any()) diff --git a/mix.exs b/mix.exs index f68e239..dda0ef5 100644 --- a/mix.exs +++ b/mix.exs @@ -121,9 +121,36 @@ defmodule Trinity.MixProject do # (`__libc_single_threaded: symbol not found`, package run 35600216451); started at boot # it took the whole release down, started on demand it is the tier's reason. defp exla_release_applications do - case :os.type() do - {:win32, _} -> [] - _ -> [exla: :load] + cond do + without_ml?() -> [] + match?({:win32, _}, :os.type()) -> [] + true -> [exla: :load] + end + end + + # Slice 133, AC8: `TRINITY_WITHOUT_ML=1` builds the tree without the neural group (nx, exla, + # and bumblebee, which brings axon and tokenizers; xla comes with exla). Dropping them from + # the release alone is not possible: pgvector lists nx as an optional application, and + # `Mix.Release` includes an optional application whenever it is compiled, so the group leaves + # only a build that never had it. Use a build root of its own (`MIX_BUILD_ROOT`): the adapter of + # dependencies differs from the default build's. `config/config.exs` makes the static floor the + # embedder of such a build, and the modules that name the group check for it at run time + # (`scripts/static_floor_release.sh` assembles one and boots it). + defp without_ml?, do: System.get_env("TRINITY_WITHOUT_ML") == "1" + + defp ml_deps do + if without_ml?() do + [] + else + exla_deps() ++ + [ + # Slice 032: local embeddings (Trinity.Memory.Embedders.Bumblebee). The 0.13 line of nx + # and exla is what bumblebee 0.7.1 accepts (nx 1.0.0 shipped 2026-09-10 and bumblebee + # has no release for it at G1); measured for bundle size and latency in the slice's + # NOTES.md. + {:nx, "~> 0.13.1"}, + {:bumblebee, "~> 0.7.1"} + ] end end @@ -148,7 +175,7 @@ defmodule Trinity.MixProject do end defp deps do - exla_deps() ++ + ml_deps() ++ [ {:phoenix, "~> 1.8.13"}, {:phoenix_ecto, "~> 4.5"}, @@ -172,11 +199,6 @@ defmodule Trinity.MixProject do {:jcs, "~> 0.2"}, # Slice 022: HTML to text for web_fetch (Trinity.Tools.Web.Fetch). {:floki, "~> 0.38"}, - # Slice 032: local embeddings (Trinity.Memory.Embedders.Bumblebee). The 0.13 line of nx - # and exla is what bumblebee 0.7.1 accepts (nx 1.0.0 shipped 2026-09-10 and bumblebee has - # no release for it at G1); measured for bundle size and latency in the slice's NOTES.md. - {:nx, "~> 0.13.1"}, - {:bumblebee, "~> 0.7.1"}, # Slice 032: vectors on the Postgres job (Trinity.Memory.VectorStores.Pgvector). {:pgvector, "~> 0.4.1"}, # Slice 040: SKILL.md frontmatter (Trinity.Skills.Parser) and the skill roots' watcher diff --git a/priv/repo/migrations/20261008120000_create_memory_embeddings.exs b/priv/repo/migrations/20261008120000_create_memory_embeddings.exs new file mode 100644 index 0000000..2ba3a45 --- /dev/null +++ b/priv/repo/migrations/20261008120000_create_memory_embeddings.exs @@ -0,0 +1,257 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.Repo.Migrations.CreateMemoryEmbeddings do + @moduledoc """ + Slice 133, AC1. Vectors move out of `memories` into `memory_embeddings`, keyed + `(memory_id, space_id)`, and every space a row belongs to is an `embedding_spaces` row holding + its fourteen-field manifest (`Trinity.Memory.Space`). + + Why a table and not columns: a memory can hold a vector in two spaces at once, which is what a + re-tier needs (the new space is built beside the old one, and the old one stays until the + operator drops it). And on Postgres `memories.embedding_vector vector(384)` cannot hold a 256-wide + vector beside a 384-wide one, so the column here is an **untyped** `vector`, with one partial + expression HNSW index per space (`(embedding_vector::vector())`, `WHERE space_id = ''`). + + The move: each distinct `(embedding_model, embedding_dim)` in `memories` becomes a legacy space + (every field the row never recorded is `"unrecorded"`; `legacy_manifest/2` is a frozen copy of + `Trinity.Memory.Space.legacy/2`, held equal to it by a test), every row with a vector is copied + under its space with its bytes unchanged (float32, as they were written), and the space of the + most recently embedded row is the store's active space (NOTES, decision 3). Then the old columns + and their indexes go. `down/0` puts the active space's vectors back on the rows. + """ + use Ecto.Migration + + def up do + create table(:embedding_spaces, primary_key: false) do + add :id, :string, primary_key: true + add :manifest, :map, null: false + add :dim, :integer, null: false + add :quantization, :string, null: false + add :state, :string, null: false, default: "complete" + add :active, :boolean, null: false, default: false + add :completed_at, :utc_datetime_usec + timestamps(type: :utc_datetime_usec) + end + + # One active space per store, held by the database: a second `active = true` row is refused. + create unique_index(:embedding_spaces, [:active], + where: "active", + name: :embedding_spaces_one_active + ) + + create table(:memory_embeddings, primary_key: false) do + add :memory_id, references(:memories, type: :binary_id, on_delete: :delete_all), + primary_key: true, + null: false + + add :space_id, references(:embedding_spaces, type: :string, on_delete: :restrict), + primary_key: true, + null: false + + add :dim, :integer, null: false + add :vector, :binary, null: false + add :inserted_at, :utc_datetime_usec, null: false + end + + create index(:memory_embeddings, [:space_id]) + + if postgres?() do + execute("ALTER TABLE memory_embeddings ADD COLUMN embedding_vector vector") + end + + flush() + move_rows() + + if postgres?() do + execute("DROP INDEX IF EXISTS memories_embedding_vector_idx") + execute("ALTER TABLE memories DROP COLUMN IF EXISTS embedding_vector") + end + + drop index(:memories, [:persona_id, :tier, :embedding_model]) + + alter table(:memories) do + remove :embedding + remove :embedding_model + remove :embedding_dim + end + end + + def down do + alter table(:memories) do + add :embedding, :binary + add :embedding_model, :string + add :embedding_dim, :integer + end + + create index(:memories, [:persona_id, :tier, :embedding_model]) + + if postgres?() do + execute("ALTER TABLE memories ADD COLUMN embedding_vector vector(384)") + end + + flush() + + # The active space's vectors return to their rows, under the space's model id, which for a + # legacy space is the `embedding_model` it came from. Other spaces' vectors have no column + # to go back to and are dropped with the table. + %{rows: active} = + repo().query!("SELECT id, manifest FROM embedding_spaces WHERE active = #{true_literal()}") + + for [space_id, manifest] <- active do + model = decode(manifest)["model_id"] + + repo().query!( + "UPDATE memories SET embedding = (SELECT vector FROM memory_embeddings e WHERE e.memory_id = memories.id AND e.space_id = #{p(1)}), " <> + "embedding_dim = (SELECT dim FROM memory_embeddings e WHERE e.memory_id = memories.id AND e.space_id = #{p(1)}), " <> + "embedding_model = #{p(2)} " <> + "WHERE id IN (SELECT memory_id FROM memory_embeddings WHERE space_id = #{p(1)})", + [space_id, model] + ) + + if postgres?() do + repo().query!( + "UPDATE memories SET embedding_vector = (SELECT embedding_vector::vector(384) FROM memory_embeddings e WHERE e.memory_id = memories.id AND e.space_id = $1) " <> + "WHERE embedding_dim = 384 AND id IN (SELECT memory_id FROM memory_embeddings WHERE space_id = $1)", + [space_id] + ) + end + end + + if postgres?() do + execute( + "CREATE INDEX memories_embedding_vector_idx ON memories USING hnsw (embedding_vector vector_cosine_ops) WHERE tier = 'semantic'" + ) + end + + drop table(:memory_embeddings) + drop table(:embedding_spaces) + end + + defp move_rows do + %{rows: models} = + repo().query!( + "SELECT embedding_model, embedding_dim, MAX(updated_at) FROM memories " <> + "WHERE embedding IS NOT NULL AND embedding_model IS NOT NULL AND embedding_dim IS NOT NULL " <> + "GROUP BY embedding_model, embedding_dim" + ) + + now = DateTime.utc_now() + + spaces = + for [model, dim, last] <- models do + manifest = legacy_manifest(model, dim) + id = manifest_id(manifest) + + repo().query!( + "INSERT INTO embedding_spaces (id, manifest, dim, quantization, state, active, completed_at, inserted_at, updated_at) " <> + "VALUES (#{p(1)}, #{p(2)}, #{p(3)}, 'f32', 'complete', #{false_literal()}, #{p(4)}, #{p(4)}, #{p(4)})", + [id, encode(manifest), dim, now] + ) + + repo().query!( + "INSERT INTO memory_embeddings (memory_id, space_id, dim, vector, inserted_at" <> + if(postgres?(), do: ", embedding_vector", else: "") <> + ") SELECT id, #{p(1)}, embedding_dim, embedding, #{p(2)}" <> + if(postgres?(), do: ", embedding_vector::vector", else: "") <> + " FROM memories WHERE embedding IS NOT NULL AND embedding_model = #{p(3)} AND embedding_dim = #{p(4)}", + [id, now, model, dim] + ) + + if postgres?() and dim <= 2000 do + fill_pg_vectors(id) + create_space_index(id, dim) + end + + {id, last} + end + + case Enum.max_by(spaces, &elem(&1, 1), &later?/2, fn -> nil end) do + nil -> + :ok + + {id, _} -> + repo().query!( + "UPDATE embedding_spaces SET active = #{true_literal()} WHERE id = #{p(1)}", + [id] + ) + end + end + + # Slice 032 filled `memories.embedding_vector` only for 384-wide vectors; a row of another + # width under 2,000 has its bytes and no pgvector value. It gets one here, decoded from its + # float32 bytes, so every row of an indexed space can be ranked by the index. + defp fill_pg_vectors(id) do + %{rows: rows} = + repo().query!( + "SELECT memory_id, vector FROM memory_embeddings WHERE space_id = $1 AND embedding_vector IS NULL", + [id] + ) + + for [memory_id, bytes] <- rows do + floats = for <>, do: f + + repo().query!( + "UPDATE memory_embeddings SET embedding_vector = $1 WHERE memory_id = $2 AND space_id = $3", + [Pgvector.new(floats), memory_id, id] + ) + end + end + + defp later?(a, b), do: compare(a, b) != :lt + + defp compare(%DateTime{} = a, %DateTime{} = b), do: DateTime.compare(a, b) + defp compare(%NaiveDateTime{} = a, %NaiveDateTime{} = b), do: NaiveDateTime.compare(a, b) + defp compare(a, b) when is_binary(a) and is_binary(b), do: if(a >= b, do: :gt, else: :lt) + + # The per-space index, the same statement `Trinity.Memory.Spaces` issues for a new space, and + # serial for the same reason: a parallel HNSW build wants dynamic shared memory a container's + # default /dev/shm does not have. + defp create_space_index(id, dim) do + true = id =~ ~r/\A[0-9a-f]{64}\z/ + repo().query!("SET LOCAL max_parallel_maintenance_workers = 0") + + repo().query!( + "CREATE INDEX IF NOT EXISTS memory_embeddings_hnsw_#{binary_part(id, 0, 16)} ON memory_embeddings " <> + "USING hnsw ((embedding_vector::vector(#{dim})) vector_cosine_ops) WHERE space_id = '#{id}'" + ) + end + + # A frozen copy of `Trinity.Memory.Space.legacy/2` and `Space.id/1` as of slice 133: a migration + # must keep meaning what it meant when it ran, whatever the module becomes later. + defp legacy_manifest(model, dim) do + u = "unrecorded" + + %{ + "model_id" => model, + "revision" => u, + "weights_digest" => u, + "tokenizer_digest" => u, + "dim" => dim, + "pooling" => u, + "normalisation" => u, + "quantization" => "f32", + "query_prompt" => u, + "document_prompt" => u, + "max_input" => %{"tokens" => u, "truncation" => u}, + "runtime" => %{"name" => u, "version" => u}, + "locality" => u, + "num_ctx" => u + } + end + + defp manifest_id(manifest), + do: + manifest |> Jcs.encode() |> then(&:crypto.hash(:sha256, &1)) |> Base.encode16(case: :lower) + + defp encode(manifest) do + if postgres?(), do: manifest, else: Jason.encode!(manifest) + end + + defp decode(m) when is_map(m), do: m + defp decode(m) when is_binary(m), do: Jason.decode!(m) + + defp postgres?, do: repo().__adapter__() == Ecto.Adapters.Postgres + defp p(n), do: if(postgres?(), do: "$#{n}", else: "?#{n}") + defp true_literal, do: if(postgres?(), do: "TRUE", else: "1") + defp false_literal, do: if(postgres?(), do: "FALSE", else: "0") +end diff --git a/scripts/eval/reference_vectors.py b/scripts/eval/reference_vectors.py new file mode 100644 index 0000000..f376797 --- /dev/null +++ b/scripts/eval/reference_vectors.py @@ -0,0 +1,88 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +"""Vectors for the memory eval's candidates that do not run in Trinity's process (slice 133, AC14). + +MiniLM (all-MiniLM-L6-v2, the 3-point rule's reference) and potion-retrieval-32M (the challenger, +whose tokenizer is not the bert-base-uncased pipeline Trinity's WordPiece implements) embed the +eval's corpus and queries through sentence-transformers at pinned revisions. Writes, per model: + + /.corpus.f32 corpus vectors, float32 little-endian, corpus order, L2-normalised + /.queries.f32 query vectors, same layout + /.json dim, model, revision, the ids in order, the floor, versions, digests + +Derived from weights, so written outside the tree (slice 133 NOTES, D7). Offline: + + HF_HUB_OFFLINE=1 python -I scripts/eval/reference_vectors.py --eval --out \ + --model minilm=@1110a243fdf4706b3f48f1d95db1a4f5529b4d41 \ + --model potion-retrieval-32M=@6fc8051fab2a1e0ee76689cf08c853792ac285e7 +""" +import argparse +import hashlib +import json +import os + +import numpy as np +import sentence_transformers +import torch +from sentence_transformers import SentenceTransformer + + +def jsonl(path): + with open(path, encoding="utf-8") as f: + return [json.loads(line) for line in f] + + +def sha256(path): + h = hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--eval", required=True) + ap.add_argument("--out", required=True) + ap.add_argument("--model", action="append", required=True, help="name=@") + ap.add_argument("--floor", type=float, default=0.3) + args = ap.parse_args() + + corpus = jsonl(os.path.join(args.eval, "corpus.jsonl")) + queries = jsonl(os.path.join(args.eval, "queries.jsonl")) + os.makedirs(args.out, exist_ok=True) + + for spec in args.model: + name, rest = spec.split("=", 1) + snapshot, revision = rest.rsplit("@", 1) + model = SentenceTransformer(snapshot, device="cpu") + with torch.no_grad(): + cv = model.encode([m["text"] for m in corpus], batch_size=128, normalize_embeddings=True, convert_to_numpy=True) + qv = model.encode([q["text"] for q in queries], batch_size=128, normalize_embeddings=True, convert_to_numpy=True) + for suffix, v in (("corpus", cv), ("queries", qv)): + v.astype(" MIX_ENV=test mix run --no-start scripts/scorer_bench.exs +# +# With TRINITY_STATIC_MODEL_DIR set, the 10^4 vectors are the static model's own embeddings of +# 10^4 generated memory-shaped sentences (their distribution is what the scorer meets); without +# it, seeded Gaussian vectors quantized the same way. Each figure is one query at a time, 200 +# queries after 20 discarded warm-ups, p50 and p95 by sorting the 200 times. +# +# Four scorers over the same rows: +# exact the int8 cosine over every row (`Scorer.exact/3` on a prepared index) +# prefilter sign bits by Hamming distance, then the exact int8 cosine over the 256 nearest +# (`Scorer.prefilter/4`): the path D-scoring names +# prepare building the prepared index from the rows (norms and sign bits), once per load +# naive the deliberately unoptimised scorer: every row's bytes decoded to a list of +# floats and the float cosine taken, per query, which is the Elixir path slice 032 +# measured at 390 ms over 10^4 x 384 floats. It is here to show the measurement can +# tell a slow scorer from a fast one (AC13's red). +# And recall@10 of the prefilter against the exact search, since the prefilter is approximate. + +alias Trinity.Memory.{Embedder, Scorer} + +n = 10_000 +queries = 200 +warm = 20 +candidates = 256 + +{source, vectors, query_vectors} = + case System.get_env("TRINITY_STATIC_MODEL_DIR") do + dir when dir not in [nil, ""] -> + Application.put_env(:trinity, :memory, embedder: :static, static_dir: dir) + alias Trinity.Memory.Embedders.Static + topics = ~w(coffee tea Lisbon Boston sister brother dog cat garden piano budget project Postgres + SQLite meeting Tuesday allergy peanuts bicycle river library trip passport doctor) + verbs = ~w(likes prefers decided remembers mentioned dislikes planned moved bought visited) + :rand.seed(:exsss, {133, 13, 1}) + + sentence = fn -> + "The person #{Enum.random(verbs)} #{Enum.random(topics)} and #{Enum.random(topics)} " <> + "on day #{:rand.uniform(365)}." + end + + {:ok, vs} = Static.embed(for _ <- 1..n, do: sentence.()) + {:ok, qs} = Static.embed(for _ <- 1..(queries + warm), do: sentence.()) + {"static-retrieval-mrl-en-v1 256 int8, generated sentences", vs, qs} + + _ -> + :rand.seed(:exsss, {133, 13, 2}) + gauss = fn -> for _ <- 1..256, do: :rand.normal() end + {"seeded Gaussian", for(_ <- 1..n, do: gauss.()), for(_ <- 1..(queries + warm), do: gauss.())} + end + +rows = vectors |> Enum.with_index() |> Enum.map(fn {v, i} -> {i, Scorer.quantize(v)} end) +qs = Enum.map(query_vectors, &Scorer.quantize/1) + +{prepare_us, index} = :timer.tc(fn -> Scorer.prepare(rows) end) + +time = fn f -> + all = for q <- qs, do: elem(:timer.tc(fn -> f.(q) end), 0) + sorted = all |> Enum.drop(warm) |> Enum.sort() + {Enum.at(sorted, div(queries, 2)) / 1000, Enum.at(sorted, round(queries * 0.95) - 1) / 1000} +end + +{exact50, exact95} = time.(fn q -> Scorer.exact(index, q, 10) end) +{pre50, pre95} = time.(fn q -> Scorer.prefilter(index, q, 10, candidates) end) + +naive = fn q -> + qf = Scorer.to_floats(q) + + rows + |> Enum.map(fn {id, bin} -> {id, Embedder.cosine(qf, for(<>, do: x * 1.0))} end) + |> Enum.sort_by(&elem(&1, 1), :desc) + |> Enum.take(10) +end + +{naive50, naive95} = time.(naive) + +recall = + qs + |> Enum.drop(warm) + |> Enum.map(fn q -> + exact = q |> then(&Scorer.exact(index, &1, 10)) |> MapSet.new(&elem(&1, 0)) + pre = q |> then(&Scorer.prefilter(index, &1, 10, candidates)) |> MapSet.new(&elem(&1, 0)) + MapSet.size(MapSet.intersection(exact, pre)) / 10 + end) + |> then(&(Enum.sum(&1) / length(&1))) + +IO.puts(""" +scorer bench: #{n} rows x 256 int8 (#{source}); #{queries} queries after #{warm} warm-ups + exact p50 #{exact50} ms p95 #{exact95} ms + prefilter p50 #{pre50} ms p95 #{pre95} ms (#{candidates} candidates; recall@10 against exact #{Float.round(recall, 4)}) + prepare #{div(prepare_us, 1000)} ms once per load (norms and sign bits for #{n} rows) + naive p50 #{naive50} ms p95 #{naive95} ms (decode to floats per query: the red) + budget 50 ms p95: exact #{if exact95 <= 50, do: "within", else: "OVER"}, prefilter #{if pre95 <= 50, do: "within", else: "OVER"}, naive #{if naive95 <= 50, do: "within", else: "OVER"} +otp #{System.otp_release()} erts #{:erlang.system_info(:version)} elixir #{System.version()} schedulers #{System.schedulers_online()} +""") diff --git a/scripts/static/reference_fixtures.py b/scripts/static/reference_fixtures.py new file mode 100644 index 0000000..a434188 --- /dev/null +++ b/scripts/static/reference_fixtures.py @@ -0,0 +1,200 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +"""Writes the static embedder's parity fixtures into a model cache directory (slice 133). + +Three files, all derived from the pinned reference and none of them committed (D7: nothing derived +from the weights enters the tree before legal review; the tokenizer fixtures are kept with them +because they are derived from the model repository's tokenizer.json): + + tokenizer-natural.jsonl 5,000 natural English sentences and the reference token ids (AC9) + tokenizer-adversarial.jsonl 1,000 generated hostile inputs and the reference token ids (AC9) + embeddings.jsonl 500 texts and the reference's un-normalised outputs at 1024 and 256 + dimensions (AC10) + +and `fixtures-manifest.json` naming every input: the model revision, the dataset revision, the +seed, the library versions and each output file's SHA-256. + + HF_HUB_OFFLINE=1 python -I scripts/static/reference_fixtures.py \ + --snapshot \ + --sentences --out /fixtures + +The natural sentences are from WikiText-103 (raw, test split): Wikipedia prose, which carries +accents, numbers, dates and punctuation the way real text does. +""" +import argparse +import hashlib +import json +import os +import random +import re +import sys + +import numpy as np +import pyarrow.parquet as pq +import sentence_transformers +import tokenizers +import torch +from sentence_transformers import SentenceTransformer +from tokenizers import Tokenizer + +SEED = 20261007 + + +def natural_sentences(parquet, n, rng): + lines = pq.read_table(parquet).column("text").to_pylist() + sentences = [] + for line in lines: + line = line.strip() + if not line or line.startswith("="): + continue + # WikiText's tokenisation puts spaces around punctuation and writes " @-@ " for a hyphen; + # undo the commonest so the sentences read as prose. + line = line.replace(" @-@ ", "-").replace(" @,@ ", ",").replace(" @.@ ", ".") + line = re.sub(r" ([,.;:!?)\]'])", r"\1", line).replace("( ", "(") + for s in re.split(r"(?<=[.!?])\s+(?=[A-Z\"'(])", line): + if 20 <= len(s) <= 400: + sentences.append(s) + sentences = sorted(set(sentences)) + rng.shuffle(sentences) + if len(sentences) < n: + sys.exit(f"only {len(sentences)} sentences available, {n} wanted") + return sentences[:n] + + +# The adversarial categories AC9 names, and the ones the answer sheet's parity risks add. +ACCENTED = "àáâãäåāăąçćĉċčďđèéêëēĕėęěĝğġģĥħìíîïĩīĭįıĵķĺļľŀłñńņňʼnòóôõöøōŏőœŕŗřśŝşšţťŧùúûüũūŭůűųŵýÿŷźżžÀÉÎÕÜÇÑØÅÆß" +DECOMPOSED = ["é", "à", "ö", "ñ", "ç", "ů", "Ą́", "i̇"] +VIETNAMESE = ["Tiếng Việt", "người", "Nguyễn", "đường", "ữ", "Ặ", "quốc ngữ"] +CJK = ["野口里佳", "中华人民共和国", "東京都", "𠀀𠀁𠀂", "𪜀", "豈", "日本語のテキスト", "カタカナ", "ひらがな", "한국어 문장", "가나다", "ＡＢＣ１２３", "　", "〜", "、。"] +EMOJI = ["😀", "👍🏽", "👨‍👩‍👧‍👦", "🏳️‍🌈", "🇺🇸", "🇯🇵", "1️⃣", "❤️", "🧑‍💻", "🫠", "🥲", "🪿", "🫨", "🩷", "🐦‍🔥", "🙂‍↕️", "☕", "✅", "©", "™"] +ZERO_WIDTH = ["​", "‌", "‍", "⁠", "", "­", "᠎", "‎", "‏", "‪", "‮"] +CONTROL = ["\x00", "\x01", "\x07", "\x08", "\x0b", "\x0c", "\x1b", "\x1f", "\x7f", "\x80", "\x85", "\x9f", "\t", "\n", "\r", "\r\n", "�", "", "", "\U000f0000", "\U0010fffd", "͸", "\U0001fbca", "\U000e0001", "\U000e0041"] +SPECIAL = ["[CLS]", "[SEP]", "[MASK]", "[PAD]", "[UNK]", "[cls]", "[[SEP]]", "[MASK]x", "a[UNK]b", "[CLS][CLS]", "[SEP ]", "[ MASK]"] +SCRIPTS = ["Ελληνικά ΣΟΦΟΣ σοφός", "İstanbul ıi", "financial flow", "Straße STRASSE ẞ", "𝐀𝐁𝐂 𝔄𝔅", "عربي", "עברית", "हिन्दी", "ไทย", "Ⅻ ⅷ", "½ ¾ ²", "DŽ Dž dž", "ʼn", "ΐ", "ᾳ", "Å Ω K"] +PUNCT = ["!", "?", "...", "…", "—", "–", "«»", "„“", "‘’", "¿¡", "§¶", "†‡", "※", "‽", "⸘", "〈〉", "【】", "@#$%^&*", "~`|\\", "<>", "{}", "[]", "()", "₹€£¥", "°", "·", "•", "⁂", "꙳", "𑁇"] +SPACES = [" ", " ", " ", " ", " ", " ", "
", "
", " ", " ", "　", " ", "\t\t"] +WORDS = ["memory", "Trinity", "remember", "preference", "decided", "coffee", "Lisbon", "unaffable", "tokenization", "XLII", "x86_64", "e-mail", "don't", "it's", "U.S.A.", "3.14159", "2026-10-07", "user@example.com", "https://example.org/a?b=c", "#hashtag", "@mention", "C++", "naïve", "café", "résumé", "coöperate"] + + +def long_word(rng): + n = rng.choice([99, 100, 101, 150, 300]) + alphabet = rng.choice(["abcdefghijklmnopqrstuvwxyz", "aé", "x", "ab-", "日本", "a😀"]) + return "".join(rng.choice(alphabet) for _ in range(n)) + + +def fragment(rng): + pick = rng.random() + if pick < 0.12: + return rng.choice(ACCENTED) * rng.randint(1, 3) + rng.choice(WORDS) + if pick < 0.18: + return rng.choice(WORDS) + rng.choice(DECOMPOSED) + if pick < 0.24: + return rng.choice(VIETNAMESE) + if pick < 0.34: + return rng.choice(CJK) + if pick < 0.44: + return rng.choice(EMOJI) + if pick < 0.52: + w = rng.choice(WORDS) + i = rng.randint(0, len(w)) + return w[:i] + rng.choice(ZERO_WIDTH) + w[i:] + if pick < 0.60: + return rng.choice(CONTROL) + if pick < 0.66: + return long_word(rng) + if pick < 0.72: + return rng.choice(SPECIAL) + if pick < 0.82: + return rng.choice(SCRIPTS) + if pick < 0.92: + return rng.choice(PUNCT) + return rng.choice(WORDS) + + +def adversarial(n, rng): + items, seen = [], set() + while len(items) < n: + parts = [fragment(rng) for _ in range(rng.randint(1, 6))] + sep = [rng.choice(["", " ", " ", rng.choice(SPACES)]) for _ in parts] + text = "".join(p + s for p, s in zip(parts, sep)) + if text not in seen: + seen.add(text) + items.append(text) + return items + + +def sha256(path): + h = hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + + +def write_jsonl(path, rows): + with open(path, "w", encoding="utf-8") as f: + for row in rows: + f.write(json.dumps(row, ensure_ascii=False) + "\n") + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--snapshot", required=True) + ap.add_argument("--revision", default="f60985c706f192d45d218078e49e5a8b6f15283a") + ap.add_argument("--sentences", required=True) + ap.add_argument("--sentences-revision", default="b08601e04326c79dfdd32d625aee71d232d685c3") + ap.add_argument("--out", required=True) + args = ap.parse_args() + + os.makedirs(args.out, exist_ok=True) + rng = random.Random(SEED) + tok = Tokenizer.from_file(os.path.join(args.snapshot, "0_StaticEmbedding", "tokenizer.json")) + + def ids(text): + # StaticEmbedding.preprocess: encode_batch(..., add_special_tokens=False). + return tok.encode(text, add_special_tokens=False).ids + + natural = natural_sentences(args.sentences, 5000, rng) + write_jsonl(os.path.join(args.out, "tokenizer-natural.jsonl"), [{"text": t, "ids": ids(t)} for t in natural]) + + hostile = adversarial(1000, rng) + write_jsonl(os.path.join(args.out, "tokenizer-adversarial.jsonl"), [{"text": t, "ids": ids(t)} for t in hostile]) + + # AC10: 400 natural sentences and 100 adversarial items that tokenize to at least one id (an + # empty bag's mean is the zero vector, which has no direction to compare). + model = SentenceTransformer(args.snapshot, device="cpu") + pool = [t for t in natural[:1000]][:400] + [t for t in hostile if ids(t)][:100] + with torch.no_grad(): + full = model.encode(pool, convert_to_numpy=True, normalize_embeddings=False, batch_size=64) + full = full.astype(np.float32) + rows = [ + {"text": t, "ids": ids(t), "v1024": [float(x) for x in v], "v256": [float(x) for x in v[:256]]} + for t, v in zip(pool, full) + ] + write_jsonl(os.path.join(args.out, "embeddings.jsonl"), rows) + + manifest = { + "model": "sentence-transformers/static-retrieval-mrl-en-v1", + "revision": args.revision, + "sentences": "Salesforce/wikitext wikitext-103-raw-v1 test", + "sentences_revision": args.sentences_revision, + "seed": SEED, + "versions": { + "tokenizers": tokenizers.__version__, + "sentence_transformers": sentence_transformers.__version__, + "torch": torch.__version__, + "numpy": np.__version__, + }, + "files": { + name: sha256(os.path.join(args.out, name)) + for name in ["tokenizer-natural.jsonl", "tokenizer-adversarial.jsonl", "embeddings.jsonl"] + }, + "script_sha256": sha256(os.path.abspath(__file__)), + } + with open(os.path.join(args.out, "fixtures-manifest.json"), "w") as f: + json.dump(manifest, f, indent=2, sort_keys=True) + print(json.dumps(manifest, indent=2, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/scripts/static/wordpiece_tables.py b/scripts/static/wordpiece_tables.py new file mode 100644 index 0000000..7737239 --- /dev/null +++ b/scripts/static/wordpiece_tables.py @@ -0,0 +1,144 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +"""Writes lib/trinity/memory/word_piece/tables.ex from the reference tokenizer's own behaviour. + +Slice 133. The pure-Elixir WordPiece must give the ids Hugging Face `tokenizers` gives, and that +library classifies characters with its own Unicode tables (the `unicode_categories` crate for +control, punctuation and non-spacing marks; `unicode-normalization-alignments` for NFD; Rust's std +for whitespace and lowercase), which are not the Unicode version OTP ships. So the tables are not +written from a Unicode database at all: every code point is put through the reference +normaliser and pre-tokeniser, one at a time, and what they do is recorded. + +Run it with the pinned reference (requirements in the slice's NOTES), outside the tree's own +environment, isolated: + + python -I scripts/static/wordpiece_tables.py > lib/trinity/memory/word_piece/tables.ex + +It reads no model file: the tables depend on the library, not on any vocabulary or weights. +""" +import sys + +import tokenizers +from tokenizers.normalizers import BertNormalizer +from tokenizers.pre_tokenizers import BertPreTokenizer + +NORM = BertNormalizer(clean_text=True, handle_chinese_chars=True, strip_accents=None, lowercase=True) +PRE = BertPreTokenizer() + +HANGUL_FIRST, HANGUL_LAST = 0xAC00, 0xD7A3 + + +def codepoints(): + for cp in range(0x110000): + if 0xD800 <= cp <= 0xDFFF: + continue + yield cp + + +def hangul_nfd(cp): + s = cp - HANGUL_FIRST + l, v, t = 0x1100 + s // 588, 0x1161 + (s % 588) // 28, 0x11A7 + s % 28 + return chr(l) + chr(v) + (chr(t) if t != 0x11A7 else "") + + +def ranges(cps): + out = [] + for cp in sorted(cps): + if out and out[-1][1] == cp - 1: + out[-1][1] = cp + else: + out.append([cp, cp]) + return out + + +def main(): + remove, space, cjk, punct, ws = [], [], [], [], [] + mapping = {} + for cp in codepoints(): + c = chr(cp) + out = NORM.normalize_str(c) + if out == "" and cp in range(HANGUL_FIRST, HANGUL_LAST + 1): + raise SystemExit(f"unexpected removal of a Hangul syllable {cp:#x}") + if HANGUL_FIRST <= cp <= HANGUL_LAST: + if out != hangul_nfd(cp): + raise SystemExit(f"Hangul {cp:#x} is not the algorithmic decomposition: {out!r}") + elif out == "": + remove.append(cp) + elif out == " ": + space.append(cp) + elif out == " " + c + " ": + cjk.append(cp) + elif out != c: + mapping[cp] = out + # The pre-tokeniser: whitespace splits and is dropped, punctuation is isolated. + pieces = [p for p, _ in PRE.pre_tokenize_str("a" + c + "b")] + if pieces == ["a", "b"]: + ws.append(cp) + elif pieces == ["a", c, "b"]: + punct.append(cp) + elif pieces != ["a" + c + "b"]: + raise SystemExit(f"unexpected pre-tokenisation of {cp:#x}: {pieces!r}") + + def ex_ranges(name, cps): + body = ",\n ".join(f"{{0x{a:X}, 0x{b:X}}}" for a, b in ranges(cps)) + return f" @{name} [\n {body}\n ]\n" + + def ex_string(s): + return '"' + "".join(f"\\u{{{ord(ch):X}}}" for ch in s) + '"' + + entries = ",\n ".join(f"0x{cp:X} => {ex_string(v)}" for cp, v in sorted(mapping.items())) + + sys.stdout.write(f'''# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +# GENERATED by scripts/static/wordpiece_tables.py from tokenizers {tokenizers.__version__}. Do not edit. +defmodule Trinity.Memory.WordPiece.Tables do + @moduledoc """ + What the reference BERT normaliser and pre-tokeniser do to each code point (slice 133), + recorded by putting every code point through Hugging Face `tokenizers` {tokenizers.__version__} + one at a time (`scripts/static/wordpiece_tables.py`). Generated, not written from a Unicode + database: the reference classifies with its own, older, tables, and parity means matching + them rather than OTP's. + + Hangul syllables (U+AC00 to U+D7A3) are not listed: the generator checked that the reference + decomposes every one of them algorithmically, and `Trinity.Memory.WordPiece` does the same. + """ + +{ex_ranges("remove", remove)} +{ex_ranges("space", space)} +{ex_ranges("cjk", cjk)} +{ex_ranges("punct", punct)} +{ex_ranges("ws", ws)} + @mapping %{{ + {entries} + }} + + @doc "Code points the cleaning step removes (control, format, private use, NUL, U+FFFD)." + @spec remove() :: [{{non_neg_integer(), non_neg_integer()}}] + def remove, do: @remove + + @doc "Code points the cleaning step turns into a space." + @spec space() :: [{{non_neg_integer(), non_neg_integer()}}] + def space, do: @space + + @doc "Code points padded with spaces as CJK ideographs." + @spec cjk() :: [{{non_neg_integer(), non_neg_integer()}}] + def cjk, do: @cjk + + @doc "Code points the pre-tokeniser isolates as punctuation." + @spec punct() :: [{{non_neg_integer(), non_neg_integer()}}] + def punct, do: @punct + + @doc "Code points the pre-tokeniser splits on as whitespace." + @spec ws() :: [{{non_neg_integer(), non_neg_integer()}}] + def ws, do: @ws + + @doc "Every other code point whose normalised form is not itself: NFD, marks stripped, lowercased." + @spec mapping() :: %{{non_neg_integer() => String.t()}} + def mapping, do: @mapping +end +''') + print(f"remove={len(remove)} space={len(space)} cjk={len(cjk)} punct={len(punct)} ws={len(ws)} mapping={len(mapping)}", file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/scripts/static_floor_release.sh b/scripts/static_floor_release.sh new file mode 100755 index 0000000..9421f49 --- /dev/null +++ b/scripts/static_floor_release.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env sh +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +# +# Slice 133, AC8: a release assembled WITHOUT the neural group (nx, exla, xla, axon, bumblebee, +# tokenizers) boots, embeds and recalls through the static space, as a spawned process. +# +# TRINITY_STATIC_MODEL_DIR= ./scripts/static_floor_release.sh +# +# Four steps, each printing what it found: +# 1. compile the tree for prod with TRINITY_WITHOUT_ML=1, in a build root of its own +# (FLOOR_BUILD_ROOT, default ../_build_floor), warnings as errors +# 2. assemble the headless release from that build +# 3. list the release's lib/ and refuse it if any of the group is there +# 4. boot it (`bin/headless eval`, its own XDG_DATA_HOME), write two memories, recall one, +# and print the markers the caller asserts on +# +# The weights are not in the tree (slice 133 NOTES, D7): the release reads them from +# TRINITY_STATIC_MODEL_DIR, as an operator-installed artifact would be read from the data dir. +set -eu +cd "$(dirname "$0")/.." + +: "${TRINITY_STATIC_MODEL_DIR:?set it to the directory holding static-retrieval-mrl-en-v1-256-int8.tsw}" +build_root="${FLOOR_BUILD_ROOT:-../_build_floor}" +work="$(mktemp -d "${TMPDIR:-/tmp}/static-floor.XXXXXX")" +rel="$work/rel" +trap 'rm -rf "$work"' EXIT + +echo "== 1. compile without the neural group ==" +TRINITY_WITHOUT_ML=1 MIX_ENV=prod MIX_BUILD_ROOT="$build_root" mix compile --force --warnings-as-errors + +echo "== 2. assemble the headless release ==" +TRINITY_WITHOUT_ML=1 MIX_ENV=prod MIX_BUILD_ROOT="$build_root" \ + mix release headless --path "$rel" --overwrite --quiet + +echo "== 3. the release's applications ==" +ls "$rel/lib" | sed 's/-[0-9][0-9.]*$//' | sort | tr '\n' ' ' +echo +if ls "$rel/lib" | grep -E '^(nx|exla|xla|axon|bumblebee|tokenizers)-'; then + echo "FLOOR_GROUP_PRESENT" + exit 2 +fi +echo "FLOOR_GROUP_ABSENT" + +echo "== 4. boot, embed, recall ==" +# The desktop sidecar's heartbeat listens on a Unix socket under TMPDIR, and a socket path has +# a length limit (108 bytes on Linux): a long TMPDIR makes the listener fail with :einval and the +# boot with it. The node gets a short one. +XDG_DATA_HOME="$work/data" TRINITY_STATIC_MODEL_DIR="$TRINITY_STATIC_MODEL_DIR" \ + TMPDIR="${FLOOR_NODE_TMPDIR:-/tmp}" \ + "$rel/bin/headless" eval ' + {:ok, _} = Application.ensure_all_started(:trinity) + alias Trinity.Memory.{AlwaysOn, Retriever, Semantic, Spaces} + IO.puts("FLOOR_EMBEDDER " <> inspect(Trinity.Memory.Embedder.impl())) + IO.puts("FLOOR_NX_LOADABLE " <> inspect(Code.ensure_loaded?(Nx))) + IO.puts("FLOOR_STATUS " <> inspect(Semantic.status())) + {:ok, persona} = Trinity.Personas.create(%{name: "floor"}) + scope = AlwaysOn.persona_scope(persona.id) + for {k, body} <- [{"rex", "The person'"'"'s dog is called Rex."}, {"lisbon", "The person moved to Lisbon last spring."}] do + {:ok, _} = Semantic.add(%{persona_id: persona.id, scope: scope, key: k, body: body}, by: "floor") + end + hits = Retriever.relevant(persona.id, nil, "what is my dog named?", touch: false) + IO.puts("FLOOR_SPACE " <> Spaces.active().id <> " " <> Spaces.active().quantization) + IO.puts("FLOOR_RECALL " <> inspect(Enum.map(hits, &{&1.kind, &1.found_by, &1.text}))) + System.halt(0) + ' diff --git a/scripts/vector_bench.exs b/scripts/vector_bench.exs index 02f471e..5bb6610 100644 --- a/scripts/vector_bench.exs +++ b/scripts/vector_bench.exs @@ -29,13 +29,31 @@ else end Logger.configure(level: :info) + +# Migrated before the application starts: Oban checks its own migration at start, and a fresh +# scratch database has none yet. +for repo <- [Trinity.Repo, Trinity.Repo.Receipts] do + _ = repo.__adapter__().storage_up(Application.get_env(:trinity, repo)) + {:ok, _, _} = Ecto.Migrator.with_repo(repo, &Ecto.Migrator.run(&1, :up, all: true)) +end + {:ok, _} = Application.ensure_all_started(:trinity) -Ecto.Migrator.run(Trinity.Repo, :up, all: true) -Ecto.Migrator.run(Trinity.Repo.Receipts, :up, all: true) -alias Trinity.Memory.{AlwaysOn, Embedders.Fake, Entry, Semantic, VectorStore, VectorStores.Brute} +# Slice 133: vectors live in memory_embeddings under a space; the bench writes the fake's space +# and searches it, as slice 032 measured the model column. TRINITY_BENCH_QUANT=int8 writes the +# same vectors under an int8 space instead, which `Brute` scores with `Trinity.Memory.Scorer` +# (the static floor's path: load, fit check, exact int8 cosine). +alias Trinity.Memory.{AlwaysOn, Embedders.Fake, Entry, Semantic, Spaces, Vector, VectorStore} +alias Trinity.Memory.VectorStores.Brute {:ok, persona} = Trinity.Personas.create(%{name: "bench-#{System.unique_integer([:positive])}"}) scope = AlwaysOn.persona_scope(persona.id) +space = + case System.get_env("TRINITY_BENCH_QUANT") do + "int8" -> %{Fake.space() | model_id: "bench:int8", quantization: "int8"} + _ -> Fake.space() + end + +{:ok, space_row} = Spaces.register(space) for n <- [1_000, 10_000] do from = Semantic.count(persona.id) @@ -53,30 +71,34 @@ for n <- [1_000, 10_000] do } {:ok, e} = %Entry{} |> Entry.semantic_changeset(attrs) |> Trinity.Repo.insert() - :ok = VectorStore.upsert(e.id, Fake.vector("fact number #{i}"), Fake.model_id()) + :ok = Spaces.put_vector(e.id, space_row.id, space, Fake.vector(attrs.body)) end end, timeout: :infinity ) end) - filter = Semantic.filter(persona.id, [scope]) + filter = Semantic.filter(persona.id, [scope], space_row) q = Fake.vector("fact number #{div(n, 2)}") {first_us, _} = :timer.tc(fn -> VectorStore.search(q, 8, filter) end) times = for _ <- 1..5, do: elem(:timer.tc(fn -> VectorStore.search(q, 8, filter) end), 0) - [%{entry: top}] = VectorStore.search(q, 1, filter) + {:ok, [%{entry: top}]} = VectorStore.search(q, 1, filter) IO.puts( "rows=#{n} insert+store=#{div(ins_us, 1000)} ms search first=#{div(first_us, 1000)} ms p50=#{Enum.at(Enum.sort(times), 2) / 1000} ms max=#{Enum.max(times) / 1000} ms top=#{top.body} store=#{inspect(VectorStore.impl())}" ) - if VectorStore.impl() == Brute do + if VectorStore.impl() == Brute and space.quantization == "f32" do import Ecto.Query {load_us, rows} = :timer.tc(fn -> Trinity.Repo.all( - from e in Entry, where: e.persona_id == ^persona.id and e.tier == "semantic" + from e in Entry, + join: v in Vector, + on: v.memory_id == e.id and v.space_id == ^space_row.id, + where: e.persona_id == ^persona.id and e.tier == "semantic", + select: {e, v.vector} ) end) @@ -92,7 +114,7 @@ for n <- [1_000, 10_000] do end IO.puts( - "rows=#{n} split: load=#{div(load_us, 1000)} ms elixir_cosine=#{div(elixir_us, 1000)} ms #{exla} vector_bytes=#{Enum.sum(Enum.map(rows, &byte_size(&1.embedding)))}" + "rows=#{n} split: load=#{div(load_us, 1000)} ms elixir_cosine=#{div(elixir_us, 1000)} ms #{exla} vector_bytes=#{Enum.sum(Enum.map(rows, &byte_size(elem(&1, 1))))}" ) end end diff --git a/test/property/space_identity_test.exs b/test/property/space_identity_test.exs new file mode 100644 index 0000000..56ce3b1 --- /dev/null +++ b/test/property/space_identity_test.exs @@ -0,0 +1,110 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Property.SpaceIdentityTest do + @moduledoc """ + Slice 133, AC1's property: changing any one of the fourteen identity fields of an embedding + space yields a different space ID. + + This is the "is this the same thing" shape docs/03 asks a property for. The failure it guards + is quiet: two spaces that share an ID share a store, and a store holding vectors of two spaces + answers by comparing numbers that mean different things. Before slice 133 the identity was + `embedding_model` alone, so a new revision of the same model, the same model truncated to + another width or served by another runtime, all collided; the red run of this file against + that identity is in the slice's NOTES. + """ + use ExUnit.Case, async: true + use ExUnitProperties + + alias Trinity.Memory.Space + + # Every leaf of the struct, with a generator of its values. The two compound identity fields + # (`max_input`, `runtime`) have two leaves each, so sixteen leaves cover the fourteen fields. + @string_leaves ~w(model_id revision weights_digest tokenizer_digest pooling normalisation + quantization query_prompt document_prompt truncation runtime runtime_version + locality)a + @int_or_none_leaves ~w(max_input_tokens num_ctx)a + @leaves [:dim | @string_leaves ++ @int_or_none_leaves] + + defp text, do: string(:printable, max_length: 24) + + defp int_or_none, do: one_of([integer(1..100_000), constant("none"), constant("unrecorded")]) + + defp space do + gen all( + strings <- fixed_list(Enum.map(@string_leaves, fn _ -> text() end)), + dim <- integer(1..4096), + ints <- fixed_list(Enum.map(@int_or_none_leaves, fn _ -> int_or_none() end)) + ) do + fields = + Enum.zip(@string_leaves, strings) ++ + [{:dim, dim}] ++ Enum.zip(@int_or_none_leaves, ints) + + struct!(Space, fields) + end + end + + # A different value for one leaf, of the type that leaf holds. + defp other(:dim, old), do: filter(integer(1..4096), &(&1 != old)) + defp other(leaf, old) when leaf in @int_or_none_leaves, do: filter(int_or_none(), &(&1 != old)) + defp other(_leaf, old), do: filter(text(), &(&1 != old)) + + property "changing any one identity field changes the space ID" do + check all( + s <- space(), + leaf <- member_of(@leaves), + value <- other(leaf, Map.fetch!(s, leaf)) + ) do + mutated = Map.put(s, leaf, value) + + refute Space.id(mutated) == Space.id(s), + "the space ID did not change when #{leaf} went from " <> + "#{inspect(Map.fetch!(s, leaf))} to #{inspect(value)}" + end + end + + test "the leaves cover the fourteen identity fields named in the moduledoc" do + covered = + @leaves + |> Enum.map(fn + l when l in [:max_input_tokens, :truncation] -> "max_input" + l when l in [:runtime, :runtime_version] -> "runtime" + l -> Atom.to_string(l) + end) + |> Enum.uniq() + |> Enum.sort() + + assert covered == Enum.sort(Space.identity_fields()) + assert length(Space.identity_fields()) == 14 + # And the struct has no field this test does not mutate. + assert Enum.sort(Map.keys(Space.__struct__()) -- [:__struct__]) == Enum.sort(@leaves) + end + + test "two revisions of one model are two spaces (the case today's identity merged)" do + a = example() + b = %{a | revision: "a-later-commit"} + refute Space.id(a) == Space.id(b) + assert Space.id(a) == Space.id(Space.from_manifest(Space.manifest(a))) + assert Space.id(a) =~ ~r/\A[0-9a-f]{64}\z/ + end + + defp example do + %Space{ + model_id: "sentence-transformers/static-retrieval-mrl-en-v1", + revision: "f60985c706f192d45d218078e49e5a8b6f15283a", + weights_digest: "unrecorded", + tokenizer_digest: "unrecorded", + dim: 256, + pooling: "mean", + normalisation: "l2", + quantization: "int8", + query_prompt: "", + document_prompt: "", + max_input_tokens: "none", + truncation: "none", + runtime: "trinity-static", + runtime_version: "1", + locality: "in_process", + num_ctx: "none" + } + end +end diff --git a/test/release/static_floor_release_test.exs b/test/release/static_floor_release_test.exs new file mode 100644 index 0000000..56e1d9d --- /dev/null +++ b/test/release/static_floor_release_test.exs @@ -0,0 +1,52 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.StaticFloorReleaseTest do + @moduledoc """ + Slice 133, AC8: a release assembled without the neural group boots, embeds and recalls + through the static space, as a spawned process. `scripts/static_floor_release.sh` does the + work (a prod build with `TRINITY_WITHOUT_ML=1` in a build root of its own, the headless release, + the group's absence from its `lib/`, a boot with two memories written and one recalled); this + runs it and asserts on the lines it prints. + + `:static_weights`: the release reads the artifact from `TRINITY_STATIC_MODEL_DIR`. It compiles + a whole prod tree the first time, which takes minutes. + """ + use ExUnit.Case, async: false + + @moduletag :static_weights + @moduletag timeout: 1_800_000 + + defp line(out, prefix), + do: out |> String.split("\n") |> Enum.find("", &String.starts_with?(&1, prefix)) + + test "AC8: without nx, exla, xla, axon, bumblebee and tokenizers, the release boots, embeds and recalls" do + dir = Trinity.StaticWeights.dir!() + + {out, status} = + System.cmd("sh", ["scripts/static_floor_release.sh"], + env: [ + {"TRINITY_STATIC_MODEL_DIR", dir}, + {"MIX_ENV", nil}, + {"FLOOR_BUILD_ROOT", System.get_env("FLOOR_BUILD_ROOT", "../_build_floor")} + ], + stderr_to_stdout: true + ) + + lines = + ~w(FLOOR_GROUP FLOOR_EMBEDDER FLOOR_NX_LOADABLE FLOOR_STATUS FLOOR_SPACE FLOOR_RECALL) + |> Enum.map_join("\n", &line(out, &1)) + + IO.puts("\n" <> lines) + assert status == 0, String.slice(out, -4000, 4000) + assert line(out, "FLOOR_GROUP") == "FLOOR_GROUP_ABSENT" + assert line(out, "FLOOR_EMBEDDER") == "FLOOR_EMBEDDER Trinity.Memory.Embedders.Static" + assert line(out, "FLOOR_NX_LOADABLE") == "FLOOR_NX_LOADABLE false" + assert line(out, "FLOOR_STATUS") == "FLOOR_STATUS :on" + + assert line(out, "FLOOR_SPACE") == + "FLOOR_SPACE #{Trinity.Memory.Space.id(Trinity.Memory.Embedders.Static.space())} int8" + + assert line(out, "FLOOR_RECALL") =~ + ~s({:memory, [:vector], "The person's dog is called Rex."}) + end +end diff --git a/test/smoke_test.exs b/test/smoke_test.exs index 014ab3b..0cee6e5 100644 --- a/test/smoke_test.exs +++ b/test/smoke_test.exs @@ -119,7 +119,8 @@ defmodule SmokeTest do test "the EXLA line on this machine (the NIF is compiled and loads here; the bundle's answer is AC7's)" do assert Smoke.exla_line() == "TRINITY_SMOKE_EXLA=ok" - assert Smoke.semantic_line() == "TRINITY_SMOKE_SEMANTIC=on" + # Slice 133: the semantic line reads the store's active space, so it needs the database; + # Trinity.Memory.VectorStoreTest asserts it through the probe, under the sandbox. end end end diff --git a/test/support/data_case.ex b/test/support/data_case.ex index 571b3b1..78e538b 100644 --- a/test/support/data_case.ex +++ b/test/support/data_case.ex @@ -31,6 +31,9 @@ defmodule Trinity.DataCase do setup tags do Trinity.DataCase.setup_sandbox(tags) + # Slice 133: the semantic tier's runtime fault lives outside the database (it is a fact about + # the node, not the store), so a test that provoked one would otherwise hand it to the next. + Trinity.Memory.Semantic.clear_fault(:all) :ok end diff --git a/test/support/excluded_counter.ex b/test/support/excluded_counter.ex new file mode 100644 index 0000000..a3f812e --- /dev/null +++ b/test/support/excluded_counter.ex @@ -0,0 +1,54 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.ExcludedCounter do + @moduledoc """ + An ExUnit formatter that says, at the end of a run, how many tests each opt-in tag excluded + (slice 133). + + The weight-dependent tests are excluded when their variable is unset, and the owner's rule + is that the run names how many were excluded: a silent exclusion would let the floor ship + untested with a green summary. ExUnit's own summary gives one total for every reason at once; + this prints one line per tag that excluded anything, after it: + + excluded by tag: static_weights 9 (TRINITY_STATIC_MODEL_DIR unset), local_model 1 (...) + + It counts from the tests' own tags as ExUnit reports them, not from a list, so a new tagged + test is counted without editing this module. + """ + use GenServer + + @reasons %{ + static_weights: "TRINITY_STATIC_MODEL_DIR unset", + local_model: "TRINITY_LOCAL_MODEL_CACHE unset", + postgres: "not the Postgres adapter", + fips: "TRINITY_FIPS_LEG unset" + } + + @impl true + def init(_opts), do: {:ok, %{}} + + @impl true + def handle_cast({:test_finished, %ExUnit.Test{state: {:excluded, _}, tags: tags}}, counts) do + counts = + Enum.reduce(Map.keys(@reasons), counts, fn tag, acc -> + if Map.get(tags, tag), do: Map.update(acc, tag, 1, &(&1 + 1)), else: acc + end) + + {:noreply, counts} + end + + def handle_cast({:suite_finished, _times}, counts) do + if counts != %{} do + line = + counts + |> Enum.sort() + |> Enum.map_join(", ", fn {tag, n} -> "#{tag} #{n} (#{@reasons[tag]})" end) + + IO.puts("excluded by tag: " <> line) + end + + {:noreply, counts} + end + + def handle_cast(_event, counts), do: {:noreply, counts} +end diff --git a/test/support/fake_embedder_b.ex b/test/support/fake_embedder_b.ex new file mode 100644 index 0000000..ec9750c --- /dev/null +++ b/test/support/fake_embedder_b.ex @@ -0,0 +1,39 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.FakeEmbedderB do + @moduledoc """ + A second fake embedder for the suite (slice 133): 256 wide, its own seed, so its own space. + `Trinity.Memory.Embedders.Fake` takes its width from configuration and so can be only one + space at a time; a re-tier needs two configured at once (`embedder: [Trinity.FakeEmbedderB, + :fake]`), and AC2 needs a 256-wide space beside the 384-wide one on Postgres. + """ + @behaviour Trinity.Memory.Embedder + + alias Trinity.Memory.Embedders.Fake + alias Trinity.Memory.Space + + @dim 256 + + @impl true + def dim, do: @dim + + @impl true + def model_id, do: "fake:sha256-256-b" + + @impl true + def availability, do: :ok + + @impl true + def space, + do: %{Space.legacy(model_id(), @dim) | locality: "in_process", runtime: "trinity-fake"} + + @impl true + def thresholds, do: %{floor: 0.3, dedupe: 0.92} + + @impl true + def embed(texts), do: {:ok, Enum.map(texts, &vector/1)} + + @doc "The vector for one text." + @spec vector(String.t()) :: [float()] + def vector(text), do: Fake.vector(text, @dim, "b") +end diff --git a/test/support/spaces_helper.ex b/test/support/spaces_helper.ex new file mode 100644 index 0000000..6ed94fd --- /dev/null +++ b/test/support/spaces_helper.ex @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.SpacesHelper do + @moduledoc """ + Test-only: points a store's active space at a space directly, standing in for an operator's + completed re-tier where a test needs a store already pinned somewhere (slice 133). Nothing in + `lib/` moves the pointer this way; `Trinity.Memory.Spaces.retier/2` is the operator's path. + """ + import Ecto.Query + + alias Trinity.Memory.{Space, SpaceRow, Spaces} + alias Trinity.Repo + + @doc "Registers the space and makes it the only active one." + @spec pin!(Space.t()) :: SpaceRow.t() + def pin!(%Space{} = space) do + {:ok, row} = Spaces.register(space) + + {:ok, _} = + Repo.transaction(fn -> + Repo.update_all(from(s in SpaceRow, where: s.active == true), set: [active: false]) + Repo.update_all(from(s in SpaceRow, where: s.id == ^row.id), set: [active: true]) + end) + + Spaces.get(row.id) + end +end diff --git a/test/support/static_weights.ex b/test/support/static_weights.ex new file mode 100644 index 0000000..439ff46 --- /dev/null +++ b/test/support/static_weights.ex @@ -0,0 +1,120 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.StaticWeights do + @moduledoc """ + Where the `:static_weights` tests find the static model's artifact and the reference + fixtures (slice 133): the directory `TRINITY_STATIC_MODEL_DIR` names, holding the artifacts + `mix trinity.static.build` writes and a `fixtures/` directory `scripts/static/reference_fixtures.py` + writes. None of it is in the tree (NOTES, D7). + + The tests that use this are tagged `:static_weights` and excluded when the variable is unset + (`test/test_helper.exs`). When it is set, a missing file **fails** the test, by `flunk/1` with + the path: never a skip, so a set variable pointing at nothing cannot pass for a tested floor. + """ + import ExUnit.Assertions, only: [flunk: 1] + + @doc "The directory, or a failure naming the variable." + @spec dir!() :: Path.t() + def dir! do + case System.get_env("TRINITY_STATIC_MODEL_DIR") do + blank when blank in [nil, ""] -> + flunk("TRINITY_STATIC_MODEL_DIR is unset, yet a :static_weights test ran") + + dir -> + unless File.dir?(dir), do: flunk("TRINITY_STATIC_MODEL_DIR=#{dir} is not a directory") + dir + end + end + + @doc "A file under the directory, or a failure naming what is missing." + @spec file!(Path.t()) :: Path.t() + def file!(relative) do + path = Path.join(dir!(), relative) + unless File.regular?(path), do: flunk("missing #{path} (TRINITY_STATIC_MODEL_DIR is set)") + path + end + + @doc "The rows of a fixture JSONL file, decoded." + @spec fixtures!(String.t()) :: [map()] + def fixtures!(name) do + ("fixtures/" <> name) |> file!() |> File.stream!() |> Enum.map(&Jason.decode!/1) + end + + @doc """ + Points the static embedder at the directory for the rest of the test (`variant`, the 256 int8 + by default) and forgets any artifact an earlier test loaded. Returns the previous memory + configuration for `on_exit`. + """ + @spec use_static!(String.t()) :: keyword() + def use_static!(variant \\ "256-int8") do + file!(Trinity.Memory.Embedders.Static.file_name(variant)) + previous = Application.get_env(:trinity, :memory, []) + + Application.put_env( + :trinity, + :memory, + Keyword.merge(previous, embedder: :static, static_dir: dir!(), static_variant: variant) + ) + + Trinity.Memory.Embedders.Static.reload() + previous + end + + @synthetic_vocab ~w([PAD] [UNK] [CLS] [SEP] [MASK] the a cat dog sat on mat is was called + rex lisbon coffee tea person likes moved to in . , ##s ##ing ##ed) + + @doc """ + Writes a **synthetic** artifact into `dir`: the real file format and the 256 int8 variant's + name, over a 33-token vocabulary and seeded random weights. No model is involved, so it is in + reach of every leg; tests that need the static path's plumbing (the store, the int8 scorer, + the digest check) use it, and tests that need the real model use `use_static!/1`. Returns the + file's SHA-256, which the caller configures as `static_sha256:` since no pin names it. + """ + @spec synthetic!(Path.t()) :: String.t() + def synthetic!(dir) do + File.mkdir_p!(dir) + :rand.seed(:exsss, {133, 133, 133}) + rows = length(@synthetic_vocab) + + matrix = + for _ <- 1..(rows * 256), into: <<>>, do: <<:rand.normal()::float-little-32>> + + bytes = + Trinity.Memory.StaticArtifact.build(matrix, rows, 256, @synthetic_vocab, + dim: 256, + quantization: :int8, + provenance: %{"model" => "synthetic", "revision" => "none"} + ) + + File.write!(Path.join(dir, Trinity.Memory.Embedders.Static.file_name("256-int8")), bytes) + :crypto.hash(:sha256, bytes) |> Base.encode16(case: :lower) + end + + @doc "Points the static embedder at a synthetic artifact in `dir`; returns the previous config." + @spec use_synthetic!(Path.t()) :: keyword() + def use_synthetic!(dir) do + digest = synthetic!(dir) + previous = Application.get_env(:trinity, :memory, []) + + Application.put_env( + :trinity, + :memory, + Keyword.merge(previous, + embedder: :static, + static_dir: dir, + static_variant: "256-int8", + static_sha256: digest + ) + ) + + Trinity.Memory.Embedders.Static.reload() + previous + end + + @doc "Restores what `use_static!/1` replaced." + @spec restore!(keyword()) :: :ok + def restore!(previous) do + Application.put_env(:trinity, :memory, previous) + Trinity.Memory.Embedders.Static.reload() + end +end diff --git a/test/test_helper.exs b/test/test_helper.exs index be84b45..f7389cf 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -10,13 +10,25 @@ # Slice 032: :local_model (the real embedder through the serving) runs where # TRINITY_LOCAL_MODEL_CACHE names a cache that already holds all-MiniLM-L6-v2 (no download, # no network: a missing model is a failure there, not a skip); excluded by tag elsewhere. +# Slice 133: :static_weights (the static embedder's real weights and its parity fixtures) runs +# where TRINITY_STATIC_MODEL_DIR names the directory holding them; excluded by tag elsewhere, +# and where the variable is set a missing file is a failure, never a skip (the owner's rule, +# 2026-10-07). No weights are in the tree before legal review (slice 133 NOTES, D7). +# :postgres (EXPLAIN plans and the per-space HNSW indexes) runs only on the Postgres adapter. +# `Trinity.ExcludedCounter` prints, after the summary, how many each of these tags excluded. fips_leg? = System.get_env("TRINITY_FIPS_LEG") == "1" local_model? = System.get_env("TRINITY_LOCAL_MODEL_CACHE") not in [nil, ""] +static_weights? = System.get_env("TRINITY_STATIC_MODEL_DIR") not in [nil, ""] +postgres? = Application.get_env(:trinity, :db_adapter) == Ecto.Adapters.Postgres ExUnit.start( exclude: [:live, :desktop, :eval] ++ - if(fips_leg?, do: [], else: [:fips]) ++ if(local_model?, do: [], else: [:local_model]) + if(fips_leg?, do: [], else: [:fips]) ++ + if(local_model?, do: [], else: [:local_model]) ++ + if(static_weights?, do: [], else: [:static_weights]) ++ + if(postgres?, do: [], else: [:postgres]), + formatters: [ExUnit.CLIFormatter, Trinity.ExcludedCounter] ) # Slice 024's boot receipt is written by a transient Task after the tree starts; once the diff --git a/test/trinity/embedder_boot_node_test.exs b/test/trinity/embedder_boot_node_test.exs new file mode 100644 index 0000000..3335441 --- /dev/null +++ b/test/trinity/embedder_boot_node_test.exs @@ -0,0 +1,281 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.EmbedderBootNodeTest do + @moduledoc """ + Slice 133, AC5 and AC6's second half, where they are real: a boot, in a separate OS process. + + AC5: under `:regulated` each embedder configuration fault refuses the boot with its own named + error (an endpoint off `TRINITY_REGULATED_LLM_ENDPOINTS`, an embedder with no `locality:`, an + `:external` embedder without the operator's opt-in), and a correct one boots. A `localhost` + URL with no `locality:` is a fault under **both** profiles: under `:default` the node boots and + semantic memory is OFF with the same reason, which is what "fails config validation" means + where the profile never refuses a boot. + + AC6: one corrupted byte in the static weights turns semantic memory OFF with + `:weights_digest_mismatch` and the application still boots. Here on a synthetic artifact (the + real format, no model), so it runs on every leg; `:static_weights` repeats it on the real file. + + The harness is `Trinity.RegulatedBootNodeTest`'s: `MIX_ENV=test`, `XDG_DATA_HOME` and + databases per case (`Trinity.BootIsolation`), the child printing `BOOT_OK` or `BOOT_REFUSED` + with the reason at `limit: :infinity`, and assertions on that one line only. + """ + use ExUnit.Case, async: false + + @moduletag timeout: 180_000 + + @isolate """ + Trinity.BootIsolation.isolate!(System.fetch_env!("TRINITY_BOOT_TAG")) + """ + + # After a boot that succeeded: the tier's status, from inside the node. + @report """ + case Application.ensure_all_started(:trinity) do + {:ok, _} -> + IO.puts("BOOT_OK") + IO.puts("SEMANTIC " <> inspect(Trinity.Memory.Semantic.status(), limit: :infinity)) + System.halt(0) + + {:error, reason} -> + IO.puts("BOOT_REFUSED " <> inspect(reason, limit: :infinity, printable_limit: :infinity)) + System.halt(3) + end + """ + + @fixture """ + defmodule EmbedderBootFixture do + @behaviour Trinity.Authority + def stage(staged, _ctx), do: {:ok, staged} + def decide(_staged, decision, _ctx), do: {:ok, decision, "fixture"} + def execute(_staged, _decision, _ctx), do: {:error, :not_used} + def receipt(_kind, _attrs), do: {:error, :not_used} + end + Application.put_env(:trinity, :mcp_auth, profile: :production) + """ + + # The chat model on the allow-list, and an embedding model at `url`; then the memory config. + defp models_and_memory(url, memory) do + """ + llm = Application.get_env(:trinity, :llm, []) + Application.put_env(:trinity, :llm, Keyword.put(llm, :models, [ + %{id: "allowed", provider: :req_llm, model: "openai:x", + base_url: "https://models.internal/v1", api_key_env: "NONE", + caps: [:stream], price: %{input: 0.0, output: 0.0}}, + %{id: "embedder", provider: :req_llm, model: "openai_compatible:e5", + base_url: #{inspect(url)}, api_key_env: "NONE", + caps: [:embed], price: %{input: 0.0, output: 0.0}} + ])) + memory = Application.get_env(:trinity, :memory, []) + Application.put_env(:trinity, :memory, Keyword.merge(memory, #{inspect(memory)})) + """ + end + + defp case_setup(name) do + tag = Trinity.BootIsolation.tag(name) + on_exit(fn -> Trinity.BootIsolation.drop!(tag) end) + dir = Path.join(System.tmp_dir!(), "embboot-#{name}-#{System.unique_integer([:positive])}") + File.mkdir_p!(dir) + on_exit(fn -> File.rm_rf(dir) end) + {dir, tag} + end + + defp boot(name, env, script) do + {root, tag} = case_setup(name) + + base = [ + {"MIX_ENV", "test"}, + {"XDG_DATA_HOME", root}, + {"TRINITY_BOOT_TAG", tag}, + {"TRINITY_PROFILE", nil}, + {"TRINITY_AUTHORITY", nil}, + {"TRINITY_REGULATED_LLM_ENDPOINTS", nil}, + {"TRINITY_MCP_AUTH_PROFILE", nil}, + {"TRINITY_STATIC_MODEL_DIR", nil} + ] + + System.cmd("mix", ["run", "--no-start", "-e", @isolate <> script], + env: base ++ env, + stderr_to_stdout: true + ) + end + + defp regulated(extra \\ []) do + [ + {"TRINITY_PROFILE", "regulated"}, + {"TRINITY_AUTHORITY", "EmbedderBootFixture"}, + {"TRINITY_REGULATED_LLM_ENDPOINTS", "https://models.internal"} + ] ++ extra + end + + defp line(out, prefix), + do: out |> String.split("\n") |> Enum.find("", &String.starts_with?(&1, prefix)) + + defp refused!(out, status, reason) do + assert status == 3, "the node started:\n#{String.slice(out, -3000, 3000)}" + refusal = line(out, "BOOT_REFUSED") + + assert refusal =~ reason, + "it refused, but the printed reason is not the one under test:\n#{refusal}" + + IO.puts("\n" <> refusal) + refusal + end + + describe "AC5: config faults refuse a regulated boot, each with its name" do + test "a within_boundary endpoint not on the allow-list: :not_allow_listed" do + script = + @fixture <> + models_and_memory("https://embed.internal/v1", + embedder: :hosted, + hosted_model: "embedder", + locality: :within_boundary + ) <> @report + + {out, status} = boot("emb_not_listed", regulated(), script) + refusal = refused!(out, status, "not_allow_listed") + assert refusal =~ "https://embed.internal" + end + + test "an embedder with an endpoint and no locality: :embedder_locality_undeclared" do + script = + @fixture <> + models_and_memory("https://models.internal/v1", + embedder: :hosted, + hosted_model: "embedder" + ) <> @report + + {out, status} = boot("emb_no_locality", regulated(), script) + refused!(out, status, "embedder_locality_undeclared") + end + + test "an external embedder without the operator's opt-in: :external_not_opted_in" do + script = + @fixture <> + models_and_memory("https://models.internal/v1", + embedder: :hosted, + hosted_model: "embedder", + locality: :external + ) <> @report + + {out, status} = boot("emb_external", regulated(), script) + refused!(out, status, "external_not_opted_in") + end + + test "the correct configuration boots: within_boundary on the allow-list, and external with the opt-in" do + for {name, memory} <- [ + {"emb_good_within", + [embedder: :hosted, hosted_model: "embedder", locality: :within_boundary]}, + {"emb_good_external", + [ + embedder: :hosted, + hosted_model: "embedder", + locality: :external, + external_opt_in: true + ]} + ] do + script = @fixture <> models_and_memory("https://models.internal/v1", memory) <> @report + {out, status} = boot(name, regulated(), script) + + assert status == 0, """ + a correctly configured regulated node refused (#{name}): if this fails while the refusals + pass, the check refuses everything and proves nothing. + #{String.slice(out, -3000, 3000)} + """ + + assert line(out, "BOOT_OK") == "BOOT_OK" + end + end + end + + describe "AC5: locality is never inferred from an address, under both profiles" do + test "a localhost URL with no locality refuses a regulated boot" do + script = + @fixture <> + models_and_memory("http://localhost:11434/v1", + embedder: :hosted, + hosted_model: "embedder" + ) <> @report + + {out, status} = + boot( + "emb_localhost_reg", + regulated([ + {"TRINITY_REGULATED_LLM_ENDPOINTS", "https://models.internal,http://localhost"} + ]), + script + ) + + refusal = refused!(out, status, "embedder_locality_undeclared") + assert refusal =~ "http://localhost:11434/v1" + end + + test "a localhost URL with no locality fails validation under default: it boots with semantic memory OFF and the same reason" do + script = + models_and_memory("http://localhost:11434/v1", + embedder: :hosted, + hosted_model: "embedder" + ) <> @report + + {out, status} = boot("emb_localhost_def", [], script) + assert status == 0, String.slice(out, -3000, 3000) + assert line(out, "BOOT_OK") == "BOOT_OK" + + IO.puts("\n" <> line(out, "SEMANTIC")) + + assert line(out, "SEMANTIC") =~ + ~s(SEMANTIC {:off, {:config, {:embedder_locality_undeclared, :hosted, "http://localhost:11434/v1"}}}) + end + end + + describe "AC6: a corrupted byte in the weights" do + test "turns semantic memory OFF with :weights_digest_mismatch, and the application still boots" do + {root, _} = case_setup("weights_src") + digest = Trinity.StaticWeights.synthetic!(root) + file = Path.join(root, Trinity.Memory.Embedders.Static.file_name("256-int8")) + + memory = + "[embedder: :static, static_dir: #{inspect(root)}, static_sha256: #{inspect(digest)}]" + + script = """ + memory = Application.get_env(:trinity, :memory, []) + Application.put_env(:trinity, :memory, Keyword.merge(memory, #{memory})) + #{@report} + """ + + # The good file first: the same configuration is on, so the corruption is the only change. + {out, 0} = boot("weights_good", [], script) + assert line(out, "SEMANTIC") == "SEMANTIC :on" + + bin = File.read!(file) + at = div(byte_size(bin), 2) + <> = bin + File.write!(file, <>) + + {out, status} = boot("weights_bad", [], script) + assert status == 0, String.slice(out, -3000, 3000) + assert line(out, "BOOT_OK") == "BOOT_OK" + IO.puts("\n" <> line(out, "SEMANTIC")) + assert line(out, "SEMANTIC") == "SEMANTIC {:off, :weights_digest_mismatch}" + end + + @tag :static_weights + test "on the real artifact: one flipped byte, OFF with :weights_digest_mismatch, still boots" do + {root, _} = case_setup("weights_real") + name = Trinity.Memory.Embedders.Static.file_name("256-int8") + File.cp!(Trinity.StaticWeights.file!(name), Path.join(root, name)) + bin = File.read!(Path.join(root, name)) + at = div(byte_size(bin), 3) + <> = bin + File.write!(Path.join(root, name), <>) + + script = """ + memory = Application.get_env(:trinity, :memory, []) + Application.put_env(:trinity, :memory, Keyword.merge(memory, [embedder: :static, static_dir: #{inspect(root)}])) + #{@report} + """ + + {out, status} = boot("weights_real_bad", [], script) + assert status == 0, String.slice(out, -3000, 3000) + assert line(out, "SEMANTIC") == "SEMANTIC {:off, :weights_digest_mismatch}" + end + end +end diff --git a/test/trinity/eval/memory_eval_test.exs b/test/trinity/eval/memory_eval_test.exs new file mode 100644 index 0000000..743e7a7 --- /dev/null +++ b/test/trinity/eval/memory_eval_test.exs @@ -0,0 +1,243 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.Eval.MemoryEvalTest do + @moduledoc """ + Slice 133, AC14's harness: the corpus pipeline, the audit sample, the label freeze, and a + scored run that refuses until the labels are frozen. + + **None of this is the eval's scored run.** The corpus tests generate and inspect the real + corpus; the scoring test runs the harness on a six-memory fixture written here, whose labels + it freezes by writing the digest itself (no audit applies to a fixture). The real run waits + for the owner's audit of 100 labels (slice 133 NOTES). + """ + use ExUnit.Case, async: false + + alias Trinity.Eval.{Corpus, Labels, MemoryRun} + + @moduletag timeout: 300_000 + + setup do + dir = Path.join(System.tmp_dir!(), "eval-#{System.unique_integer([:positive])}") + File.mkdir_p!(dir) + on_exit(fn -> File.rm_rf(dir) end) + {:ok, dir: dir} + end + + describe "the corpus" do + setup do + {:ok, generated: Corpus.generate(20_261_007)} + end + + test "10^4 memories in the fixed proportions at every size, each size a prefix of the next", + %{generated: %{memories: memories}} do + assert length(memories) == 10_000 + + for {size, n} <- [{100, 100}, {1_000, 1_000}, {10_000, 10_000}] do + subset = Enum.filter(memories, &(&1.size <= size)) + assert length(subset) == n + assert subset == Enum.take(memories, n) + kinds = Enum.frequencies_by(subset, & &1.kind) + + assert kinds == %{ + "fact" => div(n, 4), + "preference" => div(n, 4), + "decision" => div(n, 5), + "observed" => div(n * 3, 10) + } + end + + # No text twice: a store whose observer dedupes does not hold one. + assert memories |> Enum.map(& &1.text) |> Enum.uniq() |> length() == 10_000 + + assert Enum.all?(memories, fn m -> + length(String.split(m.text, ~r/(?<=[.?!])\s/)) in 1..3 + end) + end + + test "300 queries, 40/30/20/10 by kind, every answer in the 10^2 core", %{ + generated: %{queries: queries} + } do + assert length(queries) == 300 + + assert Enum.frequencies_by(queries, & &1.kind) == + %{"paraphrase" => 120, "keyword" => 90, "multi_hop" => 60, "no_answer" => 30} + + core = for i <- 1..100, into: MapSet.new(), do: "m" <> String.pad_leading("#{i}", 5, "0") + + for q <- queries do + expected = %{"no_answer" => 0, "multi_hop" => 2}[q.kind] || 1 + assert length(q.targets) == expected, "#{q.id} has #{length(q.targets)} targets" + assert Enum.all?(q.targets, &MapSet.member?(core, &1)) + end + + assert queries |> Enum.map(& &1.text) |> Enum.uniq() |> length() == 300 + end + + test "the same seed gives the same corpus; another seed another one" do + assert Corpus.generate(7) == Corpus.generate(7) + refute Corpus.generate(7).memories == Corpus.generate(8).memories + end + end + + describe "the audit and the freeze" do + test "the audit draws 40/30/20/10; the freeze refuses until 100 boxes are ticked and an auditor named", + %{dir: dir} do + record = Labels.write!(dir, 20_261_007) + assert record["files"]["labels.jsonl"] == Labels.sha256_file(Path.join(dir, "labels.jsonl")) + audit = Labels.write_audit!(dir, 20_261_007) + text = File.read!(audit) + + kinds = Regex.scan(~r/^- \[ \] q\d+ \((\w+)\)/m, text) |> Enum.map(&List.last/1) + + assert Enum.frequencies(kinds) == + %{"paraphrase" => 40, "keyword" => 30, "multi_hop" => 20, "no_answer" => 10} + + assert {:error, :labels_not_frozen} = Labels.verify(dir) + assert {:error, {:audit_incomplete, 0, 100}} = Labels.freeze(dir) + + ticked = String.replace(text, "- [ ] q", "- [x] q") + File.write!(audit, ticked) + assert {:error, :auditor_unnamed} = Labels.freeze(dir) + refute File.exists?(Path.join(dir, "labels.sha256")) + + File.write!(audit, String.replace(ticked, "Audited by:", "Audited by: the owner")) + assert {:ok, digest} = Labels.freeze(dir) + assert digest == record["files"]["labels.jsonl"] + assert :ok = Labels.verify(dir) + + # A label changed after the freeze is caught. + File.write!(Path.join(dir, "labels.jsonl"), "{}\n", [:append]) + assert {:error, {:labels_changed, ^digest, _}} = Labels.verify(dir) + assert {:error, {:labels_changed, _, _}} = MemoryRun.run(dir, candidates: [{"fts5", :fts5}]) + end + + test "the scored run refuses labels that were never frozen", %{dir: dir} do + Labels.write!(dir, 1) + assert {:error, :labels_not_frozen} = MemoryRun.run(dir, candidates: [{"fts5", :fts5}]) + refute File.exists?(Path.join(dir, "results.json")) + end + end + + describe "the scored run, on a six-memory fixture" do + defp write_fixture!(dir) do + rows = fn name, list -> + File.write!(Path.join(dir, name), Enum.map(list, &[Jason.encode!(&1), "\n"])) + end + + rows.("corpus.jsonl", [ + %{id: "m1", kind: "fact", size: 100, text: "The person has a dog called Rex."}, + %{id: "m2", kind: "fact", size: 100, text: "The person moved to Lisbon in 2021."}, + %{id: "m3", kind: "preference", size: 100, text: "The person prefers green tea."}, + %{id: "m4", kind: "fact", size: 1_000, text: "Omar has a dog called Rex."}, + %{id: "m5", kind: "decision", size: 1_000, text: "The person decided to use SQLite."}, + %{id: "m6", kind: "observed", size: 1_000, text: "Anna reviews the thesis on Monday."} + ]) + + rows.("queries.jsonl", [ + %{id: "q1", kind: "keyword", text: "dog Rex"}, + %{id: "q2", kind: "keyword", text: "Lisbon"}, + %{id: "q3", kind: "no_answer", text: "harpsichord"} + ]) + + rows.("labels.jsonl", [ + %{query: "q1", targets: ["m1"]}, + %{query: "q2", targets: ["m2"]}, + %{query: "q3", targets: []} + ]) + + # A fixture has no audit: its digest is written directly, which only a test does. + File.write!( + Path.join(dir, "labels.sha256"), + Labels.sha256_file(Path.join(dir, "labels.jsonl")) <> " labels.jsonl\n" + ) + end + + test "full text alone, a dense leg fused by reciprocal rank, the intervals and the verdict", + %{ + dir: dir + } do + write_fixture!(dir) + + {:ok, results} = + MemoryRun.run(dir, + candidates: [ + {"fts5", :fts5}, + {"fake", {:embedder, Trinity.Memory.Embedders.Fake, []}} + ], + sizes: [100, 1_000], + resamples: 200 + ) + + at = fn size, cand, leg, metric -> results["sizes"][size][cand][leg][metric] end + + # At 10^2 "dog Rex" finds only m1, "Lisbon" m2: both answered, both first. + assert at.("100", "fts5", "hybrid", "recall@10")["mean"] == 1.0 + assert at.("100", "fts5", "hybrid", "mrr@10")["mean"] == 1.0 + # At 10^3 Omar's dog shares the words, and the target may no longer be first. + assert at.("1000", "fts5", "hybrid", "recall@10")["mean"] == 1.0 + assert at.("1000", "fts5", "hybrid", "mrr@10")["mean"] <= 1.0 + # The no-answer query found nothing in full text. + assert at.("100", "fts5", "hybrid", "no_answer_empty") == 1.0 + + # The fake's dense leg exists and its fused list still finds the keyword answers. + assert at.("100", "fake", "hybrid", "recall@10")["mean"] == 1.0 + assert is_map(at.("100", "fake", "dense_only", "recall@10")) + [lo, hi] = at.("100", "fake", "hybrid", "recall@10")["ci95"] + assert lo <= 1.0 and hi <= 1.0 + + # The verdict names its rule and does not pass on candidates that were not run. + assert results["verdict"]["static_floor_is_regulated_default"] == false + assert results["verdict"]["rule"] =~ "within 3.0 points" + + on_disk = Jason.decode!(File.read!(Path.join(dir, "results.json"))) + assert on_disk["labels_sha256"] == results["labels_sha256"] + refute File.read!(Path.join(dir, "results.json")) =~ "per_query" + end + + test "precomputed vectors are read in corpus order and must cover the corpus", %{dir: dir} do + write_fixture!(dir) + vdir = Path.join(dir, "vectors") + File.mkdir_p!(vdir) + vec = fn text -> Trinity.Memory.Embedders.Fake.vector(text, 32, "v") end + corpus = Labels.read!(dir, "corpus.jsonl") + queries = Labels.read!(dir, "queries.jsonl") + f32 = fn list -> for t <- list, x <- vec.(t), into: <<>>, do: <> end + File.write!(Path.join(vdir, "v.corpus.f32"), f32.(Enum.map(corpus, & &1["text"]))) + File.write!(Path.join(vdir, "v.queries.f32"), f32.(Enum.map(queries, & &1["text"]))) + + File.write!( + Path.join(vdir, "v.json"), + Jason.encode!(%{ + dim: 32, + corpus_ids: Enum.map(corpus, & &1["id"]), + query_ids: Enum.map(queries, & &1["id"]), + floor: 0.3 + }) + ) + + assert {:ok, %{"sizes" => %{"100" => %{"v" => %{"hybrid" => _}}}}} = + MemoryRun.run(dir, + candidates: [{"v", {:vectors, "v"}}], + sizes: [100], + resamples: 50 + ) + + File.write!( + Path.join(vdir, "v.json"), + Jason.encode!(%{dim: 32, corpus_ids: ["m1"], query_ids: [], floor: 0.3}) + ) + + assert_raise ArgumentError, ~r/does not cover/, fn -> + MemoryRun.run(dir, candidates: [{"v", {:vectors, "v"}}], sizes: [100], resamples: 50) + end + end + + test "the bootstrap is seeded and brackets the mean" do + values = [0.0, 1.0, 1.0, 0.0, 1.0, 1.0, 1.0, 0.0] + assert MemoryRun.bootstrap(values, 2_000, 1) == MemoryRun.bootstrap(values, 2_000, 1) + [lo, hi] = MemoryRun.bootstrap(values, 2_000, 1) + assert lo < 0.625 and 0.625 < hi + assert MemoryRun.bootstrap([], 10, 1) == [nil, nil] + end + end +end diff --git a/test/trinity/memory/embedder_config_test.exs b/test/trinity/memory/embedder_config_test.exs new file mode 100644 index 0000000..171e37a --- /dev/null +++ b/test/trinity/memory/embedder_config_test.exs @@ -0,0 +1,98 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.Memory.EmbedderConfigTest do + @moduledoc """ + Slice 133, D2: the decisions `Trinity.Memory.EmbedderConfig.check/4` makes, as a pure function. + These prove the decisions; the boots in `test/trinity/embedder_boot_node_test.exs` prove they are + asked, and asked before the node starts (AC5). + """ + use ExUnit.Case, async: true + + alias Trinity.Memory.EmbedderConfig + + defp models(url), do: [%{id: "e", base_url: url}] + defp hosted(extra), do: [embedder: :hosted, hosted_model: "e"] ++ extra + + @allow "https://models.internal" + + test "in-process embedders are in_process by construction; declaring another locality is a fault" do + for e <- [:static, :local, :fake, [:static, :local]], profile <- [:default, :regulated] do + assert :ok = EmbedderConfig.check(profile, [embedder: e], [], nil) + assert :ok = EmbedderConfig.check(profile, [embedder: e, locality: :in_process], [], nil) + end + + assert {:error, {:embedder_locality_mismatch, :static, :external}} = + EmbedderConfig.check(:default, [embedder: :static, locality: :external], [], nil) + end + + test "an endpoint with no locality is a fault under both profiles, whatever the address" do + for url <- ["http://localhost:11434/v1", "http://10.0.0.5/v1", "https://models.internal/v1"], + profile <- [:default, :regulated] do + assert {:error, {:embedder_locality_undeclared, :hosted, ^url}} = + EmbedderConfig.check(profile, hosted([]), models(url), @allow) + end + end + + test "in_process for an endpoint, or an unknown locality, is a fault" do + assert {:error, {:embedder_locality_in_process_with_endpoint, :hosted}} = + EmbedderConfig.check( + :default, + hosted(locality: :in_process), + models("http://x"), + nil + ) + + assert {:error, {:embedder_locality_invalid, :hosted, :enclave}} = + EmbedderConfig.check(:default, hosted(locality: :enclave), models("http://x"), nil) + end + + test "external needs the opt-in under both profiles; within_boundary does not" do + url = "https://models.internal/v1" + + for profile <- [:default, :regulated] do + assert {:error, {:external_not_opted_in, :hosted}} = + EmbedderConfig.check(profile, hosted(locality: :external), models(url), @allow) + + assert :ok = + EmbedderConfig.check( + profile, + hosted(locality: :external, external_opt_in: true), + models(url), + @allow + ) + + assert :ok = + EmbedderConfig.check( + profile, + hosted(locality: :within_boundary), + models(url), + @allow + ) + end + end + + test "under regulated only, the endpoint must be stated and allow-listed" do + off = models("https://embed.internal/v1") + + assert :ok = EmbedderConfig.check(:default, hosted(locality: :within_boundary), off, nil) + + assert {:error, {:not_allow_listed, :hosted, "https://embed.internal"}} = + EmbedderConfig.check(:regulated, hosted(locality: :within_boundary), off, @allow) + + assert {:error, {:not_allow_listed, :hosted, _}} = + EmbedderConfig.check( + :regulated, + hosted(locality: :external, external_opt_in: true), + off, + @allow + ) + + assert {:error, {:embedder_endpoint_unstated, :hosted}} = + EmbedderConfig.check( + :regulated, + hosted(locality: :within_boundary), + models(nil), + @allow + ) + end +end diff --git a/test/trinity/memory/embedder_test.exs b/test/trinity/memory/embedder_test.exs index 1a42e58..db54555 100644 --- a/test/trinity/memory/embedder_test.exs +++ b/test/trinity/memory/embedder_test.exs @@ -5,7 +5,8 @@ defmodule Trinity.Memory.EmbedderTest do Slice 032, G1 line 2: the fake's determinism, the tier's status by configuration, and the rule that the hosted embedder is never selected without the configuration (decision 2). """ - use ExUnit.Case, async: false + # Slice 133: the status reads the store's active space, so the tests run under the sandbox. + use Trinity.DataCase, async: false alias Trinity.Memory.{Embedder, Embedders, Semantic} diff --git a/test/trinity/memory/migration_test.exs b/test/trinity/memory/migration_test.exs new file mode 100644 index 0000000..6c1f370 --- /dev/null +++ b/test/trinity/memory/migration_test.exs @@ -0,0 +1,298 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.Memory.MigrationTest do + @moduledoc """ + Slice 133, AC1's storage half: the migration moves every `memories.embedding*` row into + `memory_embeddings`, and the count before equals the count after. + + On a database of its own (a file under the temporary directory on SQLite, a database named for + the run on Postgres), through a second instance of `Trinity.Repo` that is not the suite's + sandboxed one: migrated to the version before slice 133, given rows the way slice 032 wrote + them (two models, rows with and without vectors, rows in other tiers), migrated up, read, and + migrated down again. + """ + use ExUnit.Case, async: false + + alias Trinity.Memory.Space + + @before 20_260_924_120_000 + @this 20_261_008_120_000 + @minilm "bumblebee:sentence-transformers/all-MiniLM-L6-v2" + @fake "fake:sha256-384" + + # The migrator compiles the migration files again for each run in this VM, which the suite's + # own migration already loaded; the redefinition is expected, and its warning only noise. + setup_all do + previous = Code.get_compiler_option(:ignore_module_conflict) + Code.put_compiler_option(:ignore_module_conflict, true) + on_exit(fn -> Code.put_compiler_option(:ignore_module_conflict, previous) end) + end + + setup do + {repo_pid, config, drop} = start_repo!() + on_exit(drop) + Trinity.Repo.put_dynamic_repo(repo_pid) + on_exit(fn -> Trinity.Repo.put_dynamic_repo(Trinity.Repo) end) + {:ok, repo: repo_pid, config: config} + end + + defp postgres?, do: Application.get_env(:trinity, :db_adapter) == Ecto.Adapters.Postgres + + defp start_repo! do + base = Application.get_env(:trinity, Trinity.Repo) |> Keyword.drop([:pool, :name]) + name = "trinity_mig_#{System.unique_integer([:positive])}" + + config = + case Keyword.get(base, :url) do + url when is_binary(url) -> + uri = URI.parse(url) + [user, pass] = String.split(uri.userinfo, ":", parts: 2) + + base + |> Keyword.delete(:url) + |> Keyword.merge( + username: user, + password: pass, + hostname: uri.host, + port: uri.port, + database: name + ) + + _ -> + Keyword.put(base, :database, Path.join(System.tmp_dir!(), name <> ".db")) + end + + :ok = Trinity.Repo.__adapter__().storage_up(config) + + # `url: nil` matters: the repo merges its application config under these options and then + # lets a URL override the explicit keys, so the suite's DATABASE_URL would otherwise point + # this instance back at the suite's own database. + {:ok, pid} = + Trinity.Repo.start_link( + Keyword.merge(config, + url: nil, + name: nil, + pool: DBConnection.ConnectionPool, + pool_size: 2 + ) + ) + + # Guard the guard: this instance is on its own database, never the suite's. + %{rows: [[db]]} = database(pid) + if db != expected_name(config), do: raise("the migration test's repo is on #{db}") + + # Unlinked, so the test process's exit does not take it down before `on_exit` stops it. + Process.unlink(pid) + + drop = fn -> + Supervisor.stop(pid) + Trinity.Repo.__adapter__().storage_down(config) + end + + {pid, config, drop} + end + + defp database(pid) do + sql = + if Application.get_env(:trinity, :db_adapter) == Ecto.Adapters.Postgres, + do: "SELECT current_database()", + else: "SELECT file FROM pragma_database_list WHERE name = 'main'" + + Trinity.Repo.put_dynamic_repo(pid) + result = Trinity.Repo.query!(sql, []) + Trinity.Repo.put_dynamic_repo(Trinity.Repo) + result + end + + defp expected_name(config) do + if Application.get_env(:trinity, :db_adapter) == Ecto.Adapters.Postgres, + do: config[:database], + else: Path.expand(config[:database]) + end + + defp migrate(pid, direction, to) do + Ecto.Migrator.run(Trinity.Repo, Ecto.Migrator.migrations_path(Trinity.Repo), direction, + to: to, + dynamic_repo: pid, + log: false + ) + end + + # `?` placeholders, numbered for Postgres. + defp q(sql, params) do + sql = + if postgres?() do + {s, _} = + Enum.reduce(String.split(sql, "?"), {"", 0}, fn + part, {"", 0} -> {part, 0} + part, {acc, n} -> {acc <> "$#{n + 1}" <> part, n + 1} + end) + + s + else + sql + end + + Trinity.Repo.query!(sql, params) + end + + defp uuid(id), do: if(postgres?(), do: Ecto.UUID.dump!(id), else: id) + + defp vector(seed, dim), do: for(i <- 1..dim, do: :math.sin(seed * 1.7 + i)) + + defp f32(v), do: for(x <- v, into: <<>>, do: <>) + + # Slice 032's shape: a vector on the row, the model that made it, its width. + defp legacy_row!(persona, i, tier, model, at) do + id = Trinity.UUID.generate() + bytes = if model, do: f32(vector(i, 384)) + + q( + "INSERT INTO memories (id, persona_id, tier, scope, key, body, embedding, embedding_model, embedding_dim, inserted_at, updated_at) " <> + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + [ + uuid(id), + uuid(persona), + tier, + "persona:#{persona}", + "k-#{i}", + "body #{i}", + bytes, + model, + if(model, do: 384), + at, + at + ] + ) + + if postgres?() and model do + q("UPDATE memories SET embedding_vector = ? WHERE id = ?", [ + Pgvector.new(vector(i, 384)), + uuid(id) + ]) + end + + {id, bytes} + end + + defp ts(seconds) do + t = DateTime.add(~U[2026-09-30 00:00:00.000000Z], seconds, :second) + if postgres?(), do: DateTime.to_naive(t), else: DateTime.to_iso8601(t) + end + + test "AC1: every memories.embedding row moves into memory_embeddings; the count before equals the count after", + %{repo: pid} do + migrate(pid, :up, @before) + persona = Trinity.UUID.generate() + + q("INSERT INTO personas (id, name, inserted_at, updated_at) VALUES (?, ?, ?, ?)", [ + uuid(persona), + "migrated", + ts(0), + ts(0) + ]) + + # 40 MiniLM rows written earlier, 25 fake rows written later, and rows with no vector. + minilm = for i <- 1..40, do: legacy_row!(persona, i, "semantic", @minilm, ts(i)) + fake = for i <- 41..65, do: legacy_row!(persona, i, "semantic", @fake, ts(1_000 + i)) + for i <- 66..70, do: legacy_row!(persona, i, "semantic", nil, ts(i)) + for i <- 71..75, do: legacy_row!(persona, i, "always_on", nil, ts(i)) + + %{rows: [[before]]} = q("SELECT count(*) FROM memories WHERE embedding IS NOT NULL", []) + %{rows: [[memories_before]]} = q("SELECT count(*) FROM memories", []) + assert before == 65 + + migrate(pid, :up, @this) + + %{rows: [[after_count]]} = q("SELECT count(*) FROM memory_embeddings", []) + %{rows: [[memories_after]]} = q("SELECT count(*) FROM memories", []) + + IO.puts( + "\nAC1 migration: #{before} memories with a vector before, #{after_count} memory_embeddings rows after; memories #{memories_before} before, #{memories_after} after" + ) + + assert after_count == before + assert memories_after == memories_before + + # Each row under its legacy space, its bytes unchanged. + a = Space.id(Space.legacy(@minilm, 384)) + b = Space.id(Space.legacy(@fake, 384)) + + for {rows, space} <- [{minilm, a}, {fake, b}], {id, bytes} <- rows do + %{rows: [[stored, dim]]} = + q("SELECT vector, dim FROM memory_embeddings WHERE memory_id = ? AND space_id = ?", [ + uuid(id), + space + ]) + + assert stored == bytes and dim == 384 + end + + # Two spaces, the migration's manifests equal to Space.legacy/2's (the frozen copy holds), + # and the active one is the space of the most recently written vectors. + %{rows: spaces} = q("SELECT id, active FROM embedding_spaces ORDER BY id", []) + assert Enum.sort(Enum.map(spaces, &hd/1)) == Enum.sort([a, b]) + active = Map.new(spaces, fn [id, flag] -> {id, flag in [true, 1]} end) + assert active == %{a => false, b => true} + + # The old columns are gone. + cols = columns("memories") + refute "embedding" in cols or "embedding_model" in cols or "embedding_dim" in cols + + if postgres?() do + %{rows: [[type]]} = + q( + "SELECT format_type(atttypid, atttypmod) FROM pg_attribute WHERE attrelid = 'memory_embeddings'::regclass AND attname = 'embedding_vector'", + [] + ) + + assert type == "vector" + + %{rows: [[copied]]} = + q("SELECT count(*) FROM memory_embeddings WHERE embedding_vector IS NOT NULL", []) + + assert copied == before + + %{rows: indexes} = + q("SELECT indexname FROM pg_indexes WHERE tablename = 'memory_embeddings'", []) + + for space <- [a, b] do + assert [Trinity.Memory.Spaces.index_name(space)] in indexes + end + end + + # Down: the active space's vectors return to their rows; the tables go. + migrate(pid, :down, @before) + %{rows: [[back]]} = q("SELECT count(*) FROM memories WHERE embedding IS NOT NULL", []) + assert back == length(fake) + + %{rows: [[model]]} = + q("SELECT DISTINCT embedding_model FROM memories WHERE embedding IS NOT NULL", []) + + assert model == @fake + refute "memory_embeddings" in tables() + end + + test "an empty store migrates to no space and no active pointer", %{repo: pid} do + migrate(pid, :up, @this) + assert %{rows: [[0]]} = q("SELECT count(*) FROM embedding_spaces", []) + assert %{rows: [[0]]} = q("SELECT count(*) FROM memory_embeddings", []) + end + + defp columns(table) do + if postgres?() do + q("SELECT column_name FROM information_schema.columns WHERE table_name = ?", [table]).rows + |> List.flatten() + else + q("SELECT name FROM pragma_table_info(?)", [table]).rows |> List.flatten() + end + end + + defp tables do + if postgres?() do + q("SELECT tablename FROM pg_tables WHERE schemaname = 'public'", []).rows |> List.flatten() + else + q("SELECT name FROM sqlite_master WHERE type = 'table'", []).rows |> List.flatten() + end + end +end diff --git a/test/trinity/memory/no_nif_test.exs b/test/trinity/memory/no_nif_test.exs new file mode 100644 index 0000000..638274d --- /dev/null +++ b/test/trinity/memory/no_nif_test.exs @@ -0,0 +1,113 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.Memory.NoNifTest do + @moduledoc """ + Slice 133, AC7: embedding adds no NIF. Through the static space, 1,000 embeds (each memory + written embeds once) and 100 recalls over the 10^3 vectors they made leave the node with + exactly the native code it booted with. + + Asked where it lives: in a node of its own. This VM has run every other test, EXLA included, + so nothing it has loaded says anything about the static path. A child OS process boots + Trinity on its own databases (`Trinity.BootIsolation`), takes the snapshot, does the work, + takes it again, and prints both. + + The snapshot is two sets: the shared objects mapped into the process (`/proc/self/maps`: a NIF + is a shared object the VM loaded, and this is where every loaded one shows, whoever loaded it) + and the loaded modules of the applications the static path must never reach (`nx`, `exla`, + `xla`, `axon`, `bumblebee`, `tokenizers`). Linux only, which is every leg that runs it. + + The static embedder needs its weights, so this is `:static_weights`. + """ + use ExUnit.Case, async: false + + @moduletag :static_weights + @moduletag timeout: 300_000 + + @forbidden ~w(nx exla xla axon bumblebee tokenizers)a + + @child """ + Trinity.BootIsolation.isolate!(System.fetch_env!("TRINITY_BOOT_TAG")) + memory = Application.get_env(:trinity, :memory, []) + Application.put_env(:trinity, :memory, Keyword.merge(memory, embedder: :static, observer: false, + static_dir: System.fetch_env!("TRINITY_STATIC_MODEL_DIR"))) + {:ok, _} = Application.ensure_all_started(:trinity) + + forbidden = #{inspect(@forbidden)} + snapshot = fn -> + sos = + "/proc/self/maps" |> File.read!() |> String.split("\\n") + |> Enum.flat_map(fn line -> case Regex.run(~r{(/\\S+\\.so[\\.0-9]*)$}, line) do + [_, path] -> [path] + _ -> [] + end end) + |> Enum.uniq() |> Enum.sort() + mods = + for app <- forbidden, {:ok, ms} <- [:application.get_key(app, :modules)], m <- ms, + :erlang.module_loaded(m), do: m + {sos, Enum.sort(mods)} + end + + {sos0, mods0} = snapshot.() + {:ok, persona} = Trinity.Personas.create(%{name: "nif-check"}) + scope = Trinity.Memory.AlwaysOn.persona_scope(persona.id) + topics = ~w(coffee Lisbon dog sister budget garden piano bicycle river library) + for i <- 1..1000 do + body = "fact #{"#"}{i}: the person mentioned #{"#"}{Enum.at(topics, rem(i, 10))} and item #{"#"}{i * 7}" + {:ok, _} = Trinity.Memory.Semantic.add(%{persona_id: persona.id, scope: scope, key: "f#{"#"}{i}", body: body}, by: "test") + end + vectors = for i <- 1..100, reduce: 0 do + acc -> + hits = Trinity.Memory.Retriever.relevant(persona.id, nil, "what about #{"#"}{Enum.at(topics, rem(i, 10))}", touch: false) + acc + Enum.count(hits, &(&1.kind == :memory)) + end + {sos1, mods1} = snapshot.() + IO.puts("NIF_STATUS " <> inspect(Trinity.Memory.Semantic.status())) + IO.puts("NIF_VECTOR_HITS #{"#"}{vectors}") + IO.puts("NIF_SOS_ADDED " <> inspect(sos1 -- sos0, limit: :infinity)) + IO.puts("NIF_MODS_ADDED " <> inspect(mods1 -- mods0, limit: :infinity)) + IO.puts("NIF_SOS_COUNT #{"#"}{length(sos0)} #{"#"}{length(sos1)}") + System.halt(0) + """ + + defp line(out, prefix) do + out |> String.split("\n") |> Enum.find("", &String.starts_with?(&1, prefix)) + end + + test "AC7: the shared objects and forbidden-application modules are the same before and after" do + tag = Trinity.BootIsolation.tag("no_nif") + on_exit(fn -> Trinity.BootIsolation.drop!(tag) end) + root = Path.join(System.tmp_dir!(), "no-nif-#{System.unique_integer([:positive])}") + File.mkdir_p!(root) + on_exit(fn -> File.rm_rf(root) end) + + {out, status} = + System.cmd("mix", ["run", "--no-start", "-e", @child], + env: [ + {"MIX_ENV", "test"}, + {"XDG_DATA_HOME", root}, + {"TRINITY_BOOT_TAG", tag}, + {"TRINITY_STATIC_MODEL_DIR", Trinity.StaticWeights.dir!()}, + {"TRINITY_PROFILE", nil} + ], + stderr_to_stdout: true + ) + + assert status == 0, String.slice(out, -4000, 4000) + + IO.puts( + "\n" <> + Enum.map_join( + ~w(NIF_STATUS NIF_VECTOR_HITS NIF_SOS_COUNT NIF_SOS_ADDED NIF_MODS_ADDED), + "\n", + &line(out, &1) + ) + ) + + assert line(out, "NIF_STATUS") == "NIF_STATUS :on" + # The recalls ranked vectors: the static path was exercised, not skipped. + assert ["NIF_VECTOR_HITS", n] = String.split(line(out, "NIF_VECTOR_HITS")) + assert String.to_integer(n) > 0 + assert line(out, "NIF_SOS_ADDED") == "NIF_SOS_ADDED []" + assert line(out, "NIF_MODS_ADDED") == "NIF_MODS_ADDED []" + end +end diff --git a/test/trinity/memory/no_nif_xref_test.exs b/test/trinity/memory/no_nif_xref_test.exs new file mode 100644 index 0000000..9e4ae54 --- /dev/null +++ b/test/trinity/memory/no_nif_xref_test.exs @@ -0,0 +1,92 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.Memory.NoNifXrefTest do + @moduledoc """ + Slice 133, AC7's static half: the static embedder and the fallback scorer call nothing in + `nx`, `exla`, `xla`, `axon`, `bumblebee` or `tokenizers`, directly or through anything of + Trinity's they call. + + Erlang's `xref` over the compiled beams, as a closure: start from the static path's modules, + follow every call into another `Trinity` module, and collect every call that leaves Trinity. + The population is the call graph the compiler produced, not a list of modules someone typed; + the starting set is the static path's entry points (the embedder, its tokenizer and tables, its + weights file, the int8 scorer, the space encoding). The runtime half, a node whose loaded native + code does not change, is `test/trinity/memory/no_nif_test.exs`. + """ + use ExUnit.Case, async: true + + @forbidden ~w(nx exla xla axon bumblebee tokenizers)a + @roots [ + Trinity.Memory.Embedders.Static, + Trinity.Memory.WordPiece, + Trinity.Memory.WordPiece.Tables, + Trinity.Memory.StaticArtifact, + Trinity.Memory.Scorer, + Trinity.Memory.Space + ] + + setup_all do + name = :"xref_#{System.unique_integer([:positive])}" + {:ok, _} = :xref.start(name, xref_mode: :functions) + on_exit(fn -> :xref.stop(name) end) + :ok = :xref.set_default(name, warnings: false, verbose: false) + {:ok, _} = :xref.add_directory(name, ~c"#{:code.lib_dir(:trinity)}/ebin") + {:ok, xref: name} + end + + defp calls_out_of(xref, module) do + {:ok, calls} = :xref.q(xref, ~c"XC | '#{Atom.to_string(module)}' : Mod") + for {_from, {to, _f, _a}} <- calls, do: to + end + + defp trinity?(module), do: module |> Atom.to_string() |> String.starts_with?("Elixir.Trinity") + + defp closure(_xref, [], seen, outside), do: {seen, outside} + + defp closure(xref, [m | rest], seen, outside) do + targets = calls_out_of(xref, m) |> Enum.uniq() + {inside, out} = Enum.split_with(targets, &trinity?/1) + new = Enum.reject(inside, &MapSet.member?(seen, &1)) + + closure( + xref, + rest ++ new, + Enum.reduce(new, seen, &MapSet.put(&2, &1)), + MapSet.union(outside, MapSet.new(out)) + ) + end + + test "AC7: nothing the static path reaches calls into nx, exla, xla, axon, bumblebee or tokenizers", + %{xref: xref} do + {reached, outside} = closure(xref, @roots, MapSet.new(@roots), MapSet.new()) + + forbidden_modules = + for app <- @forbidden, + :ok == Application.ensure_loaded(app) or true, + {:ok, mods} <- [:application.get_key(app, :modules)], + m <- mods, + into: MapSet.new(), + do: m + + # The forbidden applications are loadable here (the test tree carries them), so an empty + # intersection is a statement about the calls, not about an empty list. + assert MapSet.member?(forbidden_modules, Nx) + assert MapSet.member?(forbidden_modules, EXLA.Backend) + + hits = MapSet.intersection(outside, forbidden_modules) + + named = + Enum.filter( + outside, + &(&1 |> Atom.to_string() =~ ~r/^Elixir\.(Nx|EXLA|Axon|Bumblebee|Tokenizers)/) + ) + + IO.puts( + "\nAC7 xref: #{MapSet.size(reached)} Trinity modules reached from the static path; " <> + "calls leave Trinity for #{MapSet.size(outside)} modules; forbidden among them: #{inspect(MapSet.to_list(hits))}" + ) + + assert MapSet.size(hits) == 0, "the static path calls #{inspect(MapSet.to_list(hits))}" + assert named == [] + end +end diff --git a/test/trinity/memory/observer_test.exs b/test/trinity/memory/observer_test.exs index 2b4bee0..27534d1 100644 --- a/test/trinity/memory/observer_test.exs +++ b/test/trinity/memory/observer_test.exs @@ -77,7 +77,9 @@ defmodule Trinity.Memory.ObserverTest do 6 ) - assert entry.embedding_model == "fake:sha256-384" + # Slice 133: the vector is a row of memory_embeddings under the fake's space. + assert Trinity.Repo.get_by!(Trinity.Memory.Vector, memory_id: entry.id).space_id == + Trinity.Memory.Space.id(Trinity.Memory.Embedders.Fake.space()) assert Enum.map(Semantic.entries(persona.id, [pscope]), & &1.id) |> Enum.sort() == Enum.sort([planted.id, entry.id]) diff --git a/test/trinity/memory/pgvector_spaces_test.exs b/test/trinity/memory/pgvector_spaces_test.exs new file mode 100644 index 0000000..e8de179 --- /dev/null +++ b/test/trinity/memory/pgvector_spaces_test.exs @@ -0,0 +1,161 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.Memory.PgvectorSpacesTest do + @moduledoc """ + Slice 133, AC2: on Postgres a 256-dimension space and a 384-dimension space coexist in one + table, and each search uses its own partial index, shown by `EXPLAIN`. + + Both spaces' vectors are in `memory_embeddings.embedding_vector`, an untyped `vector` column; + each space has its index, `((embedding_vector::vector()) vector_cosine_ops) WHERE space_id + = ''`. The plans are read for the exact statement a search runs + (`Trinity.Memory.VectorStores.Pgvector.sql/1`), with sequential scans disabled for the + statement so the planner shows whether the index **can** serve it: a partial index whose + predicate or expression did not match the query would leave a sequential scan in the plan + whatever the cost. + + `:postgres`: excluded on SQLite, run by the postgres job. + """ + use Trinity.DataCase, async: false + + @moduletag :postgres + + alias Trinity.{Factory, FakeEmbedderB, Repo} + alias Trinity.Memory.{AlwaysOn, Embedders.Fake, Entry, Semantic, Spaces, VectorStore} + alias Trinity.Memory.VectorStores.Pgvector, as: Store + + @rows 3_000 + + setup do + persona = Factory.persona!() + scope = AlwaysOn.persona_scope(persona.id) + + # Spaces of this test's own: the rows it writes are rolled back, and the dead entries they leave + # in an HNSW index cost recall to whatever test reads that index next (slice 133 NOTES, F3). + {:ok, a} = Spaces.register(space_a()) + {:ok, b} = Spaces.register(space_b()) + + now = DateTime.utc_now() + + for chunk <- Enum.chunk_every(1..@rows, 500) do + entries = + for i <- chunk do + %{ + id: Trinity.UUID.generate(), + persona_id: persona.id, + tier: "semantic", + scope: scope, + key: "k#{i}", + body: "fact #{i}", + inserted_at: now, + updated_at: now + } + end + + Repo.insert_all(Entry, entries) + + # Both spaces' vectors in the one untyped column, written as `Spaces.put_vector/5` writes + # them (bytes and the pgvector value), in bulk. + rows = + for e <- entries, + {space, row, v} <- [ + {space_a(), a, va(e.body)}, + {space_b(), b, vb(e.body)} + ] do + %{ + memory_id: Ecto.UUID.dump!(e.id), + space_id: row.id, + dim: space.dim, + vector: Trinity.Memory.Space.encode_vector(space, v), + embedding_vector: Pgvector.new(v), + inserted_at: now + } + end + + Repo.insert_all("memory_embeddings", rows) + end + + Repo.query!("ANALYZE memory_embeddings") + Repo.query!("ANALYZE memories") + {:ok, persona: persona, scope: scope, a: a, b: b} + end + + defp space_a, do: %{Fake.space() | model_id: "ac2:384"} + defp space_b, do: %{FakeEmbedderB.space() | model_id: "ac2:256"} + defp va(text), do: Fake.vector(text, 384, "ac2") + defp vb(text), do: Fake.vector(text, 256, "ac2") + + defp plan(space, query, persona, scope) do + {:ok, plan} = + Repo.transaction(fn -> + Repo.query!("SET LOCAL hnsw.iterative_scan = strict_order") + + Repo.query!("EXPLAIN " <> Store.sql(space), [ + Pgvector.new(query), + Ecto.UUID.dump!(persona.id), + [scope], + 5 + ]).rows + |> Enum.map_join("\n", &hd/1) + end) + + plan + end + + test "AC2: a 256 and a 384 space in one untyped column, each search planned on its own index", + %{persona: persona, scope: scope, a: a, b: b} do + %{rows: [[type]]} = + Repo.query!( + "SELECT format_type(atttypid, atttypmod) FROM pg_attribute WHERE attrelid = 'memory_embeddings'::regclass AND attname = 'embedding_vector'" + ) + + assert type == "vector" + + %{rows: dims} = + Repo.query!( + "SELECT space_id, vector_dims(embedding_vector), count(*) FROM memory_embeddings GROUP BY 1, 2 ORDER BY 2" + ) + + assert dims == [[b.id, 256, @rows], [a.id, 384, @rows]] + + plan_a = plan(a, va("fact 7"), persona, scope) + plan_b = plan(b, vb("fact 7"), persona, scope) + shown = &String.replace(&1, ~r/'\[[-0-9.,e]+\]'/, "'[query]'") + + IO.puts( + "\nAC2 EXPLAIN, the 384 space:\n#{shown.(plan_a)}\n\nAC2 EXPLAIN, the 256 space:\n#{shown.(plan_b)}" + ) + + assert plan_a =~ "Index Scan using #{Spaces.index_name(a.id)}" + refute plan_a =~ Spaces.index_name(b.id) + assert plan_b =~ "Index Scan using #{Spaces.index_name(b.id)}" + refute plan_b =~ Spaces.index_name(a.id) + + # And the searches answer from their own space. + {:ok, [hit_a | _]} = + VectorStore.search(va("fact 7"), 5, Semantic.filter(persona.id, [scope], a)) + + {:ok, [hit_b | _]} = + VectorStore.search( + vb("fact 7"), + 5, + Semantic.filter(persona.id, [scope], b) + ) + + assert {hit_a.entry.body, hit_a.space_id} == {"fact 7", a.id} + assert {hit_b.entry.body, hit_b.space_id} == {"fact 7", b.id} + end + + test "the database refuses a vector of the wrong width under a space's index", %{a: a} do + %{rows: [[memory_id]]} = Repo.query!("SELECT memory_id FROM memory_embeddings LIMIT 1") + + error = + catch_error( + Repo.query!( + "UPDATE memory_embeddings SET embedding_vector = $1 WHERE memory_id = $2 AND space_id = $3", + [Pgvector.new(List.duplicate(0.5, 256)), memory_id, a.id] + ) + ) + + assert Exception.message(error) =~ "expected 384 dimensions, not 256" + end +end diff --git a/test/trinity/memory/pinning_test.exs b/test/trinity/memory/pinning_test.exs new file mode 100644 index 0000000..f65adff --- /dev/null +++ b/test/trinity/memory/pinning_test.exs @@ -0,0 +1,105 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.Memory.PinningTest do + @moduledoc """ + Slice 133, AC3: the active space changes only through the operator's command. + + A store holding space A's vectors, with configuration then naming embedder B. Before this + slice the semantic leg returned nothing and said nothing: the status read `:on`, because the + status asked only whether B could embed, while every search filtered on B's model and found + none of A's rows. After it, semantic memory is OFF with a reason that names the mismatch, the + active pointer stays on A, and no re-embed job is enqueued. + """ + use Trinity.DataCase, async: false + + alias Trinity.Factory + alias Trinity.Memory.{AlwaysOn, Embedders, Retriever, Semantic, Space, Spaces} + + setup do + previous = Application.get_env(:trinity, :memory, []) + on_exit(fn -> Application.put_env(:trinity, :memory, previous) end) + persona = Factory.persona!() + {:ok, persona: persona, scope: AlwaysOn.persona_scope(persona.id), previous: previous} + end + + defp configure_b(previous) do + # Embedder B: the suite's fake with another seed and width, so another space. + Application.put_env( + :trinity, + :memory, + Keyword.merge(previous, embedder: :fake, fake_dim: 256, fake_seed: "b") + ) + end + + defp jobs(worker) do + Trinity.Repo.aggregate( + from(j in Oban.Job, where: j.worker == ^worker), + :count + ) + end + + test "AC3: pinned to A, configured B: OFF with a named reason, nothing enqueued", %{ + persona: persona, + scope: scope, + previous: previous + } do + {:ok, _} = + Semantic.add( + %{persona_id: persona.id, scope: scope, key: "rex", body: "the dog is called Rex"}, + by: "test" + ) + + assert Semantic.status() == :on + # The first write pinned the empty store to A, the fake's space. + a = Space.id(Embedders.Fake.space()) + assert Spaces.active().id == a + + configure_b(previous) + b = Space.id(Embedders.Fake.space()) + refute b == a + + # The reason names the mismatch, with both spaces: not :on, and not an unrelated reason. + assert {:off, {:space_mismatch, %{active: active, configured: [configured]}}} = + Semantic.status() + + assert {active, configured} == {Space.short(a), Space.short(b)} + + # The pointer did not move, B's space was not even registered, and no vector was written. + assert Spaces.active().id == a + assert Spaces.get(b) == nil + assert Spaces.count(a) == 1 + + # Recall still answers (full text), and the vector leg ranks nothing from either space. + hits = Retriever.relevant(persona.id, nil, "the dog is called Rex") + assert Enum.all?(hits, &(&1.kind != :memory)) + + assert jobs("Trinity.Memory.ReEmbedWorker") == 0 + + # A write while off is refused, and still moves nothing. + assert {:error, {:embedder_off, {:space_mismatch, _}}} = + Semantic.add(%{persona_id: persona.id, scope: scope, key: "k2", body: "another"}, + by: "test" + ) + + assert Spaces.active().id == a + assert Spaces.count(a) == 1 + end + + test "an empty store is pinned on its first write, and only then", %{ + persona: persona, + scope: scope, + previous: previous + } do + assert Spaces.active() == nil + # Before any write the configured embedder serves an empty store: on, nothing pinned yet. + assert Semantic.status() == :on + configure_b(previous) + assert Semantic.status() == :on + assert Spaces.active() == nil + + {:ok, _} = + Semantic.add(%{persona_id: persona.id, scope: scope, key: "k", body: "first"}, by: "test") + + assert Spaces.active().id == Space.id(Embedders.Fake.space()) + end +end diff --git a/test/trinity/memory/retier_test.exs b/test/trinity/memory/retier_test.exs new file mode 100644 index 0000000..0ddb474 --- /dev/null +++ b/test/trinity/memory/retier_test.exs @@ -0,0 +1,254 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.Memory.RetierTest do + @moduledoc """ + Slice 133, AC4: the operator's re-tier builds the new space beside the old; while it runs over + 10^4 memories, 1,000 concurrent queries each return results from exactly one space; the + pointer moves in one transaction when the new space is complete; the old space's rows remain + until `mix trinity.space.drop --confirm`, and the command without `--confirm` refuses. A + mixed-space row makes the retriever refuse rather than rank, shown by injecting one. + + Embedder A is the suite's fake (384 wide), B is `Trinity.FakeEmbedderB` (256 wide), and the + configuration names both, B first (`embedder: [Trinity.FakeEmbedderB, :fake]`): the store is + served by whichever writes the active space, so it answers from A while B is built and from B + once the pointer moves. + + The queries run in tasks against the sandbox's shared connection, so they interleave with the + builder's batch transactions rather than running beside them in parallel; the cutover is one + transaction on that same connection, which is what a query can or cannot observe half of. + """ + use Trinity.DataCase, async: false + + alias Trinity.{Factory, FakeEmbedderB, Repo} + + alias Trinity.Memory.{ + AlwaysOn, + Embedders.Fake, + Entry, + Retriever, + Semantic, + Space, + Spaces, + Vector + } + + @moduletag timeout: 600_000 + # The re-tier writes 10^4 vectors through the sandbox's one connection; on Postgres each also + # updates an HNSW index, which takes longer than the sandbox's default ownership. + @moduletag ownership_timeout: 600_000 + + @total 10_000 + @mine 100 + + setup do + previous = Application.get_env(:trinity, :memory, []) + + # A's fake is seeded apart from the suite's own, so its space (and on Postgres its HNSW + # index) is this test's alone: ten thousand rows rolled back leave ten thousand dead index + # entries behind, and an approximate index that other tests read exactly should not carry them. + Application.put_env( + :trinity, + :memory, + Keyword.merge(previous, embedder: [FakeEmbedderB, :fake], fake_seed: "retier") + ) + + on_exit(fn -> + Application.put_env(:trinity, :memory, previous) + Semantic.clear_fault(:all) + end) + + me = Factory.persona!() + others = for _ <- 1..9, do: Factory.persona!() + {:ok, a} = Spaces.pin_if_empty(Fake.space()) + seed!(me, others, a) + {:ok, me: me, scope: AlwaysOn.persona_scope(me.id), a: a} + end + + # 10^4 semantic memories with A's vectors: 100 of them the querying persona's, the rest spread + # over nine others. Inserted in bulk, as a store that had been running for a while would hold. + defp seed!(me, others, a) do + now = DateTime.utc_now() + owners = List.duplicate(me, @mine) ++ Enum.flat_map(others, &List.duplicate(&1, 1_100)) + + owners + |> Enum.with_index(1) + |> Enum.chunk_every(1_000) + |> Enum.each(fn chunk -> + entries = + for {p, i} <- chunk do + %{ + id: Trinity.UUID.generate(), + persona_id: p.id, + tier: "semantic", + scope: AlwaysOn.persona_scope(p.id), + key: "fact-#{i}", + body: "fact number #{i} about topic #{rem(i, 37)}", + inserted_at: now, + updated_at: now + } + end + + Repo.insert_all(Entry, entries) + + # As `Spaces.put_vector/5` writes them: the bytes, and on Postgres the pgvector value. + Repo.insert_all( + vector_table(), + for e <- entries do + v = Fake.vector(e.body) + + %{ + memory_id: Ecto.UUID.dump!(e.id), + space_id: a.id, + dim: 384, + vector: Space.encode_vector("f32", v), + inserted_at: now + } + |> pg_vector(v) + end + ) + end) + + assert Spaces.count(a.id) == @total + end + + defp postgres?, do: Application.get_env(:trinity, :db_adapter) == Ecto.Adapters.Postgres + + # On Postgres schemaless, to write the pgvector column the schema does not map; on SQLite + # through the schema, so the bytes are a BLOB (as `Spaces.put_vectors/4` writes them). + defp vector_table, do: if(postgres?(), do: "memory_embeddings", else: Vector) + + defp pg_vector(row, v) do + if postgres?(), + do: Map.put(row, :embedding_vector, Pgvector.new(v)), + else: Map.update!(row, :memory_id, &Ecto.UUID.load!/1) + end + + defp spaces_of(persona, scope, i) do + {:ok, hits} = + Semantic.search(persona.id, [scope], "fact number #{i} about topic #{rem(i, 37)}", 5) + + hits |> Enum.map(& &1.space_id) |> Enum.uniq() + end + + test "AC4: a re-tier beside the old space, 1,000 concurrent queries each answered from one space, then the cutover", + %{me: me, scope: scope, a: a} do + b_id = Space.id(FakeEmbedderB.space()) + test = self() + + # The builder pauses after its first batch until 500 queries have answered, so at least half + # the queries run while B is being built; the rest run on while it finishes and cuts over. + builder = + Task.async(fn -> + Spaces.retier(FakeEmbedderB, + batch: 500, + on_batch: fn + 500 -> + send(test, :building) + receive do: (:go -> :ok) + + _ -> + :ok + end + ) + end) + + assert_receive :building, 60_000 + assert Spaces.get(b_id).state == "building" + assert Spaces.active().id == a.id + + results = + 1..1_000 + |> Task.async_stream( + fn i -> + if i == 500, do: send(builder.pid, :go) + spaces_of(me, scope, i) + end, + max_concurrency: 16, + timeout: 120_000, + ordered: false + ) + |> Enum.map(fn {:ok, spaces} -> spaces end) + + assert {:ok, %{space: ^b_id, previous: previous, embedded: @total}} = + Task.await(builder, 240_000) + + assert previous == a.id + + # Each query answered, from exactly one space. + assert length(results) == 1_000 + assert Enum.all?(results, &match?([_], &1)), "a query saw no space or two spaces" + seen = results |> List.flatten() |> Enum.frequencies() + + IO.puts( + "\nAC4: 1,000 queries during the re-tier, by the space that answered: #{inspect(seen)}" + ) + + assert Map.get(seen, a.id, 0) >= 499 + assert Map.keys(seen) -- [a.id, b_id] == [] + + # The pointer moved, in one transaction: B active and complete, A kept, inactive, whole. + assert Spaces.active().id == b_id + assert Spaces.get(b_id).state == "complete" + assert Spaces.get(a.id).active == false + assert Spaces.count(a.id) == @total + assert Spaces.count(b_id) == @total + assert spaces_of(me, scope, 1) == [b_id] + + # The old space stays until the operator confirms. + assert {:error, :confirm_required} = Spaces.drop(a.id) + assert Spaces.count(a.id) == @total + + assert_raise Mix.Error, ~r/--confirm/, fn -> + Mix.Tasks.Trinity.Space.Drop.run([Space.short(a.id)]) + end + + assert Spaces.count(a.id) == @total + # The active space cannot be dropped, confirmed or not. + assert {:error, :active} = Spaces.drop(b_id, confirm: true) + + assert {:ok, %{vectors: @total}} = Spaces.drop(a.id, confirm: true) + assert Spaces.get(a.id) == nil + assert Spaces.count(a.id) == 0 + end + + test "AC4: an injected mixed-space row makes the retriever refuse rather than rank", %{ + me: me, + scope: scope, + a: a + } do + past = Factory.session!(%{persona_id: me.id, title: "Earlier"}) + Factory.message!(past.id, %{role: "user", content: "fact number 3 about topic 3 again"}) + + hits = Retriever.relevant(me.id, nil, "fact number 3 about topic 3", touch: false) + assert Enum.any?(hits, &(&1.kind == :memory)) + + # One row filed under the active space with another space's width. + {:ok, odd} = + %Entry{} + |> Entry.semantic_changeset(%{persona_id: me.id, scope: scope, key: "odd", body: "odd one"}) + |> Repo.insert() + + Repo.insert_all(Vector, [ + %{ + memory_id: odd.id, + space_id: a.id, + dim: 256, + vector: Space.encode_vector("f32", FakeEmbedderB.vector("odd one")), + inserted_at: DateTime.utc_now() + } + ]) + + assert {:error, {:mixed_space, 1}} = Semantic.search(me.id, [scope], "fact number 3", 5) + assert Semantic.status() == {:off, {:mixed_space, 1}} + + # Refused, not ranked: no memory hit at all, and the full-text half still answers. + hits = Retriever.relevant(me.id, nil, "fact number 3 about topic 3", touch: false) + refute Enum.any?(hits, &(&1.kind == :memory)) + assert Enum.any?(hits, &(&1.kind == :message)) + + # The row removed, the next search ranks again and the fault clears. + Repo.delete_all(from(v in Vector, where: v.memory_id == ^odd.id)) + assert {:ok, [_ | _]} = Semantic.search(me.id, [scope], "fact number 3 about topic 3", 5) + assert Semantic.status() == :on + end +end diff --git a/test/trinity/memory/runtime_faults_test.exs b/test/trinity/memory/runtime_faults_test.exs new file mode 100644 index 0000000..fa5fa02 --- /dev/null +++ b/test/trinity/memory/runtime_faults_test.exs @@ -0,0 +1,136 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.Memory.RuntimeFaultsTest do + @moduledoc """ + Slice 133, AC6's first half: a runtime fault turns semantic memory OFF with a reason, and + full-text recall goes on. + + The embedder here is the hosted one, through the real req_llm adapter, at an endpoint the test + serves itself on the loopback (Bandit on a random port), answering every request `503`. That + is the whole path an embedding request takes, adapter, retry and classification included, + with nothing leaving the machine. Its locality is declared `:within_boundary` (D2), so the + configuration is valid and the fault is a runtime one. + + AC6's second half (a corrupted byte in the weights, and the application still boots) is a boot, + and is in `test/trinity/embedder_boot_node_test.exs`. + """ + use Trinity.SessionCase + + alias Trinity.Factory + alias Trinity.Memory.{AlwaysOn, Embedders, Retriever, Semantic, Spaces} + + defmodule Unavailable do + @moduledoc false + import Plug.Conn + def init(opts), do: opts + + def call(conn, counter) do + Agent.update(counter, &(&1 + 1)) + conn |> put_resp_content_type("application/json") |> send_resp(503, ~s({"error":"down"})) + end + end + + setup do + {:ok, counter} = Agent.start_link(fn -> 0 end) + + {:ok, server} = + Bandit.start_link(plug: {Unavailable, counter}, ip: :loopback, port: 0, startup_log: false) + + {:ok, {_ip, port}} = ThousandIsland.listener_info(server) + + llm = Application.get_env(:trinity, :llm, []) + memory = Application.get_env(:trinity, :memory, []) + System.put_env("TRINITY_TEST_EMBED_KEY", "unused") + + Application.put_env( + :trinity, + :llm, + llm + |> Keyword.update!(:providers, &Map.put(&1, :req_llm, Trinity.LLM.Providers.ReqLLM)) + |> Keyword.update!(:models, fn models -> + models ++ + [ + %{ + id: "local:embed", + provider: :req_llm, + model: "openai_compatible:e5", + base_url: "http://127.0.0.1:#{port}/v1", + api_key_env: "TRINITY_TEST_EMBED_KEY", + caps: [:embed], + price: %{input: 0.0, output: 0.0} + } + ] + end) + ) + + Application.put_env( + :trinity, + :memory, + Keyword.merge(memory, + embedder: :hosted, + hosted_model: "local:embed", + hosted_dim: 8, + locality: :within_boundary + ) + ) + + on_exit(fn -> + Application.put_env(:trinity, :llm, llm) + Application.put_env(:trinity, :memory, memory) + System.delete_env("TRINITY_TEST_EMBED_KEY") + Semantic.clear_fault(:all) + end) + + persona = Factory.persona!() + # A store pinned to the hosted embedder's space, as an operator's re-tier would leave it. + Trinity.SpacesHelper.pin!(Embedders.Hosted.space()) + {:ok, persona: persona, counter: counter} + end + + test "AC6: an endpoint answering 503 is :endpoint_unreachable, and full-text hits still come back", + %{persona: persona, counter: counter} do + past = Factory.session!(%{persona_id: persona.id, title: "Earlier"}) + Factory.message!(past.id, %{role: "user", content: "my sister lives in Boston"}) + + # Configuration is valid and nothing has failed yet: the tier is on. + assert Semantic.status() == :on + active = Spaces.active().id + + hits = Retriever.relevant(persona.id, nil, "sister Boston", touch: false) + + # The endpoint was asked (and answered 503), the vector leg is empty, the message is found. + assert Agent.get(counter, & &1) > 0 + assert [%{kind: :message, found_by: [:fts]}] = hits + assert Semantic.status() == {:off, :endpoint_unreachable} + + assert Semantic.describe(Semantic.status()) == + "semantic recall is unavailable: the embedder's endpoint is unreachable" + + # A runtime fault moves nothing: the pointer is where it was and no job was enqueued. + assert Spaces.active().id == active + assert Trinity.Repo.aggregate(Oban.Job, :count) == 0 + + # The observer stands down while the tier is off. + assert Trinity.Memory.Observer.run( + %{session_id: past.id, persona_id: persona.id, model: nil}, + [] + ) == :off + + IO.puts( + "\nAC6: #{Agent.get(counter, & &1)} requests answered 503; status #{inspect(Semantic.status())}; hits #{inspect(Enum.map(hits, &{&1.kind, &1.found_by}))}" + ) + end + + test "a write while the endpoint is down is refused with the reason, and stores nothing", + %{persona: persona} do + scope = AlwaysOn.persona_scope(persona.id) + + assert {:error, :endpoint_unreachable} = + Semantic.add(%{persona_id: persona.id, scope: scope, key: "k", body: "a fact"}, + by: "test" + ) + + assert Semantic.entries(persona.id, [scope]) == [] + assert Spaces.count(Spaces.active().id) == 0 + end +end diff --git a/test/trinity/memory/spaces_test.exs b/test/trinity/memory/spaces_test.exs new file mode 100644 index 0000000..a411451 --- /dev/null +++ b/test/trinity/memory/spaces_test.exs @@ -0,0 +1,136 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.Memory.SpacesTest do + @moduledoc """ + Slice 133: the spaces' bookkeeping around AC3 and AC4: one active space held by the database, + ids resolved from the prefixes the commands print, a re-tier refused when it has nothing to do + or its embedder cannot serve, the commands' output, and the re-embed job as the one way a + running node re-tiers (enqueued by nobody but an operator). + """ + use Trinity.DataCase, async: false + + import ExUnit.CaptureIO + + alias Trinity.{Factory, FakeEmbedderB, Repo} + + alias Trinity.Memory.{ + AlwaysOn, + Embedders.Fake, + ReEmbedWorker, + Semantic, + Space, + SpaceRow, + Spaces + } + + setup do + previous = Application.get_env(:trinity, :memory, []) + on_exit(fn -> Application.put_env(:trinity, :memory, previous) end) + persona = Factory.persona!() + scope = AlwaysOn.persona_scope(persona.id) + + for i <- 1..3 do + {:ok, _} = + Semantic.add(%{persona_id: persona.id, scope: scope, key: "k#{i}", body: "fact #{i}"}, + by: "test" + ) + end + + {:ok, previous: previous, a: Spaces.active()} + end + + test "the database holds at most one active space", %{a: a} do + {:ok, b} = Spaces.register(FakeEmbedderB.space()) + + # Refused by the database itself (SQLite and Postgres name it differently). + error = + catch_error(Repo.update_all(from(s in SpaceRow, where: s.id == ^b.id), set: [active: true])) + + assert Exception.message(error) =~ + ~r/UNIQUE constraint failed: embedding_spaces.active|embedding_spaces_one_active/ + + assert Spaces.active().id == a.id + end + + test "register is idempotent; pin_if_empty refuses a store pinned elsewhere or holding vectors", + %{a: a} do + assert {:ok, %{id: id}} = Spaces.register(Fake.space()) + assert id == a.id + assert {:ok, ^a} = Spaces.pin_if_empty(Fake.space()) + other = Space.id(FakeEmbedderB.space()) + assert {:error, {:pinned_elsewhere, ^id}} = Spaces.pin_if_empty(FakeEmbedderB.space()) + + Repo.update_all(SpaceRow, set: [active: false]) + assert {:error, :store_not_empty} = Spaces.pin_if_empty(FakeEmbedderB.space()) + assert Spaces.get(other) == nil + # A store holding vectors with no active space serves nothing, and says so. + assert Semantic.status() == {:off, :no_active_space} + end + + test "resolve takes a full id or a unique prefix of twelve or more; drop refuses unknown ids", + %{ + a: a + } do + assert {:ok, %{id: id}} = Spaces.resolve(Space.short(a.id)) + assert id == a.id + assert {:error, :not_found} = Spaces.resolve("0123456789ab") + assert {:error, :not_found} = Spaces.resolve("short") + assert {:error, :not_found} = Spaces.drop(String.duplicate("0", 64), confirm: true) + refute Spaces.valid_id?("x'; DROP TABLE memories; --") + end + + test "a re-tier to the active space, or through an embedder that cannot serve, is refused", + %{previous: previous} do + assert {:error, :already_active} = Spaces.retier(Fake) + Application.put_env(:trinity, :memory, Keyword.put(previous, :hosted_model, "no:such")) + + assert {:error, {:embedder_unavailable, {:off, _}}} = + Spaces.retier(Trinity.Memory.Embedders.Hosted) + end + + test "the commands: list, retier, drop with and without --confirm", %{previous: previous, a: a} do + out = capture_io(fn -> Mix.Tasks.Trinity.Space.List.run([]) end) + assert out =~ "* #{Space.short(a.id)} complete 384d f32 3 vectors fake:sha256-384" + + Application.put_env( + :trinity, + :memory, + Keyword.put(previous, :embedder, [FakeEmbedderB, :fake]) + ) + + out = capture_io(fn -> Mix.Tasks.Trinity.Space.Retier.run([inspect(FakeEmbedderB)]) end) + b = Space.id(FakeEmbedderB.space()) + assert out =~ "space #{Space.short(b)} is active (3 embedded)" + assert out =~ "mix trinity.space.drop #{Space.short(a.id)} --confirm" + + assert_raise Mix.Error, ~r/active space/, fn -> + Mix.Tasks.Trinity.Space.Drop.run([Space.short(b), "--confirm"]) + end + + assert_raise Mix.Error, ~r/unknown_embedder/, fn -> + Mix.Tasks.Trinity.Space.Retier.run(["no-such-embedder"]) + end + + out = capture_io(fn -> Mix.Tasks.Trinity.Space.Drop.run([Space.short(a.id), "--confirm"]) end) + assert out =~ "dropped space #{Space.short(a.id)}: 3 vectors deleted" + end + + test "the re-embed job re-tiers a running node, and an already-active target is done", %{ + previous: previous + } do + Application.put_env( + :trinity, + :memory, + Keyword.put(previous, :embedder, [FakeEmbedderB, :fake]) + ) + + assert :ok = perform_job(ReEmbedWorker, %{"embedder" => inspect(FakeEmbedderB)}) + assert Spaces.active().id == Space.id(FakeEmbedderB.space()) + assert :ok = perform_job(ReEmbedWorker, %{"embedder" => inspect(FakeEmbedderB)}) + + assert {:error, {:unknown_embedder, "nope"}} = + perform_job(ReEmbedWorker, %{"embedder" => "nope"}) + end + + defp perform_job(worker, args), do: worker.perform(%Oban.Job{args: args}) +end diff --git a/test/trinity/memory/static_floor_test.exs b/test/trinity/memory/static_floor_test.exs new file mode 100644 index 0000000..5fe29ef --- /dev/null +++ b/test/trinity/memory/static_floor_test.exs @@ -0,0 +1,182 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.Memory.StaticFloorTest do + @moduledoc """ + Slice 133: the static floor's plumbing on every leg, over a synthetic artifact (the real file + format, a 33-token vocabulary, seeded random weights; `Trinity.StaticWeights.synthetic!/1`). + The real model's parity is AC9 and AC10, under `:static_weights`. + + What is held here: the artifact's format and its refusals; the embedder's availability + (missing, digest mismatch, unreadable); the stored int8 vector and the pure-Elixir scorer + against the float cosine; and the whole path, a memory written and recalled through the + static space with the int8 store. + """ + use Trinity.DataCase, async: false + + alias Trinity.{Factory, StaticWeights} + alias Trinity.Memory.{AlwaysOn, Embedder, Embedders.Static, Scorer, Semantic, Space, Spaces} + alias Trinity.Memory.StaticArtifact + + setup do + dir = Path.join(System.tmp_dir!(), "static-#{System.unique_integer([:positive])}") + previous = StaticWeights.use_synthetic!(dir) + + on_exit(fn -> + StaticWeights.restore!(previous) + File.rm_rf(dir) + end) + + {:ok, dir: dir} + end + + describe "the artifact" do + test "round-trips: header, vocabulary, per-row int8 with float32 scales, and f32" do + matrix = + for x <- [1.0, -2.0, 0.5, 0.0, 0.0, 0.0, 3.0, 1.5], into: <<>>, do: <> + + bin = + StaticArtifact.build(matrix, 2, 4, ["[UNK]", "a"], + dim: 2, + quantization: :int8, + provenance: %{"model" => "m"} + ) + + assert {:ok, a} = StaticArtifact.parse(bin) + assert {a.rows, a.dim, a.quantization, a.vocab} == {2, 2, :int8, ["[UNK]", "a"]} + assert a.header["model"] == "m" and a.header["source_dim"] == 4 + # Row 0 is [1.0, -2.0] truncated from four columns; its scale is 2/127. + [x, y] = StaticArtifact.row(a, 0) + assert_in_delta x, 1.0, 2 / 127 + assert_in_delta y, -2.0, 1.0e-6 + # A zero row stays zero. + assert StaticArtifact.row(a, 1) == [0.0, 0.0] + + {:ok, f} = + StaticArtifact.parse( + StaticArtifact.build(matrix, 2, 4, ["[UNK]", "a"], dim: 4, quantization: :f32) + ) + + assert StaticArtifact.row(f, 1) == [0.0, 0.0, 3.0, 1.5] + end + + test "refuses what is not a whole artifact, and inputs that do not agree" do + assert {:error, :not_an_artifact} = StaticArtifact.parse("nope") + matrix = :binary.copy(<<0::32>>, 8) + good = StaticArtifact.build(matrix, 2, 4, ["[UNK]", "a"], dim: 4, quantization: :int8) + + assert {:error, :truncated} = + StaticArtifact.parse(binary_part(good, 0, byte_size(good) - 1)) + + assert_raise ArgumentError, fn -> + StaticArtifact.build(matrix, 2, 4, ["[UNK]"], dim: 4, quantization: :int8) + end + + assert {:error, :not_safetensors} = StaticArtifact.read_safetensors("x") + end + + test "reads a single-tensor safetensors file" do + header = + Jason.encode!(%{"w" => %{"dtype" => "F32", "shape" => [2, 2], "data_offsets" => [0, 16]}}) + + data = for x <- [1.0, 2.0, 3.0, 4.0], into: <<>>, do: <> + file = <> <> header <> data + assert {:ok, ^data, 2, 2} = StaticArtifact.read_safetensors(file) + end + end + + describe "the embedder's availability" do + test "on with a matching digest; the space names the configured digest" do + assert Static.availability() == :ok + digest = Application.get_env(:trinity, :memory)[:static_sha256] + assert Static.space().weights_digest == digest + assert Static.space().quantization == "int8" + assert Static.space().locality == "in_process" + end + + test "a missing file is :weights_missing, a changed byte :weights_digest_mismatch", %{ + dir: dir + } do + file = Path.join(dir, Static.file_name("256-int8")) + bin = File.read!(file) + File.write!(file, <>) + Static.reload() + assert Static.availability() == {:off, :weights_digest_mismatch} + assert {:error, :weights_digest_mismatch} = Static.embed(["the cat"]) + + # Missing everywhere it is looked for: the real artifact's directory, where a run with the + # weights names one, would otherwise be found next. + env = System.get_env("TRINITY_STATIC_MODEL_DIR") + System.delete_env("TRINITY_STATIC_MODEL_DIR") + on_exit(fn -> if env, do: System.put_env("TRINITY_STATIC_MODEL_DIR", env) end) + File.rm!(file) + Static.reload() + assert Static.availability() == {:off, :weights_missing} + end + + test "the pinned digests are the real artifacts', and the space without configuration uses them" do + memory = Application.get_env(:trinity, :memory) + Application.put_env(:trinity, :memory, Keyword.delete(memory, :static_sha256)) + + assert Static.space().weights_digest == + "5990c1104963d8e2402854c80b9a8f8e3a490085c037f25ced63642f4578c520" + + # The synthetic file is not the pinned one, so the pinned digest refuses it. + Static.reload() + assert Static.availability() == {:off, :weights_digest_mismatch} + end + end + + describe "the int8 scorer" do + test "the int8 cosine tracks the float cosine, and the store's order is the float order" do + :rand.seed(:exsss, {1, 2, 3}) + vs = for _ <- 1..200, do: for(_ <- 1..256, do: :rand.normal()) + q = for _ <- 1..256, do: :rand.normal() + q8 = Scorer.quantize(q) + index = Scorer.prepare(Enum.with_index(vs, fn v, i -> {i, Scorer.quantize(v)} end)) + + exact = Scorer.exact(index, q8, 10) + float = vs |> Enum.with_index() |> Enum.map(fn {v, i} -> {i, Embedder.cosine(q, v)} end) + + for {id, score} <- exact do + {_, f} = List.keyfind(float, id, 0) + assert_in_delta score, f, 0.02 + end + + # The prefilter with every row as a candidate is the exact search. + assert Scorer.prefilter(index, q8, 10, 200) == exact + assert Scorer.hamming(Scorer.signs(q8), Scorer.signs(q8)) == 0 + assert Scorer.quantize([0.0, 0.0]) == <<0, 0>> + assert Scorer.cosine(<<0, 0>>, 0, <<1, 1>>, 2) == 0.0 + end + + test "the stored vector of an int8 space is 256 signed bytes, a float32 space's 4 bytes a component" do + v = for i <- 1..256, do: :math.sin(i) + assert byte_size(Space.encode_vector("int8", v)) == 256 + assert byte_size(Space.encode_vector("f32", v)) == 1024 + assert Space.vector_bytes("int8", 256) == 256 + back = Space.decode_vector("int8", Space.encode_vector("int8", v)) + assert Embedder.cosine(back, v) > 0.9999 + end + end + + describe "the whole path through the static space" do + test "a memory written and recalled: pinned to the static space, stored int8, ranked by the scorer" do + persona = Factory.persona!() + scope = AlwaysOn.persona_scope(persona.id) + + for {k, body} <- [{"rex", "the dog is called rex"}, {"tea", "the person likes tea"}] do + {:ok, _} = + Semantic.add(%{persona_id: persona.id, scope: scope, key: k, body: body}, by: "test") + end + + row = Spaces.active() + assert row.id == Space.id(Static.space()) + assert row.quantization == "int8" and row.dim == 256 + + assert {:ok, [first | _]} = Semantic.search(persona.id, [scope], "dog called rex", 2) + assert first.entry.key == "rex" + assert first.space_id == row.id + assert Semantic.thresholds() == %{floor: 0.3, dedupe: 0.98} + end + end +end diff --git a/test/trinity/memory/static_parity_test.exs b/test/trinity/memory/static_parity_test.exs new file mode 100644 index 0000000..2d11508 --- /dev/null +++ b/test/trinity/memory/static_parity_test.exs @@ -0,0 +1,109 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.Memory.StaticParityTest do + @moduledoc """ + Slice 133, AC10: the static embedder's vectors match the reference implementation + (sentence-transformers' `StaticEmbedding` over the upstream float32 weights) to cosine 0.9999 + or better on 500 fixtures, and a deliberately wrong pooling (sum rather than mean) fails. + + **Cosine alone cannot see the wrong pooling.** A sum over a bag is the mean times the number + of tokens: the same direction, so the same cosine to anything. The comparison is therefore of + the pooled vector before normalisation, as the reference returns it: its direction (cosine at + or over 0.9999) and its length (within 0.5 % of the reference's). The red run of this file + with the cosine check alone, where the sum passed, is in the slice's NOTES. + + The tolerance on length is the int8 quantization's: per-row int8 at 256 dimensions measured a + norm ratio between 0.9983 and 1.0010 on these fixtures (NOTES); the float32 variant at 1024 is + held to the same check and measures closer. + """ + use ExUnit.Case, async: false + + alias Trinity.Memory.{Embedder, Embedders.Static} + alias Trinity.StaticWeights + + @moduletag :static_weights + @moduletag timeout: 300_000 + + @min_cosine 0.9999 + @max_norm_error 0.005 + + defp norm(v), do: :math.sqrt(Enum.reduce(v, 0.0, &(&1 * &1 + &2))) + + @doc false + def check(elixir, reference) do + cos = Embedder.cosine(elixir, reference) + ratio = norm(elixir) / norm(reference) + {cos >= @min_cosine and abs(ratio - 1.0) <= @max_norm_error, cos, ratio} + end + + defp failures(variant_key, pooling) do + for %{"text" => text, ^variant_key => reference} <- + StaticWeights.fixtures!("embeddings.jsonl"), + {:ok, elixir} = Static.pooled_vector(text, pooling), + {ok?, cos, ratio} = check(elixir, reference), + not ok?, + do: {text, cos, ratio} + end + + describe "the 256-dimension int8 variant (the provisional default)" do + setup do + previous = StaticWeights.use_static!("256-int8") + on_exit(fn -> StaticWeights.restore!(previous) end) + end + + test "AC10: 500 fixtures, each at cosine >= 0.9999 with its length within 0.5 %" do + rows = StaticWeights.fixtures!("embeddings.jsonl") + assert length(rows) == 500 + + stats = + for %{"text" => t, "v256" => r, "ids" => ids} <- rows do + {:ok, got_ids} = Static.token_ids(t) + assert got_ids == ids, "token ids differ for #{inspect(t)}" + {:ok, v} = Static.pooled_vector(t) + check(v, r) + end + + {min_cos, min_ratio, max_ratio} = + {stats |> Enum.map(&elem(&1, 1)) |> Enum.min(), + stats |> Enum.map(&elem(&1, 2)) |> Enum.min(), + stats |> Enum.map(&elem(&1, 2)) |> Enum.max()} + + IO.puts( + "\nAC10 256-int8: min cosine #{min_cos}, norm ratio #{min_ratio} to #{max_ratio} over 500" + ) + + assert Enum.all?(stats, &elem(&1, 0)), + "#{Enum.count(stats, &(not elem(&1, 0)))} fixtures outside the check" + end + + test "AC10 red: sum pooling fails the same check" do + bad = failures("v256", :sum) + IO.puts("\nAC10 red: sum pooling fails #{length(bad)} of 500") + assert bad != [] + # It fails on length, never on direction: the case the cosine alone would have passed. + assert Enum.all?(bad, fn {_, cos, ratio} -> cos >= @min_cosine and ratio > 1.0 end) + end + + test "embed/1 is the pooled vector normalised, and a text with no tokens is the zero vector" do + {:ok, [v]} = Static.embed(["The cat sat on the mat."]) + {:ok, p} = Static.pooled_vector("The cat sat on the mat.") + assert_in_delta norm(v), 1.0, 1.0e-9 + assert_in_delta Embedder.cosine(v, p), 1.0, 1.0e-9 + assert length(v) == 256 + + {:ok, [z]} = Static.embed(["\u200B\u0000"]) + assert Enum.all?(z, &(&1 == 0.0)) + end + end + + describe "the 1024-dimension float32 variant (the eval's ceiling)" do + setup do + previous = StaticWeights.use_static!("1024-f32") + on_exit(fn -> StaticWeights.restore!(previous) end) + end + + test "AC10 at 1024 float32: 500 fixtures within the same check" do + assert failures("v1024", :mean) == [] + end + end +end diff --git a/test/trinity/memory/thresholds_test.exs b/test/trinity/memory/thresholds_test.exs new file mode 100644 index 0000000..e08dc12 --- /dev/null +++ b/test/trinity/memory/thresholds_test.exs @@ -0,0 +1,70 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.Memory.ThresholdsTest do + @moduledoc """ + Slice 133, AC11: thresholds are the space's own (`thresholds/0`), not a constant. + + The static model is a bag of words: a decision and its negation share every word but one, and + measure a cosine of 0.9703 (NOTES). MiniLM's dedupe value, 0.92, would call them the same + memory and drop the second, so the person's later decision would never be remembered. The + static space's own dedupe value keeps both, and still merges a true restatement. + """ + use Trinity.SessionCase + + alias Trinity.Factory + alias Trinity.LLM.Providers.Fake + alias Trinity.Memory.{AlwaysOn, Observer, Semantic} + alias Trinity.StaticWeights + + @moduletag :static_weights + + setup do + previous = StaticWeights.use_static!() + memory = Application.get_env(:trinity, :memory, []) + Application.put_env(:trinity, :memory, Keyword.put(memory, :observer, true)) + on_exit(fn -> StaticWeights.restore!(previous) end) + persona = Factory.persona!() + row = Factory.session!(%{persona_id: persona.id}) + {:ok, persona: persona, pscope: AlwaysOn.persona_scope(persona.id), row: row} + end + + defp run_observer(persona, row, bodies) do + u = Factory.message!(row.id, %{role: "user", content: "about the project database"}) + a = Factory.message!(row.id, %{role: "assistant", content: "Noted."}) + + Fake.object(%{ + "memories" => Enum.map(bodies, &%{"kind" => "decision", "body" => &1, "confidence" => 0.9}) + }) + + Observer.run( + %{session_id: row.id, persona_id: persona.id, model: nil}, + Enum.map([u, a], &%{id: &1.id, role: &1.role, content: &1.content}) + ) + end + + test "AC11: a decision and its negation stay two memories in the static space; a restatement merges", + %{persona: persona, pscope: pscope, row: row} do + {:ok, _} = + Semantic.add( + %{ + persona_id: persona.id, + scope: pscope, + key: "postgres", + body: "The person decided to use Postgres for the project." + }, + by: "test" + ) + + assert {:ok, kept} = + run_observer(persona, row, [ + "The person decided not to use Postgres for the project.", + "The person has decided to use Postgres for the project." + ]) + + bodies = Enum.map(kept, & &1.body) + assert "The person decided not to use Postgres for the project." in bodies + + refute "The person has decided to use Postgres for the project." in bodies, + "a restatement of an existing memory was kept as a new one" + end +end diff --git a/test/trinity/memory/vector_store_test.exs b/test/trinity/memory/vector_store_test.exs index 746ae1e..74d3432 100644 --- a/test/trinity/memory/vector_store_test.exs +++ b/test/trinity/memory/vector_store_test.exs @@ -3,13 +3,14 @@ defmodule Trinity.Memory.VectorStoreTest do @moduledoc """ Slice 032, AC1: 1,000 fake vectors through the store in force (`Brute` on SQLite, - `Pgvector` on the postgres job), the known nearest in order; the scope filter (M6) and the - model filter (NOTES decision 4) hold. + `Pgvector` on the postgres job), the known nearest in order; the scope filter (M6) holds. + Slice 133: vectors live in `memory_embeddings` under a space, a search ranks one space, and + another space's vectors of the same memory are never mixed in. """ use Trinity.DataCase, async: false alias Trinity.Factory - alias Trinity.Memory.{AlwaysOn, Embedder, Embedders.Fake, Semantic, VectorStore} + alias Trinity.Memory.{AlwaysOn, Embedder, Embedders.Fake, Semantic, Space, Spaces, VectorStore} setup do persona = Factory.persona!() @@ -21,13 +22,21 @@ defmodule Trinity.Memory.VectorStoreTest do e end + # The store's active space: the fake's, pinned by the first write of each test. + defp filter(persona_id, scopes), do: Semantic.filter(persona_id, scopes, Spaces.active()) + + defp search!(query, k, filter) do + {:ok, hits} = VectorStore.search(query, k, filter) + hits + end + test "AC1: 1,000 vectors, search/3 returns the known nearest with correct ordering", %{ persona: persona, scope: scope } do texts = for i <- 1..1_000, do: "fact number #{i}" ids = for {t, i} <- Enum.with_index(texts, 1), do: add!(persona, scope, "fact-#{i}", t).id - filter = Semantic.filter(persona.id, [scope]) + filter = filter(persona.id, [scope]) assert VectorStore.count(filter) == 1_000 # The expected order comes from the same cosine over the same vectors, computed here. @@ -40,7 +49,7 @@ defmodule Trinity.Memory.VectorStoreTest do |> Enum.sort_by(&elem(&1, 1), :desc) |> Enum.take(10) - hits = VectorStore.search(query, 10, filter) + hits = search!(query, 10, filter) assert Enum.map(hits, & &1.id) == Enum.map(expected, &elem(&1, 0)) assert hd(hits).entry.body == "fact number 500" assert_in_delta hd(hits).score, 1.0, 1.0e-5 @@ -64,22 +73,22 @@ defmodule Trinity.Memory.VectorStoreTest do b = add!(persona, "global", "a", "the same text") query = Fake.vector("the same text") - assert Enum.map(VectorStore.search(query, 5, Semantic.filter(persona.id, [scope])), & &1.id) == + assert Enum.map(search!(query, 5, filter(persona.id, [scope])), & &1.id) == [a.id] assert Enum.map( - VectorStore.search(query, 5, Semantic.filter(persona.id, ["global"])), + search!(query, 5, filter(persona.id, ["global"])), & &1.id ) == [b.id] assert Enum.sort( Enum.map( - VectorStore.search(query, 5, Semantic.filter(persona.id, [scope, "global"])), + search!(query, 5, filter(persona.id, [scope, "global"])), & &1.id ) ) == Enum.sort([a.id, b.id]) - assert VectorStore.search(query, 5, Semantic.filter(persona.id, [])) == [] + assert search!(query, 5, filter(persona.id, [])) == [] assert_raise FunctionClauseError, fn -> VectorStore.search(query, 5, %{persona_id: persona.id}) @@ -90,54 +99,59 @@ defmodule Trinity.Memory.VectorStoreTest do other = Factory.persona!() add!(other, AlwaysOn.persona_scope(other.id), "a", "shared words") - assert VectorStore.search( + assert search!( Fake.vector("shared words"), 5, - Semantic.filter(persona.id, [scope, AlwaysOn.persona_scope(other.id)]) + filter(persona.id, [scope, AlwaysOn.persona_scope(other.id)]) ) == [] end - test "vectors of another model are never mixed in (decision 4)", %{ + test "another space's vector of the same memory is never mixed in (slice 133)", %{ persona: persona, scope: scope } do e = add!(persona, scope, "a", "one text") query = Fake.vector("one text") - assert [%{id: id}] = VectorStore.search(query, 5, Semantic.filter(persona.id, [scope])) - assert id == e.id + a = Spaces.active() + assert [%{id: id, space_id: space_id}] = search!(query, 5, filter(persona.id, [scope])) + assert {id, space_id} == {e.id, a.id} - :ok = VectorStore.upsert(e.id, query, "other:model-384") - assert VectorStore.search(query, 5, Semantic.filter(persona.id, [scope])) == [] - assert VectorStore.count(Semantic.filter(persona.id, [scope])) == 0 + # The same memory in a second space, 256 wide: two rows, one per space. + other = %{Fake.space() | model_id: "other:model", dim: 256} + {:ok, b} = Spaces.register(other) + :ok = Spaces.put_vector(e.id, b.id, other, Enum.take(query, 256)) - assert VectorStore.count(%{persona_id: persona.id, scopes: [scope], model: "other:model-384"}) == - 1 - end + assert [%{space_id: ^space_id}] = search!(query, 5, filter(persona.id, [scope])) - test "delete/1 clears the vector and the row stays; upsert on a missing row is an error", %{ - persona: persona, - scope: scope - } do - e = add!(persona, scope, "a", "one text") - :ok = VectorStore.delete(e.id) - assert %{embedding: nil, embedding_model: nil, embedding_dim: nil} = Trinity.Repo.reload!(e) + assert [%{space_id: b_id}] = + search!(Enum.take(query, 256), 5, Semantic.filter(persona.id, [scope], b)) + + assert b_id == b.id + assert VectorStore.count(filter(persona.id, [scope])) == 1 + assert VectorStore.count(Semantic.filter(persona.id, [scope], b)) == 1 - assert VectorStore.search(Fake.vector("one text"), 5, Semantic.filter(persona.id, [scope])) == - [] + # A vector of another width is refused at the write, never stored under the space. + assert {:error, {:wrong_width, 384, 256}} = Spaces.put_vector(e.id, b.id, other, query) + end - assert {:error, :no_such_entry} = - VectorStore.upsert(Ecto.UUID.generate(), Fake.vector("x"), "fake") + test "deleting a memory deletes its vectors in every space", %{persona: persona, scope: scope} do + e = add!(persona, scope, "a", "one text") + assert Spaces.count(Spaces.active().id) == 1 + {:ok, _} = Semantic.remove(e, by: "test") + assert Spaces.count(Spaces.active().id) == 0 + assert search!(Fake.vector("one text"), 5, filter(persona.id, [scope])) == [] end - test "the row records the embedder that produced its vector, float32 little-endian", %{ + test "the vector row records its space, its width and its bytes, float32 little-endian", %{ persona: persona, scope: scope } do e = add!(persona, scope, "a", "one text") - assert e.embedding_model == "fake:sha256-384" - assert e.embedding_dim == 384 - assert byte_size(e.embedding) == 384 * 4 - stored = Embedder.from_binary(e.embedding) + row = Trinity.Repo.get_by!(Trinity.Memory.Vector, memory_id: e.id) + assert row.space_id == Space.id(Fake.space()) + assert row.dim == 384 + assert byte_size(row.vector) == 384 * 4 + stored = Space.decode_vector("f32", row.vector) for {x, y} <- Enum.zip(stored, Fake.vector("one text")), do: assert_in_delta(x, y, 1.0e-6) end @@ -158,9 +172,14 @@ defmodule Trinity.Memory.VectorStoreTest do assert Trinity.Smoke.probe([]) == [] end - test "the EXLA product and the Elixir cosine agree, and a row of another width scores 0.0 on both", + test "the EXLA product and the Elixir cosine agree on a float32 space", %{persona: persona, scope: scope} do - rows = for i <- 1..50, do: add!(persona, scope, "k#{i}", "text #{i}") + rows = + for i <- 1..50 do + e = add!(persona, scope, "k#{i}", "text #{i}") + {e, Space.encode_vector("f32", Fake.vector("text #{i}"))} + end + query = Fake.vector("text 7") elixir = Trinity.Memory.VectorStores.Brute.elixir_scores(rows, query) assert Enum.max(elixir) > 0.999 @@ -169,9 +188,6 @@ defmodule Trinity.Memory.VectorStoreTest do if Code.ensure_loaded?(EXLA.Backend) and Trinity.Memory.Embedders.Bumblebee.exla() == :ok do exla = Trinity.Memory.VectorStores.Brute.exla_scores(rows, query) for {a, b} <- Enum.zip(elixir, exla), do: assert_in_delta(a, b, 1.0e-5) - odd = %{hd(rows) | embedding: Embedder.to_binary([1.0, 0.0])} - assert [first | _] = Trinity.Memory.VectorStores.Brute.exla_scores([odd | tl(rows)], query) - assert first == 0.0 else IO.puts("\nEXLA not loaded here: the Elixir path is the one in force") end diff --git a/test/trinity/memory/word_piece_parity_test.exs b/test/trinity/memory/word_piece_parity_test.exs new file mode 100644 index 0000000..1f89653 --- /dev/null +++ b/test/trinity/memory/word_piece_parity_test.exs @@ -0,0 +1,94 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.Memory.WordPieceParityTest do + @moduledoc """ + Slice 133, AC9: the pure-Elixir WordPiece gives the ids the reference tokenizer gives, on + 5,000 natural sentences and a 1,000-item adversarial set (accents, CJK, emoji, zero-width + joiners, control characters, words over 100 characters, the special tokens' literals). One + mismatched id fails it. + + The reference is Hugging Face `tokenizers` at the version `scripts/static/reference_fixtures.py` + records, over the static model's own `tokenizer.json`, encoded without special tokens as + sentence-transformers' `StaticEmbedding` encodes. Fixtures and vocabulary live in the model + cache, not the tree (NOTES, D7 and decision 12), so this file is `:static_weights`. + + The last test is the red the slice names: the same comparison with accent stripping broken + must fail, or the set could not tell a correct tokenizer from a careless one. + """ + use ExUnit.Case, async: false + + alias Trinity.Memory.{Embedders.Static, WordPiece} + alias Trinity.StaticWeights + + @moduletag :static_weights + @moduletag timeout: 300_000 + + setup_all do + previous = StaticWeights.use_static!() + on_exit(fn -> StaticWeights.restore!(previous) end) + {:ok, tok} = Static.tokenizer() + {:ok, tok: tok} + end + + defp mismatches(tok, rows, opts \\ []) do + for %{"text" => text, "ids" => expected} <- rows, + (got = WordPiece.encode(tok, text, opts)) != expected, + do: {text, expected, got} + end + + defp report(mismatches) do + mismatches + |> Enum.take(5) + |> Enum.map_join("\n", fn {t, e, g} -> + " #{inspect(t)}\n reference #{inspect(e)}\n elixir #{inspect(g)}" + end) + end + + test "AC9: 5,000 natural sentences, every id identical", %{tok: tok} do + rows = StaticWeights.fixtures!("tokenizer-natural.jsonl") + assert length(rows) == 5_000 + bad = mismatches(tok, rows) + assert bad == [], "#{length(bad)} of 5,000 differ:\n#{report(bad)}" + end + + test "AC9: 1,000 adversarial items, every id identical", %{tok: tok} do + rows = StaticWeights.fixtures!("tokenizer-adversarial.jsonl") + assert length(rows) == 1_000 + bad = mismatches(tok, rows) + assert bad == [], "#{length(bad)} of 1,000 differ:\n#{report(bad)}" + + # The set exercises what it claims to: count the categories in the inputs themselves. + texts = Enum.map(rows, & &1["text"]) + assert Enum.any?(texts, &String.contains?(&1, "\u200D")), "no zero-width joiner" + assert Enum.any?(texts, &(&1 =~ ~r/[\x{4E00}-\x{9FFF}]/u)), "no CJK ideograph" + assert Enum.any?(texts, &(&1 =~ ~r/[\x{1F300}-\x{1FAFF}]/u)), "no emoji" + + assert Enum.any?(texts, &(&1 =~ ~r/[\x{0}-\x{8}\x{B}\x{C}\x{E}-\x{1F}\x{7F}]/u)), + "no control character" + + assert Enum.any?(texts, &(&1 =~ ~r/\p{Mn}/u or &1 =~ ~r/[àáâãäéèêëíóöúüñç]/u)), + "no accent" + + assert Enum.any?(texts, fn t -> + t |> String.split(~r/\s+/u) |> Enum.any?(&(String.length(&1) > 100)) + end), + "no word over 100 characters" + end + + test "AC9 red: with accent stripping broken, the adversarial set fails", %{tok: tok} do + # Lowercase each code point and keep everything else, marks included: the BERT pipeline + # with its NFD-and-drop-marks step left out, which is the defect the slice names. + broken = fn text -> + for <>, into: "" do + if cp in [?\t, ?\n, ?\r], do: " ", else: String.downcase(<>) + end + end + + rows = StaticWeights.fixtures!("tokenizer-adversarial.jsonl") + bad = mismatches(tok, rows, normalize: broken) + IO.puts("\nAC9 red: #{length(bad)} of 1,000 adversarial items differ with accents kept") + assert bad != [] + # And accents are what it got wrong: a mismatch on an accented word is among them. + assert Enum.any?(bad, fn {t, _, _} -> t =~ ~r/[àáâãäéèêëíóöúüñçÀÉ]|\p{Mn}/u end) + end +end diff --git a/test/trinity/memory/word_piece_test.exs b/test/trinity/memory/word_piece_test.exs new file mode 100644 index 0000000..eec26e0 --- /dev/null +++ b/test/trinity/memory/word_piece_test.exs @@ -0,0 +1,73 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.Memory.WordPieceTest do + @moduledoc """ + Slice 133: the pure-Elixir WordPiece's rules on a hand-written vocabulary, so they run on every + leg (AC9's parity against the reference needs the model's vocabulary, which is not in the tree, + and is `test/trinity/memory/word_piece_parity_test.exs`). Each expectation below is what the + reference does: each was checked against `tokenizers` 0.23.2 over the same vocabulary and the + same pipeline (slice 133 NOTES). + """ + use ExUnit.Case, async: true + + alias Trinity.Memory.WordPiece + + @vocab ~w([PAD] [UNK] [CLS] [SEP] [MASK] the cat sat un ##aff ##able ##s cafe resume , . ! + hello world 野 口 i a ##b x ##x) + + setup_all do + {:ok, tok: WordPiece.new(@vocab)} + end + + defp t(tok, text), do: WordPiece.tokens(tok, text) + + test "lowercases, strips accents, splits punctuation, matches longest pieces", %{tok: tok} do + assert t(tok, "The CAT sat.") == ~w(the cat sat .) + assert t(tok, "Café, résumé!") == ~w(cafe , resume !) + # NFD and the marks dropped, whether the accent is precomposed or combining. + assert t(tok, "cafe\u0301") == ~w(cafe) + assert t(tok, "unaffable cats") == ~w(un ##aff ##able cat ##s) + end + + test "an unmatched remainder makes the whole word one [UNK]; so does a word over 100 characters", + %{tok: tok} do + assert t(tok, "unaffz") == ~w([UNK]) + assert t(tok, "zebra cat") == ~w([UNK] cat) + assert t(tok, String.duplicate("x", 100)) == ["x" | List.duplicate("##x", 99)] + assert t(tok, String.duplicate("x", 101)) == ~w([UNK]) + end + + test "control characters go, whitespace of every kind splits, CJK ideographs stand alone", + %{tok: tok} do + # The controls go before splitting, so the two words become one ("helloworld": no "##world"). + assert t(tok, "hello\u0000\u200Bworld") == ~w([UNK]) + assert t(tok, "hello\u00A0world\u3000cat") == ~w(hello world cat) + assert t(tok, "野口cat") == ~w(野 口 cat) + assert WordPiece.normalize("a\tb\r\nc") == "a b c" + end + + test "special tokens are matched in the raw text before normalisation, leftmost and longest", + %{tok: tok} do + assert t(tok, "[CLS]the cat[SEP]") == ~w([CLS] the cat [SEP]) + # The text after a special token is a word of its own: "b", not a continuation "##b". + assert t(tok, "a[MASK]b") == ~w(a [MASK] [UNK]) + # Lowercase is not the special token: it is normalised and split as punctuation. + assert t(tok, "[cls]") == ~w([UNK] [UNK] [UNK]) + end + + test "lowercasing is per character, so a final sigma stays sigma (the reference's rule)" do + assert WordPiece.normalize("ΣΟΦΟΣ") == "σοφοσ" + assert WordPiece.normalize("İ") == "i" + assert WordPiece.normalize("한") == "한" + end + + test "a vocabulary without [UNK] is refused; a vocab file is read in line order" do + assert_raise ArgumentError, fn -> WordPiece.new(~w(a b)) end + path = Path.join(System.tmp_dir!(), "vocab-#{System.unique_integer([:positive])}.txt") + File.write!(path, Enum.join(@vocab, "\n") <> "\n") + on_exit(fn -> File.rm(path) end) + assert {:ok, tok} = WordPiece.from_vocab_file(path) + assert WordPiece.encode(tok, "the cat") == [5, 6] + assert {:error, :enoent} = WordPiece.from_vocab_file(path <> ".missing") + end +end diff --git a/test/trinity/scheduler/curator_test.exs b/test/trinity/scheduler/curator_test.exs index e5b6ff9..81b304d 100644 --- a/test/trinity/scheduler/curator_test.exs +++ b/test/trinity/scheduler/curator_test.exs @@ -85,6 +85,8 @@ defmodule Trinity.Memory.CuratorTest do scope: scope } do vector = Enum.map(1..8, fn _ -> 0.1 end) + space = Trinity.Memory.Space.legacy("fake:embed", 8) + {:ok, row} = Trinity.Memory.Spaces.register(space) {:ok, e} = %Entry{} @@ -94,21 +96,15 @@ defmodule Trinity.Memory.CuratorTest do scope: scope, key: "cat", body: "the cat is grey", - confidence: 0.9, - embedding: Trinity.Memory.Embedder.to_binary(vector), - embedding_model: "fake:embed", - embedding_dim: 8 + confidence: 0.9 }) |> Repo.insert() - assert [_] = Semantic.entries(persona.id, [scope]) + :ok = Trinity.Memory.Spaces.put_vector(e.id, row.id, space, vector) + filter = Semantic.filter(persona.id, [scope], row) - assert [_] = - Trinity.Memory.VectorStores.Brute.search(vector, 5, %{ - persona_id: persona.id, - scopes: [scope], - model: "fake:embed" - }) + assert [_] = Semantic.entries(persona.id, [scope]) + assert {:ok, [_]} = Trinity.Memory.VectorStores.Brute.search(vector, 5, filter) Repo.update_all(from(x in Entry, where: x.id == ^e.id), set: [archived_at: DateTime.utc_now()] @@ -116,12 +112,7 @@ defmodule Trinity.Memory.CuratorTest do assert [] = Semantic.entries(persona.id, [scope]) - assert [] = - Trinity.Memory.VectorStores.Brute.search(vector, 5, %{ - persona_id: persona.id, - scopes: [scope], - model: "fake:embed" - }) + assert {:ok, []} = Trinity.Memory.VectorStores.Brute.search(vector, 5, filter) end test "the observer enqueues a memory job carrying the message ids and not their text; the job runs run/2", diff --git a/test/trinity/tools/recall_test.exs b/test/trinity/tools/recall_test.exs index 23a4526..3241f84 100644 --- a/test/trinity/tools/recall_test.exs +++ b/test/trinity/tools/recall_test.exs @@ -94,6 +94,11 @@ defmodule Trinity.Tools.RecallTest do ) ) + # Slice 133: a store pinned to the local model's space, on a machine without the model (the + # store's fake vectors stay under their own space). A store pinned to the fake's space would + # be off for another reason, the mismatch, which `Trinity.Memory.PinningTest` holds. + Trinity.SpacesHelper.pin!(Trinity.Memory.Embedders.Bumblebee.space()) + call = %{id: "c5", name: "recall", args: %{"query" => "#near:dog what is my dog called"}} assert {:ok, %{content: text, meta: meta}, _} = Effects.Runner.run(call, ctx) diff --git a/test/trinity_web/live/semantic_tab_test.exs b/test/trinity_web/live/semantic_tab_test.exs index ac07c51..b5dff7d 100644 --- a/test/trinity_web/live/semantic_tab_test.exs +++ b/test/trinity_web/live/semantic_tab_test.exs @@ -41,7 +41,7 @@ defmodule TrinityWeb.SemanticTabTest do view |> element("#tab-semantic") |> render_click() assert_patch(view, ~p"/memory?persona_id=#{persona.id}&tab=semantic") html = render(view) - assert html =~ "semantic recall is on (fake:sha256-384)" + assert html =~ "semantic recall is on (fake:sha256-384, space " refute html =~ "Download the model" refute html =~ "Nothing kept in this tier." assert has_element?(view, "#semantic-#{entry.id}", "has a dog named Rex") @@ -115,6 +115,9 @@ defmodule TrinityWeb.SemanticTabTest do Keyword.merge(old, embedder: :local, model_cache_dir: cache) ) + # Slice 133: the store pinned to the local model's space (see recall_test.exs). + Trinity.SpacesHelper.pin!(Trinity.Memory.Embedders.Bumblebee.space()) + {:ok, view, _} = live(conn, ~p"/memory?persona_id=#{persona.id}&tab=semantic") html = render(view)