From d8957bd6d59662a1c920b8214d206d7478687b64 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 19 Oct 2026 21:25:09 +0000 Subject: [PATCH 1/4] board: record #1618 and #1619 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR_ARC_INVENTORY entries for #1618 (post-merge recovery for #1601-#1617, merged dbe0696f) and #1619 (D-RMAP binding palette generation and the §7 versioning crosswalk, merged 8e004a20), and a LATEST_STATE section for main after #1619. Contract inventory delta: none. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01VyWZYKycy6MoYDxHRVnV5K --- .claude/board/LATEST_STATE.md | 6 ++++++ .claude/board/PR_ARC_INVENTORY.md | 14 ++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/.claude/board/LATEST_STATE.md b/.claude/board/LATEST_STATE.md index 4335dea7d..d9df18795 100644 --- a/.claude/board/LATEST_STATE.md +++ b/.claude/board/LATEST_STATE.md @@ -1,3 +1,9 @@ +## 2026-10-19 — main after #1619 (merged `8e004a20`) + +- **On main:** #1618 (`dbe0696f`, post-merge recovery for #1601–#1617 plus the remediation) and #1619 (`8e004a20`, D-RMAP binding palette generation and D-RMAP-1A §7 versioning crosswalk). Both have arc entries. +- **In PR:** #1620, the MORK/PathMap Schnellverschluss probe (D-MPS-0..7). +- Contract inventory delta: none. + ## 2026-10-19 — main after #1617 (merged `47697c77`); post-merge recovery in PR - **On main since #1597:** #1601–#1617 (17 PRs: research notes, plans, a qualia archive, the ACT-R crate, the mask-risc hop, the D-RMAP plans). None had board rows; the arc entries, STATUS_BOARD rows and plan index entries are added in this PR. diff --git a/.claude/board/PR_ARC_INVENTORY.md b/.claude/board/PR_ARC_INVENTORY.md index 8ec3a0196..5688b4e7b 100644 --- a/.claude/board/PR_ARC_INVENTORY.md +++ b/.claude/board/PR_ARC_INVENTORY.md @@ -1,3 +1,17 @@ +## 2026-10-19 — lance-graph PR #1619 (merged `8e004a20`, branch `ccr-08f48cec-lbv9l4`, head `79baa045`, 3 commits) — D-RMAP palette generation in the binding; versioning derived from Thought Git and #1605 + +- **Added:** D-RMAP-1A §7 (`.claude/plans/d-rmap-separate-tables-thought-palettes-v1.md`). It is an ownership crosswalk: map and palette generations are readings of `DurableRef` (sealed) or `ThoughtCommitId` (speculative), and the thought-palette binding is `(ThoughtCommitId, palette generation)`. Focus generation and the index seal are retained witnesses, and authorization scope is a separate key. §7 also specifies falsifiers X1–X9, a cost/entropy comparison, and the Schnellverschluss note. The PR also adds gap notes in `thought-git-api-v1.md` and the causal-pickaxe plan, STATUS_BOARD row D-RMAP-1A-V, ENTROPY-MILESTONES M28, and the board entry `2026-10-19-d-rmap-thought-git-versioning.md`. `a7e075d3`: `ThoughtMapPaletteBinding` stores the palette generation it bound; `act-r.yml` also runs on `rust-toolchain.toml` changes. +- **Locked:** none. Status PROPOSED. CE64 is unchanged. There is no new executor, store, type or per-fold commit, and the no-gather ruling stands. +- **Deferred / open:** `ThoughtCommitId` must be never-reused, which #1605 does not say. A thought needs a dirty bit. A multi-dataset write needs one `DurableRef` per dataset, or a refusal (the measured instance is `versioned.rs::commit_encounter_round`). `DurableRef` has no type until CPG-00 runs. X1–X9 are not run. +- **Confidence:** doc only. The code facts in §7.2 were read in source. CI caught a line-number citation (`citation-decay`); it was fixed in `79baa045` by citing symbols. + +## 2026-10-19 — lance-graph PR #1618 (merged `dbe0696f`, branch `ccr-08f48cec-lbv9l4`, head `b2c5453c`, 5 commits) — post-merge recovery for #1601–#1617 + +- **Added:** board rows for #1601–#1617 (arc entries, STATUS_BOARD, plan index, entry `2026-10-19-post-merge-recovery-1601-1617.md`). Remediation: the ACT-R crate renamed to `lance-graph-act-r`, with `Refusal::StaleCandidate` and its own CI job; #1616's `OutgoingHop` quarantined behind the off-by-default `quarantined-gather-hop` feature (`ISS-GATHER-HOP-1616`); verified review findings applied across #1603–#1613; D-RMAP tile and binding keys carry map and palette generations. +- **Locked:** the gather hop stays quarantined while the 2026-10-16 no-gather ruling holds. +- **Deferred / open:** the palette generation of a thought's binding was left to the follow-up, which landed in #1619. +- **Confidence:** the ACT-R refusal and the quarantine are pinned by disable-verified tests. The rest is documentation. + ## 2026-10-19 — lance-graph PR #1617 (merged `47697c77`, branch `design/relationship-map-implicit-register-v1`, head `55be2fed`, 7 commits) — D-RMAP: separate RelationshipMap tables and thought-selected palettes - **Added:** plans `.claude/plans/d-rmap-implicit-relationship-register-v1.md` (D-RMAP-1), `d-rmap-separate-tables-thought-palettes-v1.md` (D-RMAP-1A, the controlling contract for ownership and CE64 opacity) and `d-rmap-2-context-tile-algebra-probe-v1.md` (D-RMAP-2, a preregistered GraphBLAS vs BF16 tile comparison); entry `2026-10-19-d-rmap-implicit-relationship-register.md`; entries README row. Nothing implemented. From fcc4bd4c615fe0eaa42b1421ebfa422ca0ad2261 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 19 Oct 2026 23:07:55 +0000 Subject: [PATCH 2/4] board: correct the #1618 and #1619 arc entries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two Codex findings on #1621, both checked against the plan text: - #1619: map generation is the sealed RelationshipMap/RelationFacts DurableRef only; ThoughtCommitId applies to palette generation alone (D-RMAP-1A §7.3). - #1618: at dbe0696f the thought-palette binding still keyed only MapPaletteId; #1618 added the map generation to the tile key and a palette generation to MapPalette. The binding change is #1619's. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01VyWZYKycy6MoYDxHRVnV5K --- .claude/board/PR_ARC_INVENTORY.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.claude/board/PR_ARC_INVENTORY.md b/.claude/board/PR_ARC_INVENTORY.md index 5688b4e7b..305a81e17 100644 --- a/.claude/board/PR_ARC_INVENTORY.md +++ b/.claude/board/PR_ARC_INVENTORY.md @@ -1,13 +1,13 @@ ## 2026-10-19 — lance-graph PR #1619 (merged `8e004a20`, branch `ccr-08f48cec-lbv9l4`, head `79baa045`, 3 commits) — D-RMAP palette generation in the binding; versioning derived from Thought Git and #1605 -- **Added:** D-RMAP-1A §7 (`.claude/plans/d-rmap-separate-tables-thought-palettes-v1.md`). It is an ownership crosswalk: map and palette generations are readings of `DurableRef` (sealed) or `ThoughtCommitId` (speculative), and the thought-palette binding is `(ThoughtCommitId, palette generation)`. Focus generation and the index seal are retained witnesses, and authorization scope is a separate key. §7 also specifies falsifiers X1–X9, a cost/entropy comparison, and the Schnellverschluss note. The PR also adds gap notes in `thought-git-api-v1.md` and the causal-pickaxe plan, STATUS_BOARD row D-RMAP-1A-V, ENTROPY-MILESTONES M28, and the board entry `2026-10-19-d-rmap-thought-git-versioning.md`. `a7e075d3`: `ThoughtMapPaletteBinding` stores the palette generation it bound; `act-r.yml` also runs on `rust-toolchain.toml` changes. +- **Added:** D-RMAP-1A §7 (`.claude/plans/d-rmap-separate-tables-thought-palettes-v1.md`). It is an ownership crosswalk: map generation is a reading of the sealed RelationshipMap/RelationFacts `DurableRef` only; palette generation is the palette table's `DurableRef` once sealed, or the `ThoughtCommitId` of a speculative palette edit; and the thought-palette binding is `(ThoughtCommitId, palette generation)`. Focus generation and the index seal are retained witnesses, and authorization scope is a separate key. §7 also specifies falsifiers X1–X9, a cost/entropy comparison, and the Schnellverschluss note. The PR also adds gap notes in `thought-git-api-v1.md` and the causal-pickaxe plan, STATUS_BOARD row D-RMAP-1A-V, ENTROPY-MILESTONES M28, and the board entry `2026-10-19-d-rmap-thought-git-versioning.md`. `a7e075d3`: `ThoughtMapPaletteBinding` stores the palette generation it bound; `act-r.yml` also runs on `rust-toolchain.toml` changes. - **Locked:** none. Status PROPOSED. CE64 is unchanged. There is no new executor, store, type or per-fold commit, and the no-gather ruling stands. - **Deferred / open:** `ThoughtCommitId` must be never-reused, which #1605 does not say. A thought needs a dirty bit. A multi-dataset write needs one `DurableRef` per dataset, or a refusal (the measured instance is `versioned.rs::commit_encounter_round`). `DurableRef` has no type until CPG-00 runs. X1–X9 are not run. - **Confidence:** doc only. The code facts in §7.2 were read in source. CI caught a line-number citation (`citation-decay`); it was fixed in `79baa045` by citing symbols. ## 2026-10-19 — lance-graph PR #1618 (merged `dbe0696f`, branch `ccr-08f48cec-lbv9l4`, head `b2c5453c`, 5 commits) — post-merge recovery for #1601–#1617 -- **Added:** board rows for #1601–#1617 (arc entries, STATUS_BOARD, plan index, entry `2026-10-19-post-merge-recovery-1601-1617.md`). Remediation: the ACT-R crate renamed to `lance-graph-act-r`, with `Refusal::StaleCandidate` and its own CI job; #1616's `OutgoingHop` quarantined behind the off-by-default `quarantined-gather-hop` feature (`ISS-GATHER-HOP-1616`); verified review findings applied across #1603–#1613; D-RMAP tile and binding keys carry map and palette generations. +- **Added:** board rows for #1601–#1617 (arc entries, STATUS_BOARD, plan index, entry `2026-10-19-post-merge-recovery-1601-1617.md`). Remediation: the ACT-R crate renamed to `lance-graph-act-r`, with `Refusal::StaleCandidate` and its own CI job; #1616's `OutgoingHop` quarantined behind the off-by-default `quarantined-gather-hop` feature (`ISS-GATHER-HOP-1616`); verified review findings applied across #1603–#1613; the D-RMAP `RelationTile` key carries the map generation (or index seal) and `MapPalette` gained its own palette generation; the thought-palette binding did not yet carry it. - **Locked:** the gather hop stays quarantined while the 2026-10-16 no-gather ruling holds. - **Deferred / open:** the palette generation of a thought's binding was left to the follow-up, which landed in #1619. - **Confidence:** the ACT-R refusal and the quarantine are pinned by disable-verified tests. The rest is documentation. From 636396f5e37cf6cf2136c9d1237fbc84f5de7546 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 19 Oct 2026 23:31:05 +0000 Subject: [PATCH 3/4] D-RMAP focus probe: focus-bound RelationshipMap over existing indexes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Measures the D-RMAP-1A §6 economics on a probe, with nothing adopted: a 256-slot focus over 65 536 identities, bound to an existing sorted run or PathMap trie, prepared into a slot-space CSR and a 256x256 tile per predicate, and evaluated through the existing mask-risc executor. No gather, no copied fact, no new opcode, store or CE64 field; address translation happens only in preparation, and out-of-focus targets are reported as escapes rather than dropped. - crates/mork-pathmap-probe/src/rmap.rs: SortedRun, Focus, Csr, Tile, ThoughtDelta, incremental rebind, place focus masks and register lanes, membership planes for postings, witness refusals. - boundary.rs: SpogTrie (PathMap, p|s|o) and PostingTrie. - tests/rmap.rs: 12 tests against an independent scan oracle; 11 guards disable-verified. - examples/rmap_cost.rs: amortization at N = 1..1 000 000, churn k = 1..256, 1/8/64 thoughts, bytes. Measured (release, warm, medians of 5): break-even reuses 76 / 11 / 1.8 for sparse degree 2 / degree 8 / dense; the tile beats CSR only on the dense relation; incremental rebind loses at k = 256. Recorded in D-RMAP-1A §8, STATUS_BOARD D-RMAP-1A-P and a board entry. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01VyWZYKycy6MoYDxHRVnV5K --- .claude/board/STATUS_BOARD.md | 1 + .../entries/2026-10-19-d-rmap-focus-probe.md | 17 + .claude/board/entries/README.md | 3 +- .../d-rmap-2-context-tile-algebra-probe-v1.md | 2 + ...map-separate-tables-thought-palettes-v1.md | 66 + crates/mork-pathmap-probe/Cargo.toml | 3 + .../mork-pathmap-probe/examples/rmap_cost.rs | 503 ++++++ crates/mork-pathmap-probe/src/boundary.rs | 124 +- crates/mork-pathmap-probe/src/lib.rs | 1 + crates/mork-pathmap-probe/src/rmap.rs | 1432 +++++++++++++++++ crates/mork-pathmap-probe/tests/rmap.rs | 536 ++++++ 11 files changed, 2686 insertions(+), 2 deletions(-) create mode 100644 .claude/board/entries/2026-10-19-d-rmap-focus-probe.md create mode 100644 crates/mork-pathmap-probe/examples/rmap_cost.rs create mode 100644 crates/mork-pathmap-probe/src/rmap.rs create mode 100644 crates/mork-pathmap-probe/tests/rmap.rs diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index 74903c79d..c13c6dfba 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -24,6 +24,7 @@ These PRs merged with no board rows. Every row below is a PROPOSAL (documents an | D-RMAP-1 | implicit RelationshipMap and focus-relative registers (`d-rmap-implicit-relationship-register-v1.md`) | PROPOSAL (superseded on ownership by D-RMAP-1A) | falsifier matrix F0–F10 in the plan | | D-RMAP-1A | separate map tables, borrowed projections, thought-selected palettes (`d-rmap-separate-tables-thought-palettes-v1.md`) | PROPOSAL; generation witnesses corrected 2026-10-19 (§6a) | refusal on map regeneration, palette-slot reassignment, focus-slot reuse, incarnation reuse, stale tile, cross-tenant change | | D-RMAP-1A-V | versioning crosswalk: map/palette/binding generations derived from D-TGA `ThoughtCommitId` + #1605 `DurableRef`; focus generation and index seal retained (`d-rmap-separate-tables-thought-palettes-v1.md` §7) | PROPOSED; no code, no type; falsifiers specified, none run | X1–X9 in §7.4 (cross-dataset, compaction, palette ABA, slot reuse, uncommitted change, rebase, authorization, squash+replay, deletion/DRed) | +| D-RMAP-1A-P | focus-bound RelationshipMap probe over existing sorted/trie indexes: CSR, 256×256 tile via mask-risc, incremental rebind, thought deltas, place masks, postings (`d-rmap-separate-tables-thought-palettes-v1.md` §8; `crates/mork-pathmap-probe/src/rmap.rs`) | MEASURED, probe only; nothing adopted | 12 tests vs scan oracle; 11 guards disable-verified; break-even 76/11/1.8 reuses (sparse d2 / d8 / dense) | | D-RMAP-2 | context-declaration reuse and GraphBLAS/BF16 tile probe (`d-rmap-2-context-tile-algebra-probe-v1.md`) | PROPOSAL; not run | arms 1–5 in the plan; 297× (D-GTM-0j) is not evidence | | D-SRAM-0 | inventory Soufflé AST/ast2ram/RAM passes against D-ASM/D-RA/D-TGA owners | Queued | evidence per mapped pass; no code imported | | D-SRAM-1 | bound AST → RAM-like schedule for pure conjunctive selection | Queued | same answers and read sets on hand-built fixtures; unsupported ops typed-refused | diff --git a/.claude/board/entries/2026-10-19-d-rmap-focus-probe.md b/.claude/board/entries/2026-10-19-d-rmap-focus-probe.md new file mode 100644 index 000000000..3d4a438cb --- /dev/null +++ b/.claude/board/entries/2026-10-19-d-rmap-focus-probe.md @@ -0,0 +1,17 @@ +# D-RMAP focus probe: a focus-bound RelationshipMap over existing indexes (2026-10-19) + +**Status:** MEASURED (D-RMAP-1A-P), probe only. Record: `.claude/plans/d-rmap-separate-tables-thought-palettes-v1.md` §8. Code: `crates/mork-pathmap-probe/src/rmap.rs`, `tests/rmap.rs`, `examples/rmap_cost.rs`. + +## Measured + +- A 256-slot focus over 65 536 identities, prepared into a slot-space CSR (and a 256×256 tile per predicate), answers 1–3-hop scoped reach through the existing mask-risc executor with no gather and no copied fact. 12 tests equal a full-slice scan; 11 guards disable-verified. +- Break-even reuse count (arithmetic from measured medians): 76 (sparse, degree 2), 11 (degree 8), 1.8 (dense). The tile beats CSR only on the dense relation. +- Reporting escapes as source slots instead of identities cut the degree-8 CSR eval from 50.4 to 5.6 µs. +- Incremental rebind beats full re-preparation for small k and loses at k = 256 (dense: from k ≥ 16). +- 1/8/64 thoughts over one shared tile: flat per-eval cost; 32 B per thought as sparse edits, 16 KiB as executor planes. +- PathMap: prefix seek beats the sorted run on sparse relations; trie build is 4–6× slower and keys are ≥ 3× the bytes; `restrict` is exact but slower than CSR preparation. MORK matching not exercised. +- Place focus: zero-copy counting over canonical lanes is 20–24 µs (word-granular gate); a 3 072 B register copy counts in ~1–2 µs. + +## Open + +- Lance-backed `DurableRef` and stable-row-id seals; cold-cache measurement; bit-granular `under` gating; any gather admission. diff --git a/.claude/board/entries/README.md b/.claude/board/entries/README.md index c3bc75011..70aa89da5 100644 --- a/.claude/board/entries/README.md +++ b/.claude/board/entries/README.md @@ -25,7 +25,7 @@ index row, (3) no duplicate entry id. Checks 1 and 2 are deliberately opposite directions; the stranding this convention prevents shows up in exactly one of them, never both. -372 entries, 2026-08-06 .. 2026-10-19. +373 entries, 2026-08-06 .. 2026-10-19. | date | entry id | finding | file | |---|---|---|---| @@ -33,6 +33,7 @@ exactly one of them, never both. | 2026-10-19 | `mork-pathmap-schnellverschluss` | | [2026-10-19-mork-pathmap-schnellverschluss.md](2026-10-19-mork-pathmap-schnellverschluss.md) | | 2026-10-19 | `D-RMAP-1A` | | [2026-10-19-d-rmap-thought-git-versioning.md](2026-10-19-d-rmap-thought-git-versioning.md) | | 2026-10-19 | `D-RMAP-1` | | [2026-10-19-d-rmap-implicit-relationship-register.md](2026-10-19-d-rmap-implicit-relationship-register.md) | +| 2026-10-19 | `D-RMAP` | | [2026-10-19-d-rmap-focus-probe.md](2026-10-19-d-rmap-focus-probe.md) | | 2026-10-17 | `thinking-tables-wiring` | | [2026-10-17-thinking-tables-wiring.md](2026-10-17-thinking-tables-wiring.md) | | 2026-10-17 | `D-ASM-11` | | [2026-10-17-pgring-idea-catalogue.md](2026-10-17-pgring-idea-catalogue.md) | | 2026-10-17 | `epistemic-frontier-clicks` | | [2026-10-17-epistemic-frontier-clicks.md](2026-10-17-epistemic-frontier-clicks.md) | diff --git a/.claude/plans/d-rmap-2-context-tile-algebra-probe-v1.md b/.claude/plans/d-rmap-2-context-tile-algebra-probe-v1.md index a0de2b25e..ab4fe4721 100644 --- a/.claude/plans/d-rmap-2-context-tile-algebra-probe-v1.md +++ b/.claude/plans/d-rmap-2-context-tile-algebra-probe-v1.md @@ -5,6 +5,8 @@ > **PRIOR-ART CORRECTION (2026-10-19, precedence over earlier wording):** The operator already reverse-engineered BlasGraph, MKL-like backends, BF16 16x16, AMX, VNNI and mask-GEMM interop in AdaWorldAPI/ndarray and AdaWorldAPI/lance-graph. **Do not start a new GraphBLAS/MKL implementation, import production FFI or redo settled kernel comparisons.** The unproven subject is the separate RelationshipMap tables, zero-copy index binding and thought-selected map palettes. Use existing native kernels and probes as references; any external GraphBLAS implementation is optional independent oracle only. +> **Partial answer (2026-10-19):** the first "updated test priority" item (bind a focus, compare direct indexed seek with a repeated local Boolean fold, focus churn, external-neighbour escapes, 1/8/64 thoughts) is measured in D-RMAP-1A §8 for in-memory warm fixtures. The GraphBLAS and BF16/INT8 arms here remain unrun. + ## 0. Existing implementation and measurements: mandatory reuse | Precedent | Source | What is already present | diff --git a/.claude/plans/d-rmap-separate-tables-thought-palettes-v1.md b/.claude/plans/d-rmap-separate-tables-thought-palettes-v1.md index 988c32701..47d6dc520 100644 --- a/.claude/plans/d-rmap-separate-tables-thought-palettes-v1.md +++ b/.claude/plans/d-rmap-separate-tables-thought-palettes-v1.md @@ -226,6 +226,72 @@ A adds a second ledger beside Lance and Thought Git, the duplicated truth this p The MORK/PathMap probe ([#1620](https://github.com/AdaWorldAPI/lance-graph/pull/1620)) binds a `RelationView { rows, generation: u64 }` and refuses `Stale { bound, current }`. That `u64` is a stand-in: under §7 it becomes the `(DurableRef, IndexSeal, QueryReference)` part of the binding witness. The backend switch (MORK, PathMap, native) does not touch the witness: the witness is about the data, the backend is a physical capability BIND attests. No new executor, store, or per-fold commit; the no-gather ruling is unchanged. +## 8. Focus probe: a focus-bound RelationshipMap over existing indexes (measured 2026-10-19) + +> **Status: MEASURED, probe only.** Code: `crates/mork-pathmap-probe/src/rmap.rs` (module `rmap`), `tests/rmap.rs`, `examples/rmap_cost.rs` (`cargo run --release -p mork-pathmap-probe --example rmap_cost`). Release build, one machine, warm caches, medians of 5 repetitions. Nothing here is adopted. No new executor, store, opcode, ABI, CE64 field or gather: every evaluation runs through `lance-graph-mask-risc` `execute`/`execute_into` with the existing `MaskOp` set, and address translation happens only in preparation. + +### 8.1 What the probe builds + +| Layer | Probe type | Owns | Copies of canonical facts | +| --- | --- | --- | --- | +| Canonical facts | `&[Fact]` (`[s,p,o]` u32) | the only payload | none | +| Ordered index | `SortedRun` (row permutation, `(p,s,o)`) or `SpogTrie` (PathMap, key `p‖s‖o`) | row references / trie keys | trie: keys copy p, s, o | +| Focus | `Focus{ids, generation}`, 256 slots | slot↔identity binding | none | +| Prepared map | `Csr` (per predicate: offsets, `u8` targets, canonical row, escapes) | slot-space adjacency + `Witness` | none (row refs kept) | +| Tile | `Tile`, one 65 536-bit plane per predicate, address `(target<<8)\|source` | derived from `Csr` | none | +| Thought delta | `ThoughtDelta{thought, p, add, del}` | per-thought edits over the shared tile | none | + +`Witness{facts, seal, focus, scope}` stands in for §7.5's `BindingWitness` (`DurableRef`, `IndexSeal`, `FocusGeneration`, `AuthorizationReceipt`); `ThoughtDelta.thought` stands in for `ThoughtCommitId`. Out-of-focus targets are never dropped: a reach reports them as escapes (identities, or source slots with identities on demand), and `global()` refuses `IncompleteFocus` while any remain. + +### 8.2 Correctness (12 tests, all green, independent scan oracle) + +Every arm (direct over run, direct over trie, CSR, tile via mask-risc; identity and source-slot escape modes) equals a full-slice scan for 1, 2 and 3 hops on 18 cases. Disable runs confirmed all 11 guards load-bearing (red with the guard removed, green restored), including: per-dimension witness refusal, thought-delta refusal for another incarnation, the escape report, canonical-row retention, `Ordering` mismatch refusal, and incremental rebind ≡ full preparation (which needed an expansion fixture with incoming edges before its escape re-resolve guard could fail). + +### 8.3 Cost per representation, 65 536 identities, focus 256 (µs unless noted) + +| Fixture | facts | sorted run build | trie build | CSR prepare | tile from CSR | direct/run per eval | CSR per eval | tile per eval | break-even reuses (CSR / tile) | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| sparse, 3 preds, degree 2 | 392 238 | 29.4 ms | 126.4 ms | 341.7 | 1.5 | 8.49 | 3.33 | 5.08 | 76 / 116 | +| sparse, 3 preds, degree 8 | 1 561 508 | 236.0 ms | 1 350.6 ms | 1 187.4 | 3.0 | 114.73 | 5.58 | 5.74 | 11 / 11 | +| dense, 1 pred, degree 48 | 2 862 756 | 483.2 ms | 2 118.2 ms | 955.1 | 6.4 | 565.53 | 12.38 | 4.53 | 1.8 / 1.8 | + +Per-eval figures are a 3-hop scoped reach in source-slot mode. Break-even counts are **arithmetic** from those medians (prepare ÷ per-eval saving), not separate measurements. Amortization was measured at N = 1, 8, 64, 1 024, 65 536 and 1 000 000; direct-arm runs above 20 s were capped and extrapolated, and are labelled CAPPED in the log. + +- **Escape identities dominate when degree grows.** At degree 8, reporting escapes as identities costs CSR 50.4 µs per eval; as source slots, 5.6 µs. Materialize identities only when the next frontier needs them. +- **Tile vs CSR is a density question.** The tile wins only on the dense relation (4.5 vs 12.4); on sparse ones CSR wins (3.3 vs 5.1). Neither is a default. +- **Bytes per focus:** binding 3 072 B; CSR 18–66 KB; tile 8 KiB per predicate. + +### 8.4 Focus churn: incremental rebind vs full re-preparation (µs) + +| replaced slots k | sparse d2 | sparse d8 | dense | +| ---: | --- | --- | --- | +| 1 | 41.8 / 357.2 | 223.9 / 1 657.7 | 308.2 / 892.9 | +| 16 | 60.4 / 433.1 | 298.1 / 1 985.2 | 752.6 / 744.9 | +| 128 | 156.5 / 310.7 | 850.4 / 937.7 | 970.3 / 833.2 | +| 256 | 344.3 / 305.3 | 1 238.4 / 971.3 | 1 011.0 / 644.9 | + +Incremental rebind (forward seek for changed slots, old escapes re-resolved against the new focus; no reverse index) wins for small k and loses at k = 256; on the dense relation it loses from k ≥ 16. A rebind policy needs the crossover, not a fixed choice. + +### 8.5 Thoughts over one shared tile + +Per-thought delta: 32 B as sparse edits, 16 KiB when lowered to executor planes. Per-eval cost over 1 / 8 / 64 concurrent thoughts: 2.61 / 2.66 / 2.79 µs (sparse d2), 8.04 / 8.14 / 8.28 (d8), 5.27 / 5.56 / 5.51 (dense). The shared tile is stored once; no thought copies it. + +### 8.6 PathMap / MORK: what they do and do not buy here + +- **Prefix seek is real.** Direct traversal over the trie beats the sorted run on sparse relations (7.0 vs 8.5 µs; 100.9 vs 114.7 µs), because a prefix seek reaches the objects without an indirect fact lookup. It loses on the dense relation (1 184 vs 566 µs). +- **Build and bytes cost more.** Trie build is 4–6× the sorted run, and its keys are at least 3× the run's bytes, because keys copy p, s and o. Node overhead is not measurable under MORK's allocator. +- **`restrict` yields the focus facts exactly, without copying shared nodes**, but takes 0.45–1.44 ms (slower than CSR preparation on the sparse fixtures) and its result still needs slot translation before execution. +- **Postings:** zipper merge 2.32 µs vs sorted merge 0.41 µs vs mask-risc And+Count 0.082 µs over prepared membership planes. +- **MORK pattern matching was not needed** for these fixtures and was not exercised; nothing here measures it. + +### 8.7 Places: zero-copy focus mask vs register copy + +On 65 536 places with a focus of 256, a bounding-box + class count through mask-risc over the canonical lanes takes 20–24 µs with 0 B copied, while loading the focus's lanes into a 3 072 B register and counting there takes 0.78 + 0.38–1.27 µs. The canonical-lane path is slow because the `under` gate is word-granular: 228–235 of 1 024 words are live for 256 rows. This is an open tension between zero-copy and a register copy, not a decision; bit-granular gating would remove it. + +### 8.8 Not covered, and blockers + +Lance-backed `DurableRef` and real stable-row-id seals (the probe's seals are integers); cold-cache and page-fault measurement; MORK matching; any gather-admission ruling. The probe answers §6's economics for in-memory, warm, single-machine fixtures only. + **Stop condition:** any design that requires copying canonical map facts per thought, using RelAlg as a table store, or allocating new CE64 bits to identify MapId/MapPaletteId has chosen the wrong boundary. **Short form: RelationshipMapStore stores. OGAR types. ThoughtMapPalette selects. BIND borrows. RelAlg describes. Quack lowers. Mask-RISC executes. CE64 remains opaque.** diff --git a/crates/mork-pathmap-probe/Cargo.toml b/crates/mork-pathmap-probe/Cargo.toml index d82b4e325..6e446813b 100644 --- a/crates/mork-pathmap-probe/Cargo.toml +++ b/crates/mork-pathmap-probe/Cargo.toml @@ -31,6 +31,9 @@ leapfrog = ["mork/leapfrog"] mork = { git = "https://github.com/AdaWorldAPI/MORK", rev = "de9312e3" } mork-expr = { git = "https://github.com/AdaWorldAPI/MORK", rev = "de9312e3" } pathmap = { git = "https://github.com/AdaWorldAPI/PathMap", rev = "ef02591" } +# The native executor for the D-RMAP focus probe (`src/rmap.rs`): the prepared +# relationship operands are run as its programs, never re-implemented here. +lance-graph-mask-risc = { path = "../lance-graph-mask-risc" } # MORK's workspace names PathMap as `path = "../PathMap/"`, outside its own # repository, so a git dependency on MORK cannot resolve it. This patch points diff --git a/crates/mork-pathmap-probe/examples/rmap_cost.rs b/crates/mork-pathmap-probe/examples/rmap_cost.rs new file mode 100644 index 000000000..1719146d4 --- /dev/null +++ b/crates/mork-pathmap-probe/examples/rmap_cost.rs @@ -0,0 +1,503 @@ +//! Preparation versus reuse for the D-RMAP focus probe. +//! Run: `cd crates/mork-pathmap-probe && cargo run --release --example rmap_cost` +//! +//! Every per-evaluation figure is a median over repeated runs of a real loop +//! (evaluation `i` uses seed slot `i % 256`, so no two consecutive evaluations +//! are identical). Totals for large N are MEASURED by running the loop, up to +//! a time cap; a capped arm says so and reports how far it got. The crossover +//! line is ARITHMETIC over measured inputs, labelled as such. + +use std::time::{Duration, Instant}; + +use lance_graph_mask_risc::{Scratch, words_for}; +use mork_pathmap_probe::boundary::PostingTrie; +use mork_pathmap_probe::rmap::*; + +const HOPS: usize = 3; +const REPS: usize = 5; +const CAP: Duration = Duration::from_secs(20); + +fn median(mut v: Vec) -> f64 { + v.sort_by(|a, b| a.partial_cmp(b).unwrap()); + v[v.len() / 2] +} + +/// Median wall time of `f`, in microseconds. +fn time_us(mut f: impl FnMut()) -> f64 { + median( + (0..REPS) + .map(|_| { + let t = Instant::now(); + f(); + t.elapsed().as_secs_f64() * 1e6 + }) + .collect(), + ) +} + +/// Run `eval(i)` for i in 0..n, stopping at the cap. Returns (evals done, +/// seconds, checksum). +fn run_n(n: usize, mut eval: impl FnMut(usize) -> usize) -> (usize, f64, usize) { + let t = Instant::now(); + let mut sum = 0usize; + for i in 0..n { + sum = sum.wrapping_add(eval(i)); + if i & 1023 == 1023 && t.elapsed() > CAP { + return (i + 1, t.elapsed().as_secs_f64(), sum); + } + } + (n, t.elapsed().as_secs_f64(), sum) +} + +fn count(r: &ScopedReach) -> usize { + slots_of(&r.reached).len() + r.escaped.len() +} + +fn graph_section(label: &str, preds: &[u32], degree: u32, community: u32, local: f64) { + println!( + "\n## SPOG focus, {label}: 65 536 identities, {} predicates, degree {degree} per predicate, community {community}, local share {local}", + preds.len() + ); + let facts = graph(65_536, preds.len() as u32, degree, community, local, 0x5EED); + println!( + "facts: {} (canonical payload {} B, never copied)", + facts.len(), + facts.len() * 12 + ); + + let t = Instant::now(); + let run = SortedRun::build(&facts, false, 1); + let run_build = t.elapsed().as_secs_f64() * 1e3; + let t = Instant::now(); + let trie = SpogTrie::build(&facts); + let trie_build = t.elapsed().as_secs_f64() * 1e3; + println!( + "index build: sorted run {run_build:.1} ms ({} B of row refs) | PathMap trie {trie_build:.1} ms (>= {} B of key bytes; node overhead not measurable here)", + run.bytes(), + trie.key_bytes() + ); + + let disc_run = time_us(|| { + std::hint::black_box(discover(&run, &[17], preds, SLOTS)); + }); + let disc_trie = time_us(|| { + std::hint::black_box(discover(&trie, &[17], preds, SLOTS)); + }); + let focus = discover(&run, &[17], preds, SLOTS); + let wit = Witness { + facts: 1, + seal: 1, + focus: focus.generation, + scope: 1, + }; + let prep_csr = time_us(|| { + std::hint::black_box(Csr::prepare(&run, &focus, preds, wit)); + }); + let csr = Csr::prepare(&run, &focus, preds, wit); + let prep_tile = time_us(|| { + std::hint::black_box(Tile::from_csr(&csr)); + }); + let tile = Tile::from_csr(&csr); + let pairs: Vec<(u32, u32)> = preds + .iter() + .flat_map(|&p| focus.ids().iter().map(move |&s| (p, s))) + .collect(); + let restrict = time_us(|| { + std::hint::black_box(trie.restrict_to(pairs.iter().copied())); + }); + println!( + "focus discovery (BFS to 256): run {disc_run:.1} us | trie {disc_trie:.1} us\n\ + prepare: CSR {prep_csr:.1} us ({} in-focus edges, {} escapes, {} B) | tile from CSR {prep_tile:.1} us ({} B) | trie restrict {restrict:.1} us (shares nodes; bytes not measurable)\n\ + focus binding: {} B", + csr.edges(), + csr.escapes(), + csr.bytes(), + tile.bytes(), + focus.bytes() + ); + + // Per-evaluation: 3-hop scoped reach from seed slot i % 256. + let mut ex = TileExec::new(); + let seeds: Vec = (0..SLOTS).map(|s| slot_set([s as u8])).collect(); + // Agreement check before any timing. + for s in [0usize, 77, 255] { + let d = reach_direct(&run, &focus, preds, &seeds[s], HOPS); + assert_eq!(csr.reach(&wit, preds, &seeds[s], HOPS).unwrap(), d); + assert_eq!( + ex.reach(&tile, &csr, &wit, None, None, preds, &seeds[s], HOPS) + .unwrap(), + d + ); + assert_eq!(reach_direct(&trie, &focus, preds, &seeds[s], HOPS), d); + } + let per = |f: &mut dyn FnMut(usize) -> usize| { + median( + (0..REPS) + .map(|_| { + let t = Instant::now(); + let mut x = 0; + for i in 0..1024 { + x += f(i); + } + std::hint::black_box(x); + t.elapsed().as_secs_f64() * 1e6 / 1024.0 + }) + .collect(), + ) + }; + let e_run = per(&mut |i| count(&reach_direct(&run, &focus, preds, &seeds[i % SLOTS], HOPS))); + let e_trie = per(&mut |i| count(&reach_direct(&trie, &focus, preds, &seeds[i % SLOTS], HOPS))); + let e_csr = per(&mut |i| count(&csr.reach(&wit, preds, &seeds[i % SLOTS], HOPS).unwrap())); + let e_tile = per(&mut |i| { + count( + &ex.reach( + &tile, + &csr, + &wit, + None, + None, + preds, + &seeds[i % SLOTS], + HOPS, + ) + .unwrap(), + ) + }); + println!( + "per evaluation, escapes as IDENTITIES (3-hop scoped reach, median of {REPS} x 1024): direct over run {e_run:.2} us | direct over trie {e_trie:.2} us | CSR {e_csr:.2} us | tile via mask-risc {e_tile:.2} us" + ); + let cnt = |m: &ScopedMask| slots_of(&m.reached).len() + slots_of(&m.escape_sources).len(); + for s in [0usize, 77, 255] { + let d = reach_direct_sources(&run, &focus, preds, &seeds[s], HOPS); + assert_eq!(csr.reach_sources(&wit, preds, &seeds[s], HOPS).unwrap(), d); + } + let m_run = per(&mut |i| { + cnt(&reach_direct_sources( + &run, + &focus, + preds, + &seeds[i % SLOTS], + HOPS, + )) + }); + let m_trie = per(&mut |i| { + cnt(&reach_direct_sources( + &trie, + &focus, + preds, + &seeds[i % SLOTS], + HOPS, + )) + }); + let m_csr = per(&mut |i| { + cnt(&csr + .reach_sources(&wit, preds, &seeds[i % SLOTS], HOPS) + .unwrap()) + }); + let m_tile = per(&mut |i| { + cnt(&ex + .reach_sources( + &tile, + &csr, + &wit, + None, + None, + preds, + &seeds[i % SLOTS], + HOPS, + ) + .unwrap()) + }); + println!( + "per evaluation, escapes as SOURCE SLOTS (same reach; identities on demand): direct over run {m_run:.2} us | direct over trie {m_trie:.2} us | CSR {m_csr:.2} us | tile via mask-risc {m_tile:.2} us" + ); + + // Measured totals over N evaluations, escapes as source slots. + println!( + "N, direct_run_s, csr_total_s (discover+prepare+N), tile_total_s (discover+prepare+tile+N) [escapes as source slots]" + ); + let fixed_csr = (disc_run + prep_csr) / 1e6; + let fixed_tile = fixed_csr + prep_tile / 1e6; + for &n in &[1usize, 8, 64, 1024, 65_536, 1_000_000] { + let (dn, ds, _) = run_n(n, |i| { + cnt(&reach_direct_sources( + &run, + &focus, + preds, + &seeds[i % SLOTS], + HOPS, + )) + }); + let (cn, cs, _) = run_n(n, |i| { + cnt(&csr + .reach_sources(&wit, preds, &seeds[i % SLOTS], HOPS) + .unwrap()) + }); + let (tn, ts, _) = run_n(n, |i| { + cnt(&ex + .reach_sources( + &tile, + &csr, + &wit, + None, + None, + preds, + &seeds[i % SLOTS], + HOPS, + ) + .unwrap()) + }); + let cap = |done: usize| { + if done < n { + format!(" (CAPPED at {done})") + } else { + String::new() + } + }; + println!( + "{n}, {ds:.4}{}, {:.4}{}, {:.4}{}", + cap(dn), + fixed_csr + cs, + cap(cn), + fixed_tile + ts, + cap(tn) + ); + } + let cross = |base: f64, prep_us: f64, per_us: f64| { + if base > per_us { + format!("{:.1}", prep_us / (base - per_us)) + } else { + "never (not cheaper per evaluation)".into() + } + }; + println!( + "break-even reuse count, ARITHMETIC from the measured medians above (vs direct over run): identities mode CSR {} | tile {} ; source-slot mode CSR {} | tile {}", + cross(e_run, disc_run + prep_csr, e_csr), + cross(e_run, disc_run + prep_csr + prep_tile, e_tile), + cross(m_run, disc_run + prep_csr, m_csr), + cross(m_run, disc_run + prep_csr + prep_tile, m_tile) + ); + + // Focus churn. + let other = discover(&run, &[40_000], preds, SLOTS); + println!("rebind k slots (incremental vs full re-prepare of CSR + tile), us:"); + for k in [1usize, 8, 16, 128, 256] { + let changes: Vec<(u8, u32)> = (0..k) + .zip(other.ids()) + .filter(|(_, g)| focus.slot_of(**g).is_none()) + .map(|(s, &g)| ((SLOTS - 1 - s) as u8, g)) + .collect(); + let new = focus.replace(&changes); + let nw = Witness { + focus: new.generation, + ..wit + }; + let inc = time_us(|| { + std::hint::black_box(rebind(&run, &focus, &new, &csr, nw)); + }); + let full = time_us(|| { + let c = Csr::prepare(&run, &new, preds, nw); + std::hint::black_box(Tile::from_csr(&c)); + }); + println!( + " k={k:>3} ({} replaced): incremental {inc:.1} | full {full:.1}", + changes.len() + ); + } + + // Thoughts sharing the tile, each with a sparse delta of 16 edits. + println!("thought contexts over one shared tile (delta: 8 added + 8 retracted edges each):"); + let edges: Vec<(u32, u8, u8, u32)> = csr.edge_set().into_iter().filter(|e| e.0 == 0).collect(); + for &t in &[1usize, 8, 64] { + let mut rng = Rng(t as u64); + let deltas: Vec = (0..t) + .map(|k| { + let add: Vec<(u8, u8)> = (0..8) + .map(|_| (rng.below(256) as u8, rng.below(256) as u8)) + .collect(); + let del: Vec<(u8, u8)> = (0..8) + .map(|_| { + let e = edges[rng.below(edges.len() as u64) as usize]; + (e.1, e.2) + }) + .collect(); + ThoughtDelta::new(k as u64, 0, &add, &del) + }) + .collect(); + let per_thought = per(&mut |i| { + let d = &deltas[i % t]; + count( + &ex.reach( + &tile, + &csr, + &wit, + Some(d), + Some(d.thought), + &[0], + &seeds[i % SLOTS], + HOPS, + ) + .unwrap(), + ) + }); + println!( + " {t:>2} thoughts: per evaluation {per_thought:.2} us | per-thought memory {} B sparse, {} B as executor planes | shared tile {} B once", + deltas[0].bytes_sparse(), + deltas[0].bytes_planes(), + tile.bytes() + ); + } +} + +fn places_section() { + println!("\n## Places: 65 536 canonical rows, focus of 256"); + let places = places(65_536, 3); + let morton = morton_order(&places); + println!( + "Morton order index: {} B of row refs (coordinates not copied)", + morton.len() * 4 + ); + let mut rng = Rng(11); + let mut scattered: Vec = Vec::new(); + while scattered.len() < SLOTS { + let i = rng.below(65_536) as u32; + if !scattered.contains(&i) { + scattered.push(i); + } + } + let clustered: Vec = morton[30_000..30_000 + SLOTS].to_vec(); + let mut scratch = Scratch::new(words_for(65_536), 2); + let qs: Vec = (0..64) + .map(|k| BoxQuery { + lat: (-90_000_000 + k * 2_000_000, 90_000_000), + lon: (-180_000_000, 180_000_000 - k * 4_000_000), + class: (k as u32) % 16, + }) + .collect(); + for (name, ids) in [("scattered", &scattered), ("Morton-clustered", &clustered)] { + let bind = time_us(|| { + std::hint::black_box(FocusMask::over(65_536, ids, Ordering::Canonical, 1)); + }); + let fm = FocusMask::over(65_536, ids, Ordering::Canonical, 1); + let live = fm.words.iter().filter(|&&w| w != 0).count(); + for q in &qs { + assert_eq!( + box_count(&places, &fm, q, &mut scratch).unwrap(), + box_count_by_id(&places, ids, q) + ); + } + let native = time_us(|| { + for q in &qs { + std::hint::black_box(box_count(&places, &fm, q, &mut scratch).unwrap()); + } + }) / qs.len() as f64; + let by_id = time_us(|| { + for q in &qs { + std::hint::black_box(box_count_by_id(&places, ids, q)); + } + }) / qs.len() as f64; + let load = time_us(|| { + std::hint::black_box(FocusLanes::load(&places, ids, 1)); + }); + let lanes = FocusLanes::load(&places, ids, 1); + let mut small = Scratch::new(SLOT_WORDS, 2); + for q in &qs { + assert_eq!( + lanes.count(1, q, &mut small).unwrap(), + box_count_by_id(&places, ids, q) + ); + } + let reg = time_us(|| { + for q in &qs { + std::hint::black_box(lanes.count(1, q, &mut small).unwrap()); + } + }) / qs.len() as f64; + println!( + "{name}: bind mask {bind:.2} us (8 KiB plane, {live} live words of 1024) | bbox+class count: mask-risc over canonical lanes {native:.2} us (0 B copied) | read each id's lanes {by_id:.2} us (0 B copied) | register lanes: load {load:.2} us ({} B COPIED), then mask-risc over 256 rows {reg:.2} us", + lanes.bytes() + ); + } +} + +fn postings_section() { + println!("\n## Inverted index: 256 terms over 65 536 docs, ~40 docs per term"); + let mut postings: Vec> = vec![vec![17, 21, 39], vec![21, 39, 82]]; + let mut rng = Rng(9); + for _ in 0..254 { + let mut v: Vec = (0..40).map(|_| rng.below(65_536) as u32).collect(); + v.sort_unstable(); + v.dedup(); + postings.push(v); + } + let pairs: Vec<[u32; 2]> = postings + .iter() + .enumerate() + .flat_map(|(t, d)| d.iter().map(move |&x| [t as u32, x])) + .collect(); + let trie = PostingTrie::build(&pairs); + // Focus: the docs of terms 2..8 (a scope that fits 256 slots). + let mut scope: Vec = postings[2..8].iter().flatten().copied().collect(); + scope.sort_unstable(); + scope.dedup(); + let focus = Focus::bind(&scope, 1); + let lists: Vec<&[u32]> = postings[2..8].iter().map(|v| v.as_slice()).collect(); + let prep = time_us(|| { + std::hint::black_box(membership_planes(&lists, &focus)); + }); + let planes = membership_planes(&lists, &focus); + let mut scratch = Scratch::new(SLOT_WORDS, 2); + let pairs6: Vec<(usize, usize)> = (0..6) + .flat_map(|a| (0..6).map(move |b| (a, b))) + .filter(|(a, b)| a < b) + .collect(); + for &(a, b) in &pairs6 { + let want = intersect_sorted(lists[a], lists[b]); + assert_eq!(trie.intersect(a as u32 + 2, b as u32 + 2), want); + assert_eq!( + intersect_planes(&planes[a], &planes[b], &focus, &mut scratch), + want + ); + } + let n = pairs6.len() as f64; + let sorted = time_us(|| { + for &(a, b) in &pairs6 { + std::hint::black_box(intersect_sorted(lists[a], lists[b])); + } + }) / n; + let tri = time_us(|| { + for &(a, b) in &pairs6 { + std::hint::black_box(trie.intersect(a as u32 + 2, b as u32 + 2)); + } + }) / n; + let keep = time_us(|| { + for &(a, b) in &pairs6 { + std::hint::black_box(intersect_planes( + &planes[a], + &planes[b], + &focus, + &mut scratch, + )); + } + }) / n; + let cnt = time_us(|| { + for &(a, b) in &pairs6 { + std::hint::black_box(intersect_count(&planes[a], &planes[b], &mut scratch)); + } + }) / n; + println!( + "focus {} docs, membership planes prepared in {prep:.2} us ({} B)\n\ + per intersection: sorted merge {sorted:.3} us | PathMap zipper merge {tri:.3} us | mask-risc And+Keep -> ids {keep:.3} us | mask-risc And+Count {cnt:.3} us", + focus.len(), + planes.len() * SLOT_WORDS * 8 + ); +} + +fn main() { + println!("D-RMAP focus probe — release, one machine, medians of {REPS} repetitions"); + let three = [0u32, 1, 2]; + graph_section("sparse", &three, 2, 512, 0.9); + graph_section("sparse, degree 8", &three, 8, 512, 0.9); + graph_section("dense in-community", &[0], 48, 256, 1.0); + places_section(); + postings_section(); +} diff --git a/crates/mork-pathmap-probe/src/boundary.rs b/crates/mork-pathmap-probe/src/boundary.rs index f4802ac33..369889ef9 100644 --- a/crates/mork-pathmap-probe/src/boundary.rs +++ b/crates/mork-pathmap-probe/src/boundary.rs @@ -15,7 +15,7 @@ use std::time::{Duration, Instant}; use mork::space::Space; use pathmap::PathMap; -use pathmap::zipper::{ZipperIteration, ZipperPath}; +use pathmap::zipper::{ZipperIteration, ZipperPath, ZipperValues}; use crate::{Answers, JoinInput, Semantics}; @@ -238,6 +238,128 @@ fn path_answers(trie: &PathMap<()>) -> Answers { out } +// ---- PathMap as a relationship index (D-RMAP-1A focus probe) ---------------- + +/// A PathMap trie over `(p, s, o)` facts, keyed `p ‖ s ‖ o` big-endian, each +/// leaf holding the canonical row number of its fact. +/// +/// It is an INDEX, not the facts: the canonical rows stay in the caller's +/// slice and the leaf is a row reference. The key bytes are a copy of `p`, `s` +/// and `o` (12 bytes per fact), which [`SpogTrie::key_bytes`] reports; that is +/// the cost of a byte trie and the reason the native sorted run in +/// [`crate::rmap`] carries row references only. +pub struct SpogTrie(PathMap); + +fn spog_key(p: u32, s: u32, o: u32) -> [u8; 12] { + let mut k = [0u8; 12]; + k[0..4].copy_from_slice(&p.to_be_bytes()); + k[4..8].copy_from_slice(&s.to_be_bytes()); + k[8..12].copy_from_slice(&o.to_be_bytes()); + k +} + +impl SpogTrie { + /// Index `facts` (`[s, p, o]` rows). Row `i` becomes the leaf of its key. + pub fn build(facts: &[[u32; 3]]) -> Self { + let mut t = PathMap::new(); + for (row, &[s, p, o]) in facts.iter().enumerate() { + t.insert(spog_key(p, s, o), row as u32); + } + SpogTrie(t) + } + + /// Number of indexed facts (a full walk: PathMap's `val_count` is O(n)). + pub fn len(&self) -> usize { + self.0.val_count() + } + + /// Whether the index holds no fact. + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Lower bound on what the index stores: its key bytes. Node and + /// allocator overhead are not counted (MORK installs a global allocator, + /// so this probe cannot count allocations; see ISS-MORK-LIBRARY-GLOBAL-ALLOCATOR). + pub fn key_bytes(&self) -> usize { + self.len() * 12 + } + + /// Prefix seek to `p ‖ s`, then every `(o, row)` under it in `o` order. + pub fn objects(&self, p: u32, s: u32, mut f: impl FnMut(u32, u32)) { + let mut prefix = [0u8; 8]; + prefix[0..4].copy_from_slice(&p.to_be_bytes()); + prefix[4..8].copy_from_slice(&s.to_be_bytes()); + let mut z = self.0.read_zipper_at_path(prefix); + while z.to_next_val() { + let o = u32::from_be_bytes(z.path()[0..4].try_into().unwrap()); + let row = *z.val().expect("to_next_val stops on a value"); + f(o, row); + } + } + + /// The facts whose `(p, s)` is in `pairs`, as a trie that shares the + /// subtries it keeps with `self` (PathMap's `restrict`): the focus's + /// outgoing relationships, cut out without re-inserting a fact. + pub fn restrict_to(&self, pairs: impl IntoIterator) -> SpogTrie { + let mut keep = PathMap::new(); + for (p, s) in pairs { + let mut k = [0u8; 8]; + k[0..4].copy_from_slice(&p.to_be_bytes()); + k[4..8].copy_from_slice(&s.to_be_bytes()); + keep.insert(k, 0u32); + } + SpogTrie(self.0.restrict(&keep)) + } +} + +/// A PathMap posting index keyed `term ‖ doc` big-endian (inverted index). +pub struct PostingTrie(PathMap<()>); + +impl PostingTrie { + /// Index `(term, doc)` pairs. + pub fn build(pairs: &[[u32; 2]]) -> Self { + let mut t = PathMap::new(); + for &[term, doc] in pairs { + let mut k = [0u8; 8]; + k[0..4].copy_from_slice(&term.to_be_bytes()); + k[4..8].copy_from_slice(&doc.to_be_bytes()); + t.insert(k, ()); + } + PostingTrie(t) + } + + /// Every doc of `term`, in doc order (one prefix seek, then a walk). + pub fn docs(&self, term: u32, mut f: impl FnMut(u32)) { + let mut z = self.0.read_zipper_at_path(term.to_be_bytes()); + while z.to_next_val() { + f(u32::from_be_bytes(z.path()[0..4].try_into().unwrap())); + } + } + + /// Docs of both terms, merged from two ordered zipper walks. + pub fn intersect(&self, a: u32, b: u32) -> Vec { + let mut za = self.0.read_zipper_at_path(a.to_be_bytes()); + let mut zb = self.0.read_zipper_at_path(b.to_be_bytes()); + let read = |p: &[u8]| u32::from_be_bytes(p[0..4].try_into().unwrap()); + let mut out = Vec::new(); + let (mut ha, mut hb) = (za.to_next_val(), zb.to_next_val()); + while ha && hb { + let (x, y) = (read(za.path()), read(zb.path())); + if x == y { + out.push(x); + ha = za.to_next_val(); + hb = zb.to_next_val(); + } else if x < y { + ha = za.to_next_val(); + } else { + hb = zb.to_next_val(); + } + } + out + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/mork-pathmap-probe/src/lib.rs b/crates/mork-pathmap-probe/src/lib.rs index 341d2b0c9..81d4de382 100644 --- a/crates/mork-pathmap-probe/src/lib.rs +++ b/crates/mork-pathmap-probe/src/lib.rs @@ -23,6 +23,7 @@ use std::collections::BTreeMap; pub mod boundary; pub mod reference; +pub mod rmap; pub use boundary::{Backend, BoundIndex, Refusal, bind, capability}; diff --git a/crates/mork-pathmap-probe/src/rmap.rs b/crates/mork-pathmap-probe/src/rmap.rs new file mode 100644 index 000000000..ee715e721 --- /dev/null +++ b/crates/mork-pathmap-probe/src/rmap.rs @@ -0,0 +1,1432 @@ +//! D-RMAP-1A probe: a focus-bound RelationshipMap over existing ordered indexes. +//! +//! STATUS: PROBE. Plan: `.claude/plans/d-rmap-separate-tables-thought-palettes-v1.md` +//! (§8 is this probe's record). Nothing here is a store, an executor or a +//! contract type; every type is a stand-in for a D-RMAP-1A name. +//! +//! The question: can a scattered global neighbourhood be cut out of an +//! existing ordered index once, bound to 256 local focus slots, and then +//! evaluated many times by the native mask executor, instead of re-walking the +//! index on every evaluation? +//! +//! Layers, and which D-RMAP-1A table each stands for: +//! +//! - **Canonical facts** (`&[Fact]`, RelationFacts). Never copied, never +//! reordered. Every prepared form refers to a fact by its row number. +//! - **Ordered indexes** (RelationOrder): [`SortedRun`], a permutation of row +//! numbers sorted by `(p, s, o)` (and its `(p, o, s)` twin), and the PathMap +//! trie [`SpogTrie`]. Both answer "`(p, s)` → objects" by prefix seek. +//! - **Focus** ([`Focus`], FocusBinding): slot `0..256` → global identity, with +//! a generation. Identities are never rewritten to fit the focus. +//! - **Prepared relationship operands**: [`Csr`] (row offsets per slot, local +//! targets, the canonical row of every edge) and [`Tile`] (a 256 × 256 +//! Boolean plane per predicate). Both are DERIVED: they hold slot ordinals +//! and row references, never fact payload. +//! - **Native execution**: [`Tile`] hops run as `lance-graph-mask-risc` +//! programs over planes that share one coordinate space — the tile's address +//! is `(target << 8) | source`, the `u8:u8` rail of a 64 Ki-row table, and +//! the frontier is broadcast into the same space. No op reads at a position +//! taken from another lane's value (no `MaskOp::Gather`; the 2026-10-16 +//! ruling). +//! +//! Where the 2026-10-16 ruling bites, and why it does not here: turning a +//! global target into a slot ordinal IS an address translation. It happens +//! once, in preparation ([`Csr::prepare`]), counted, outside the fold — the +//! form H14c §1 rule 6 allows ("any address translation is explicit, counted, +//! and outside the direct-position path"). Whether a preparation stage may do +//! this as a matter of policy is the open legal question D-RMAP-1 §6 names; the +//! probe does not settle it, it measures it. +//! +//! Semantics are Boolean reachability inside the focus. A target outside the +//! focus is returned in [`ScopedReach::escaped`], never folded into +//! `reached` and never dropped; [`ScopedReach::global`] refuses while any +//! escape exists, because a path through an out-of-focus node can return into +//! the focus. Bag counts and path counts are not answered here. + +use std::collections::BTreeSet; + +use lance_graph_mask_risc::{ + Foreign, LaneRef, MaskOp, Operand, Out, Planes, Pred, Program, Scratch, Terminal, Value, + execute, execute_into, words_for, +}; + +pub use crate::boundary::SpogTrie; + +/// One canonical fact `[s, p, o]`. +pub type Fact = [u32; 3]; + +/// Focus width: local slots `0..256`. +pub const SLOTS: usize = 256; +/// Words of a 256-bit slot plane. +pub const SLOT_WORDS: usize = SLOTS / 64; +/// Rows of a 256 × 256 tile: the `u8:u8` rail. +pub const TILE_ROWS: usize = SLOTS * SLOTS; +/// Words of a tile plane (8 KiB). +pub const TILE_WORDS: usize = TILE_ROWS / 64; + +// ---- validity -------------------------------------------------------------- + +/// What a prepared operand was built against. A probe-local stand-in for the +/// D-RMAP-1A §7 binding witness: `facts` ↔ the facts' `DurableRef`, `seal` ↔ +/// the index seal, `focus` ↔ the focus generation, `scope` ↔ the +/// authorization receipt. A thought's identity is checked separately +/// ([`ThoughtDelta::thought`]). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Witness { + pub facts: u64, + pub seal: u64, + pub focus: u64, + pub scope: u64, +} + +/// Why a prepared operand may not be used. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Refusal { + /// The facts it was built from changed. + FactsChanged, + /// The index it navigated was resealed (append, compaction, rebuild). + IndexResealed, + /// A focus slot now names a different identity. + FocusChanged, + /// The scope or permissions changed. + ScopeChanged, + /// A thought delta belongs to another thought incarnation. + ThoughtMismatch, + /// Two masks over different row orderings were combined. + OrderingMismatch, + /// The answer depends on out-of-focus nodes ([`ScopedReach::global`]). + IncompleteFocus, +} + +impl Witness { + /// Refuse when any component differs; the first difference is reported. + pub fn check(&self, current: &Witness) -> Result<(), Refusal> { + if self.facts != current.facts { + return Err(Refusal::FactsChanged); + } + if self.seal != current.seal { + return Err(Refusal::IndexResealed); + } + if self.focus != current.focus { + return Err(Refusal::FocusChanged); + } + if self.scope != current.scope { + return Err(Refusal::ScopeChanged); + } + Ok(()) + } +} + +// ---- ordered index --------------------------------------------------------- + +/// A sorted run over canonical facts: row numbers ordered by `(p, s, o)`, or +/// by `(p, o, s)` for the reverse direction. Holds `u32` row references only; +/// the facts stay in the borrowed slice. +pub struct SortedRun<'f> { + facts: &'f [Fact], + perm: Vec, + reverse: bool, + /// Index seal: which build this is. A rebuild gets a new seal. + pub seal: u64, +} + +impl<'f> SortedRun<'f> { + fn key(f: &Fact, reverse: bool) -> (u32, u32, u32) { + if reverse { + (f[1], f[2], f[0]) + } else { + (f[1], f[0], f[2]) + } + } + + /// Sort row numbers by `(p, s, o)` (`reverse = false`) or `(p, o, s)`. + pub fn build(facts: &'f [Fact], reverse: bool, seal: u64) -> Self { + let mut perm: Vec = (0..facts.len() as u32).collect(); + perm.sort_unstable_by_key(|&r| Self::key(&facts[r as usize], reverse)); + SortedRun { + facts, + perm, + reverse, + seal, + } + } + + /// The rows whose `(p, first)` matches: a borrowed range of the run. + pub fn range(&self, p: u32, first: u32) -> &[u32] { + let k = |r: &u32| { + let (kp, k1, _) = Self::key(&self.facts[*r as usize], self.reverse); + (kp, k1) + }; + let lo = self.perm.partition_point(|r| k(r) < (p, first)); + let hi = self.perm.partition_point(|r| k(r) <= (p, first)); + &self.perm[lo..hi] + } + + /// `(other end, row)` for every fact under `(p, first)`. + pub fn neighbours(&self, p: u32, first: u32, mut f: impl FnMut(u32, u32)) { + let other = if self.reverse { 0 } else { 2 }; + for &r in self.range(p, first) { + f(self.facts[r as usize][other], r); + } + } + + /// Index bytes: the permutation (the facts are not counted; they are not + /// the index's). + pub fn bytes(&self) -> usize { + self.perm.len() * 4 + } +} + +/// What the focus discovery and the reference oracle navigate. +pub trait Navigate { + /// Every `(o, row)` with fact `(s, p, o)`. + fn objects(&self, p: u32, s: u32, f: &mut dyn FnMut(u32, u32)); +} + +impl Navigate for SortedRun<'_> { + fn objects(&self, p: u32, s: u32, f: &mut dyn FnMut(u32, u32)) { + debug_assert!(!self.reverse); + self.neighbours(p, s, f); + } +} + +impl Navigate for SpogTrie { + fn objects(&self, p: u32, s: u32, f: &mut dyn FnMut(u32, u32)) { + SpogTrie::objects(self, p, s, f); + } +} + +// ---- focus ----------------------------------------------------------------- + +/// FocusBinding: slot → global identity, and the sorted reverse lookup. +#[derive(Clone, Debug)] +pub struct Focus { + ids: Vec, + by_id: Vec<(u32, u8)>, + /// Bumped by every change of a slot's identity. + pub generation: u64, +} + +impl Focus { + /// Bind up to 256 distinct identities, in slot order. + pub fn bind(ids: &[u32], generation: u64) -> Self { + assert!(ids.len() <= SLOTS, "a focus holds at most 256 identities"); + let mut by_id: Vec<(u32, u8)> = + ids.iter().enumerate().map(|(s, &g)| (g, s as u8)).collect(); + by_id.sort_unstable(); + assert!( + by_id.windows(2).all(|w| w[0].0 != w[1].0), + "focus identities must be distinct" + ); + Focus { + ids: ids.to_vec(), + by_id, + generation, + } + } + + /// Slot of a global identity, if focused. + pub fn slot_of(&self, id: u32) -> Option { + self.by_id + .binary_search_by_key(&id, |&(g, _)| g) + .ok() + .map(|i| self.by_id[i].1) + } + + /// Global identity of a slot. + pub fn id(&self, slot: u8) -> u32 { + self.ids[slot as usize] + } + + pub fn len(&self) -> usize { + self.ids.len() + } + + pub fn is_empty(&self) -> bool { + self.ids.is_empty() + } + + pub fn ids(&self) -> &[u32] { + &self.ids + } + + /// Replace the identity of the given slots; returns a new generation. + pub fn replace(&self, changes: &[(u8, u32)]) -> Focus { + let mut ids = self.ids.clone(); + for &(s, g) in changes { + ids[s as usize] = g; + } + Focus::bind(&ids, self.generation + 1) + } + + /// Bytes of the binding (slot table + reverse lookup). + pub fn bytes(&self) -> usize { + self.ids.len() * 4 + self.by_id.len() * 8 + } +} + +/// Discover a focus: breadth-first from `seeds` over the index, through +/// `preds`, until `cap` identities are bound. The order of discovery is the +/// slot order. +pub fn discover(index: &dyn Navigate, seeds: &[u32], preds: &[u32], cap: usize) -> Focus { + let cap = cap.min(SLOTS); + let mut seen = BTreeSet::new(); + let mut order = Vec::with_capacity(cap); + let mut queue = std::collections::VecDeque::new(); + for &s in seeds { + if order.len() < cap && seen.insert(s) { + order.push(s); + queue.push_back(s); + } + } + while let Some(s) = queue.pop_front() { + for &p in preds { + index.objects(p, s, &mut |o, _| { + if order.len() < cap && seen.insert(o) { + order.push(o); + queue.push_back(o); + } + }); + } + if order.len() >= cap { + break; + } + } + Focus::bind(&order, 1) +} + +// ---- scoped reachability --------------------------------------------------- + +/// A 256-bit set of focus slots. +pub type SlotSet = [u64; SLOT_WORDS]; + +pub fn slot_set(slots: impl IntoIterator) -> SlotSet { + let mut s = [0u64; SLOT_WORDS]; + for x in slots { + s[(x >> 6) as usize] |= 1 << (x & 63); + } + s +} + +pub fn slots_of(set: &SlotSet) -> Vec { + (0..SLOTS) + .filter(|&i| set[i >> 6] >> (i & 63) & 1 == 1) + .map(|i| i as u8) + .collect() +} + +/// Boolean reachability within the focus, plus every out-of-focus target an +/// in-focus node reached within the hop budget. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ScopedReach { + /// Slots reachable in `0..=hops` steps (seeds included). + pub reached: SlotSet, + /// Global identities outside the focus that a reached node points at + /// within the budget. Exact, never truncated. + pub escaped: BTreeSet, +} + +impl ScopedReach { + /// The answer to the GLOBAL question, which equals the scoped one only if + /// nothing escaped. + pub fn global(&self) -> Result<&SlotSet, Refusal> { + if self.escaped.is_empty() { + Ok(&self.reached) + } else { + Err(Refusal::IncompleteFocus) + } + } +} + +/// [`ScopedReach`] without the escaped identities: the slots they leave from. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ScopedMask { + pub reached: SlotSet, + /// Slots, within the hop budget, with an edge out of the focus. + pub escape_sources: SlotSet, +} + +impl ScopedMask { + pub fn global(&self) -> Result<&SlotSet, Refusal> { + if self.escape_sources == [0; SLOT_WORDS] { + Ok(&self.reached) + } else { + Err(Refusal::IncompleteFocus) + } + } +} + +/// Reachability by walking the index on every call: the direct arm. Every hop +/// seeks the index again and translates every target. +pub fn reach_direct( + index: &dyn Navigate, + focus: &Focus, + preds: &[u32], + seeds: &SlotSet, + hops: usize, +) -> ScopedReach { + let mut reached = *seeds; + let mut frontier = *seeds; + let mut escaped = BTreeSet::new(); + for _ in 0..hops { + let mut next = [0u64; SLOT_WORDS]; + for s in slots_of(&frontier) { + let g = focus.id(s); + for &p in preds { + index.objects(p, g, &mut |o, _| match focus.slot_of(o) { + Some(t) => next[(t >> 6) as usize] |= 1 << (t & 63), + None => { + escaped.insert(o); + } + }); + } + } + for w in 0..SLOT_WORDS { + frontier[w] = next[w] & !reached[w]; + reached[w] |= next[w]; + } + if frontier == [0; SLOT_WORDS] { + break; + } + } + ScopedReach { reached, escaped } +} + +/// [`reach_direct`] with the same demand as [`Csr::reach_sources`]: escapes +/// as source slots, not identities. Still seeks the index and translates +/// every target on every call. +pub fn reach_direct_sources( + index: &dyn Navigate, + focus: &Focus, + preds: &[u32], + seeds: &SlotSet, + hops: usize, +) -> ScopedMask { + let mut reached = *seeds; + let mut frontier = *seeds; + let mut escape_sources = [0u64; SLOT_WORDS]; + for _ in 0..hops { + let mut next = [0u64; SLOT_WORDS]; + for s in slots_of(&frontier) { + let g = focus.id(s); + for &p in preds { + index.objects(p, g, &mut |o, _| match focus.slot_of(o) { + Some(t) => next[(t >> 6) as usize] |= 1 << (t & 63), + None => escape_sources[(s >> 6) as usize] |= 1 << (s & 63), + }); + } + } + for w in 0..SLOT_WORDS { + frontier[w] = next[w] & !reached[w]; + reached[w] |= next[w]; + } + if frontier == [0; SLOT_WORDS] { + break; + } + } + ScopedMask { + reached, + escape_sources, + } +} + +// ---- prepared: CSR --------------------------------------------------------- + +/// One predicate's edges out of the focus, compressed by source slot. +/// `dst[k]` is a local target slot; `row[k]` is the canonical fact. +pub struct CsrPred { + pub p: u32, + offsets: [u32; SLOTS + 1], + dst: Vec, + row: Vec, + /// `(source slot, global target, row)` for every edge that leaves the + /// focus, sorted by source slot. + escaped: Vec<(u8, u32, u32)>, + /// Slots with at least one edge in `escaped`. + has_escape: SlotSet, +} + +/// Prepared CSR over the focus for a set of predicates. +pub struct Csr { + pub preds: Vec, + pub witness: Witness, +} + +impl Csr { + /// One seek per `(predicate, slot)`; every target translated once. + // A slot number is the coordinate, not a loop counter. + #[expect(clippy::needless_range_loop)] + pub fn prepare(index: &dyn Navigate, focus: &Focus, preds: &[u32], witness: Witness) -> Csr { + let mut out = Vec::with_capacity(preds.len()); + for &p in preds { + let mut offsets = [0u32; SLOTS + 1]; + let mut dst = Vec::new(); + let mut row = Vec::new(); + let mut escaped = Vec::new(); + for s in 0..SLOTS { + offsets[s] = dst.len() as u32; + if s < focus.len() { + index.objects(p, focus.id(s as u8), &mut |o, r| match focus.slot_of(o) { + Some(t) => { + dst.push(t); + row.push(r); + } + None => escaped.push((s as u8, o, r)), + }); + } + } + offsets[SLOTS] = dst.len() as u32; + out.push(CsrPred { + p, + offsets, + dst, + row, + has_escape: slot_set(escaped.iter().map(|e| e.0)), + escaped, + }); + } + Csr { + preds: out, + witness, + } + } + + /// Edges inside the focus, over all predicates. + pub fn edges(&self) -> usize { + self.preds.iter().map(|c| c.dst.len()).sum() + } + + /// Edges leaving the focus. + pub fn escapes(&self) -> usize { + self.preds.iter().map(|c| c.escaped.len()).sum() + } + + /// Derived bytes (offsets, slot targets, row references, escape list). + pub fn bytes(&self) -> usize { + self.preds + .iter() + .map(|c| (SLOTS + 1) * 4 + c.dst.len() + c.row.len() * 4 + c.escaped.len() * 12) + .sum() + } + + /// The canonical rows behind the in-focus edges of `p` from `slot`. + pub fn rows(&self, p: u32, slot: u8) -> &[u32] { + let c = self + .preds + .iter() + .find(|c| c.p == p) + .expect("predicate prepared"); + let s = slot as usize; + &c.row[c.offsets[s] as usize..c.offsets[s + 1] as usize] + } + + /// Slots that have an out-of-focus edge under any of `preds`. + pub fn escape_plane(&self, preds: &[u32]) -> SlotSet { + let mut e = [0u64; SLOT_WORDS]; + for c in self.preds.iter().filter(|c| preds.contains(&c.p)) { + for (w, h) in e.iter_mut().zip(c.has_escape) { + *w |= h; + } + } + e + } + + /// The out-of-focus identities reached from `sources` under `preds`, + /// materialized on demand (the identities [`ScopedReach::escaped`] holds). + pub fn escape_ids(&self, sources: &SlotSet, preds: &[u32]) -> BTreeSet { + let mut out = BTreeSet::new(); + self.escapes_from(sources, preds, &mut out); + out + } + + /// Reachability without materializing escaped identities: which reached + /// slots have an edge out of the focus within the hop budget. As exact as + /// [`Csr::reach`]; [`Csr::escape_ids`] turns the sources into identities. + pub fn reach_sources( + &self, + current: &Witness, + preds: &[u32], + seeds: &SlotSet, + hops: usize, + ) -> Result { + self.witness.check(current)?; + let esc_plane = self.escape_plane(preds); + let mut reached = *seeds; + let mut frontier = *seeds; + let mut escape_sources = [0u64; SLOT_WORDS]; + for _ in 0..hops { + let mut next = [0u64; SLOT_WORDS]; + for w in 0..SLOT_WORDS { + escape_sources[w] |= frontier[w] & esc_plane[w]; + } + for c in self.preds.iter().filter(|c| preds.contains(&c.p)) { + for s in slots_of(&frontier) { + let s = s as usize; + for &t in &c.dst[c.offsets[s] as usize..c.offsets[s + 1] as usize] { + next[(t >> 6) as usize] |= 1 << (t & 63); + } + } + } + for w in 0..SLOT_WORDS { + frontier[w] = next[w] & !reached[w]; + reached[w] |= next[w]; + } + if frontier == [0; SLOT_WORDS] { + break; + } + } + Ok(ScopedMask { + reached, + escape_sources, + }) + } + + fn escapes_from(&self, frontier: &SlotSet, preds: &[u32], into: &mut BTreeSet) { + for c in self.preds.iter().filter(|c| preds.contains(&c.p)) { + for &(s, o, _) in &c.escaped { + if frontier[(s >> 6) as usize] >> (s & 63) & 1 == 1 { + into.insert(o); + } + } + } + } + + /// Reachability over the prepared CSR. Refuses a stale witness. + pub fn reach( + &self, + current: &Witness, + preds: &[u32], + seeds: &SlotSet, + hops: usize, + ) -> Result { + self.witness.check(current)?; + let mut reached = *seeds; + let mut frontier = *seeds; + let mut escaped = BTreeSet::new(); + for _ in 0..hops { + self.escapes_from(&frontier, preds, &mut escaped); + let mut next = [0u64; SLOT_WORDS]; + for c in self.preds.iter().filter(|c| preds.contains(&c.p)) { + for s in slots_of(&frontier) { + let s = s as usize; + for &t in &c.dst[c.offsets[s] as usize..c.offsets[s + 1] as usize] { + next[(t >> 6) as usize] |= 1 << (t & 63); + } + } + } + for w in 0..SLOT_WORDS { + frontier[w] = next[w] & !reached[w]; + reached[w] |= next[w]; + } + if frontier == [0; SLOT_WORDS] { + break; + } + } + Ok(ScopedReach { reached, escaped }) + } +} + +// ---- prepared: tile -------------------------------------------------------- + +/// Address of edge `source → target` in a tile plane. +pub fn tile_addr(source: u8, target: u8) -> usize { + ((target as usize) << 8) | source as usize +} + +/// 256 × 256 Boolean adjacency per predicate, addressed `(target, source)`, +/// built from a [`Csr`] (so it adds no navigation of its own). +pub struct Tile { + pub preds: Vec, + planes: Vec>, + pub witness: Witness, +} + +fn plane() -> Box<[u64]> { + vec![0u64; TILE_WORDS].into_boxed_slice() +} + +fn set(plane: &mut [u64], i: usize) { + plane[i >> 6] |= 1 << (i & 63); +} + +impl Tile { + pub fn from_csr(csr: &Csr) -> Tile { + let mut planes = Vec::with_capacity(csr.preds.len()); + for c in &csr.preds { + let mut pl = plane(); + for s in 0..SLOTS { + for &t in &c.dst[c.offsets[s] as usize..c.offsets[s + 1] as usize] { + set(&mut pl, tile_addr(s as u8, t)); + } + } + planes.push(pl); + } + Tile { + preds: csr.preds.iter().map(|c| c.p).collect(), + planes, + witness: csr.witness, + } + } + + /// Plane bytes (8 KiB per predicate). + pub fn bytes(&self) -> usize { + self.planes.len() * TILE_WORDS * 8 + } + + pub fn plane(&self, p: u32) -> &[u64] { + let i = self + .preds + .iter() + .position(|&q| q == p) + .expect("predicate in tile"); + &self.planes[i] + } +} + +/// A thought's sparse speculative change to the shared tile: edges it adds +/// and edges it retracts. The shared tile is never written. +pub struct ThoughtDelta { + /// The thought incarnation this delta belongs to (stand-in for + /// `ThoughtCommitId`, D-RMAP-1A §7). + pub thought: u64, + pub p: u32, + add: Box<[u64]>, + del: Box<[u64]>, + edits: usize, +} + +impl ThoughtDelta { + pub fn new(thought: u64, p: u32, add: &[(u8, u8)], del: &[(u8, u8)]) -> Self { + let (mut a, mut d) = (plane(), plane()); + for &(s, t) in add { + set(&mut a, tile_addr(s, t)); + } + for &(s, t) in del { + set(&mut d, tile_addr(s, t)); + } + ThoughtDelta { + thought, + p, + add: a, + del: d, + edits: add.len() + del.len(), + } + } + + /// Bytes as stored sparsely (two bytes per edit) and as the two planes + /// the executor reads. + pub fn bytes_sparse(&self) -> usize { + self.edits * 2 + } + + pub fn bytes_planes(&self) -> usize { + 2 * TILE_WORDS * 8 + } +} + +/// Executor state for tile hops: the broadcast frontier, the hop result and +/// the mask-risc scratch, reused across calls (no allocation per hop). +pub struct TileExec { + frontier: Box<[u64]>, + out: Box<[u64]>, + scratch: Scratch<'static>, +} + +impl Default for TileExec { + fn default() -> Self { + Self::new() + } +} + +impl TileExec { + pub fn new() -> Self { + TileExec { + frontier: plane(), + out: plane(), + // Up to two scratch slots for every program built below. + scratch: Scratch::new(words_for(TILE_ROWS), 2), + } + } + + /// The hop program over `n` predicate planes (masks `0..n`), an optional + /// thought delta (masks `n` = add, `n + 1` = del) and the broadcast + /// frontier (the last mask). Every operand is a plane over the same + /// `(target, source)` address space. + fn program(n: usize, delta: bool) -> Program { + let f = Operand::Plane((n + if delta { 2 } else { 0 }) as u16); + let mut ops = Vec::new(); + let mut acc = Operand::Plane(0); + for i in 1..n { + ops.push(MaskOp::Or { + a: acc, + b: Operand::Plane(i as u16), + dst: 0, + }); + acc = Operand::Scratch(0); + } + if delta { + ops.push(MaskOp::Or { + a: acc, + b: Operand::Plane(n as u16), + dst: 0, + }); + ops.push(MaskOp::AndNot { + a: Operand::Scratch(0), + b: Operand::Plane(n as u16 + 1), + dst: 0, + }); + acc = Operand::Scratch(0); + } + ops.push(MaskOp::And { + a: acc, + b: f, + dst: 1, + }); + Program::new( + ops, + Terminal::Keep { + mask: Operand::Scratch(1), + }, + ) + } + + /// One hop: `next[t] = ∃ s. frontier[s] ∧ edge(s, t)`, run by mask-risc. + fn hop( + &mut self, + planes: &[&[u64]], + delta: Option<&ThoughtDelta>, + frontier: &SlotSet, + ) -> SlotSet { + // Broadcast: frontier[s] at every (t, s). Derived, rewritten per hop. + for t in 0..SLOTS { + self.frontier[t * SLOT_WORDS..(t + 1) * SLOT_WORDS].copy_from_slice(frontier); + } + let mut masks: Vec<&[u64]> = planes.to_vec(); + if let Some(d) = delta { + masks.push(&d.add); + masks.push(&d.del); + } + masks.push(&self.frontier); + let prog = Self::program(planes.len(), delta.is_some()); + let pl = Planes { + n_rows: TILE_ROWS, + masks: &masks, + lanes: &[], + }; + let v = execute_into( + &prog, + &pl, + &Foreign::NONE, + &mut self.scratch, + Out::Mask(&mut self.out), + ) + .expect("tile program is valid"); + debug_assert!(matches!(v, Value::Mask(_))); + let mut next = [0u64; SLOT_WORDS]; + for t in 0..SLOTS { + let row = &self.out[t * SLOT_WORDS..(t + 1) * SLOT_WORDS]; + if row.iter().any(|&w| w != 0) { + next[t >> 6] |= 1 << (t & 63); + } + } + next + } + + /// Reachability over the tile, optionally through a thought's delta. + /// Escapes come from the CSR the tile was built from. + #[expect(clippy::too_many_arguments)] + pub fn reach( + &mut self, + tile: &Tile, + csr: &Csr, + current: &Witness, + delta: Option<&ThoughtDelta>, + thought: Option, + preds: &[u32], + seeds: &SlotSet, + hops: usize, + ) -> Result { + tile.witness.check(current)?; + csr.witness.check(current)?; + if let Some(d) = delta { + if Some(d.thought) != thought { + return Err(Refusal::ThoughtMismatch); + } + assert!(preds.contains(&d.p), "delta predicate is not in the query"); + } + let planes: Vec<&[u64]> = preds.iter().map(|&p| tile.plane(p)).collect(); + let mut reached = *seeds; + let mut frontier = *seeds; + let mut escaped = BTreeSet::new(); + for _ in 0..hops { + csr.escapes_from(&frontier, preds, &mut escaped); + let next = self.hop(&planes, delta, &frontier); + for w in 0..SLOT_WORDS { + frontier[w] = next[w] & !reached[w]; + reached[w] |= next[w]; + } + if frontier == [0; SLOT_WORDS] { + break; + } + } + Ok(ScopedReach { reached, escaped }) + } +} + +impl TileExec { + /// [`TileExec::reach`] without materializing escaped identities. + #[expect(clippy::too_many_arguments)] + pub fn reach_sources( + &mut self, + tile: &Tile, + csr: &Csr, + current: &Witness, + delta: Option<&ThoughtDelta>, + thought: Option, + preds: &[u32], + seeds: &SlotSet, + hops: usize, + ) -> Result { + tile.witness.check(current)?; + csr.witness.check(current)?; + if let Some(d) = delta { + if Some(d.thought) != thought { + return Err(Refusal::ThoughtMismatch); + } + assert!(preds.contains(&d.p), "delta predicate is not in the query"); + } + let esc_plane = csr.escape_plane(preds); + let planes: Vec<&[u64]> = preds.iter().map(|&p| tile.plane(p)).collect(); + let mut reached = *seeds; + let mut frontier = *seeds; + let mut escape_sources = [0u64; SLOT_WORDS]; + for _ in 0..hops { + for w in 0..SLOT_WORDS { + escape_sources[w] |= frontier[w] & esc_plane[w]; + } + let next = self.hop(&planes, delta, &frontier); + for w in 0..SLOT_WORDS { + frontier[w] = next[w] & !reached[w]; + reached[w] |= next[w]; + } + if frontier == [0; SLOT_WORDS] { + break; + } + } + Ok(ScopedMask { + reached, + escape_sources, + }) + } +} + +/// A focus's dimensions loaded into 256-slot register lanes: a COPY of the +/// canonical values (12 bytes per slot), made once per focus. The other arm +/// ([`box_count`]) copies nothing and folds over the canonical lanes. +pub struct FocusLanes { + pub lat: Vec, + pub lon: Vec, + pub class: Vec, + /// The focus generation the copy was taken from. + pub generation: u64, +} + +impl FocusLanes { + pub fn load(places: &Places, ids: &[u32], generation: u64) -> Self { + FocusLanes { + lat: ids.iter().map(|&i| places.lat[i as usize]).collect(), + lon: ids.iter().map(|&i| places.lon[i as usize]).collect(), + class: ids.iter().map(|&i| places.class[i as usize]).collect(), + generation, + } + } + + /// Canonical payload bytes copied. + pub fn bytes(&self) -> usize { + self.lat.len() * 12 + } + + /// The [`BoxQuery`] count over the register lanes, by mask-risc. + /// Refuses a copy taken under another focus generation. + pub fn count( + &self, + generation: u64, + q: &BoxQuery, + scratch: &mut Scratch<'_>, + ) -> Result { + if generation != self.generation { + return Err(Refusal::FocusChanged); + } + let n = self.lat.len(); + let mut all = vec![u64::MAX; words_for(n)]; + if !n.is_multiple_of(64) { + *all.last_mut().unwrap() = (1u64 << (n % 64)) - 1; + } + let masks = [all.as_slice()]; + let lanes = [ + LaneRef::I32(&self.lat), + LaneRef::I32(&self.lon), + LaneRef::U32(&self.class), + ]; + let pl = Planes { + n_rows: n, + masks: &masks, + lanes: &lanes, + }; + match execute(&box_program(q), &pl, scratch, None).expect("box program is valid") { + Value::Count(c) => Ok(c), + v => unreachable!("Count terminal returned {v:?}"), + } + } +} + +// ---- incremental rebinding ------------------------------------------------- + +/// Rebind replaced slots in a prepared CSR + tile without rebuilding the +/// others. A changed slot's outgoing edges come from one forward seek each. +/// Edges INTO a newly focused identity need no reverse index: from an +/// unchanged member they were escapes before (the identity was outside the +/// focus), so re-resolving the escape list finds them; from another changed +/// slot, that slot's own forward seek finds them. Returns the new CSR and +/// tile. +// A slot number is the coordinate, not a loop counter. +#[expect(clippy::needless_range_loop)] +pub fn rebind( + forward: &SortedRun<'_>, + old: &Focus, + new: &Focus, + csr: &Csr, + witness: Witness, +) -> (Csr, Tile) { + let changed: Vec = (0..new.len()) + .filter(|&s| old.id(s as u8) != new.id(s as u8)) + .map(|s| s as u8) + .collect(); + let is_changed = slot_set(changed.iter().copied()); + let hit = |s: u8| is_changed[(s >> 6) as usize] >> (s & 63) & 1 == 1; + let mut preds = Vec::with_capacity(csr.preds.len()); + for c in &csr.preds { + // Per source slot: keep unchanged edges, drop those touching a changed + // slot, then add the re-derived ones. + let mut adj: Vec> = vec![Vec::new(); SLOTS]; + let mut esc: Vec<(u8, u32, u32)> = Vec::new(); + for s in 0..new.len() { + if hit(s as u8) { + continue; + } + let lo = c.offsets[s] as usize; + let hi = c.offsets[s + 1] as usize; + for k in lo..hi { + if !hit(c.dst[k]) { + adj[s].push((c.dst[k], c.row[k])); + } + } + } + for &(s, o, r) in &c.escaped { + // An old escape may now land on a newly focused identity. + if hit(s) { + continue; + } + match new.slot_of(o) { + Some(t) => adj[s as usize].push((t, r)), + None => esc.push((s, o, r)), + } + } + for &s in &changed { + // Outgoing edges of the new identity. + forward.neighbours(c.p, new.id(s), |o, r| match new.slot_of(o) { + Some(t) => adj[s as usize].push((t, r)), + None => esc.push((s, o, r)), + }); + } + // An edge from an unchanged member to the OLD identity of a changed + // slot that is no longer focused becomes an escape. + for s in 0..new.len() { + if hit(s as u8) { + continue; + } + let lo = c.offsets[s] as usize; + let hi = c.offsets[s + 1] as usize; + for k in lo..hi { + let t = c.dst[k]; + if hit(t) { + let g = old.id(t); + match new.slot_of(g) { + Some(t2) => adj[s].push((t2, c.row[k])), + None => esc.push((s as u8, g, c.row[k])), + } + } + } + } + let mut offsets = [0u32; SLOTS + 1]; + let mut dst = Vec::new(); + let mut row = Vec::new(); + for s in 0..SLOTS { + offsets[s] = dst.len() as u32; + adj[s].sort_unstable(); + adj[s].dedup(); + for &(t, r) in &adj[s] { + dst.push(t); + row.push(r); + } + } + offsets[SLOTS] = dst.len() as u32; + esc.sort_unstable(); + esc.dedup(); + preds.push(CsrPred { + p: c.p, + offsets, + dst, + row, + has_escape: slot_set(esc.iter().map(|e| e.0)), + escaped: esc, + }); + } + let csr = Csr { preds, witness }; + let tile = Tile::from_csr(&csr); + (csr, tile) +} + +impl Csr { + /// The prepared relation as a set, for comparing two preparations. + pub fn edge_set(&self) -> BTreeSet<(u32, u8, u8, u32)> { + let mut out = BTreeSet::new(); + for c in &self.preds { + for s in 0..SLOTS { + for k in c.offsets[s] as usize..c.offsets[s + 1] as usize { + out.insert((c.p, s as u8, c.dst[k], c.row[k])); + } + } + } + out + } + + pub fn escape_set(&self) -> BTreeSet<(u32, u8, u32, u32)> { + self.preds + .iter() + .flat_map(|c| c.escaped.iter().map(move |&(s, o, r)| (c.p, s, o, r))) + .collect() + } +} + +// ---- dimensions over canonical lanes -------------------------------------- + +/// Which row order a mask's bit positions follow. Two orders over the same +/// 65 536 rows both use 16-bit positions; equal positions mean nothing across +/// them. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Ordering { + /// The canonical row order of the lanes. + Canonical, + /// A spatial (Morton) order, by index build. + Morton(u64), +} + +/// A focus as a presence plane over a row order: the zero-copy form for typed +/// dimensions, which stay in their canonical lanes. +pub struct FocusMask { + pub words: Vec, + pub ordering: Ordering, + pub generation: u64, +} + +impl FocusMask { + pub fn over(rows: usize, ids: &[u32], ordering: Ordering, generation: u64) -> Self { + let mut words = vec![0u64; words_for(rows)]; + for &i in ids { + words[(i >> 6) as usize] |= 1 << (i & 63); + } + FocusMask { + words, + ordering, + generation, + } + } +} + +/// Canonical place lanes: latitude and longitude in micro-degrees, and a class. +pub struct Places { + pub lat: Vec, + pub lon: Vec, + pub class: Vec, +} + +/// A bounding box and a class: the demanded count. +#[derive(Clone, Copy, Debug)] +pub struct BoxQuery { + pub lat: (i32, i32), + pub lon: (i32, i32), + pub class: u32, +} + +/// The mask-risc program for [`BoxQuery`]: five gated predicates, then Count. +pub fn box_program(q: &BoxQuery) -> Program { + let gate = |i: u16| Some(Operand::Scratch(i)); + Program::new( + vec![ + MaskOp::Pred { + pred: Pred::GeI32 { + lane: 0, + t: q.lat.0, + }, + under: Some(Operand::Plane(0)), + dst: 0, + }, + MaskOp::Pred { + pred: Pred::LeI32 { + lane: 0, + t: q.lat.1, + }, + under: gate(0), + dst: 1, + }, + MaskOp::Pred { + pred: Pred::GeI32 { + lane: 1, + t: q.lon.0, + }, + under: gate(1), + dst: 0, + }, + MaskOp::Pred { + pred: Pred::LeI32 { + lane: 1, + t: q.lon.1, + }, + under: gate(0), + dst: 1, + }, + MaskOp::Pred { + pred: Pred::EqU32 { + lane: 2, + v: q.class, + }, + under: gate(1), + dst: 0, + }, + ], + Terminal::Count { + mask: Operand::Scratch(0), + }, + ) +} + +/// Count focused places matching `q`, natively, over the canonical lanes. +/// Refuses a focus mask built in another row order than the lanes'. +pub fn box_count( + places: &Places, + focus: &FocusMask, + q: &BoxQuery, + scratch: &mut Scratch<'_>, +) -> Result { + if focus.ordering != Ordering::Canonical { + return Err(Refusal::OrderingMismatch); + } + let masks = [focus.words.as_slice()]; + let lanes = [ + LaneRef::I32(&places.lat), + LaneRef::I32(&places.lon), + LaneRef::U32(&places.class), + ]; + let pl = Planes { + n_rows: places.lat.len(), + masks: &masks, + lanes: &lanes, + }; + match execute(&box_program(q), &pl, scratch, None).expect("box program is valid") { + Value::Count(n) => Ok(n), + v => unreachable!("Count terminal returned {v:?}"), + } +} + +/// The same count by reading each focused identity's lanes in turn. +pub fn box_count_by_id(places: &Places, ids: &[u32], q: &BoxQuery) -> usize { + ids.iter() + .filter(|&&i| { + let i = i as usize; + (q.lat.0..=q.lat.1).contains(&places.lat[i]) + && (q.lon.0..=q.lon.1).contains(&places.lon[i]) + && places.class[i] == q.class + }) + .count() +} + +/// Morton order of the places: row numbers sorted by interleaved quantized +/// coordinates. An index of `u32` row numbers; the coordinates are not copied. +pub fn morton_order(places: &Places) -> Vec { + let q = |v: i32, lo: i64, span: i64| (((v as i64 - lo) * 65_535) / span) as u32; + let key = |i: usize| { + let x = q(places.lon[i], -180_000_000, 360_000_000); + let y = q(places.lat[i], -90_000_000, 180_000_000); + let mut k = 0u64; + for b in 0..16 { + k |= (((x >> b) & 1) as u64) << (2 * b); + k |= (((y >> b) & 1) as u64) << (2 * b + 1); + } + k + }; + let mut perm: Vec = (0..places.lat.len() as u32).collect(); + perm.sort_unstable_by_key(|&i| key(i as usize)); + perm +} + +// ---- membership planes (map × identity) ------------------------------------ + +/// Map × identity: one 256-bit plane per selected predicate over the focus +/// slots. Built from posting lists through the focus binding. +pub fn membership_planes(postings: &[&[u32]], focus: &Focus) -> Vec { + postings + .iter() + .map(|docs| slot_set(docs.iter().filter_map(|&d| focus.slot_of(d)))) + .collect() +} + +/// Intersect two membership planes with mask-risc: And, then Keep. Returns +/// the focused identities, in slot order. +pub fn intersect_planes( + a: &SlotSet, + b: &SlotSet, + focus: &Focus, + scratch: &mut Scratch<'_>, +) -> Vec { + let masks = [a.as_slice(), b.as_slice()]; + let pl = Planes { + n_rows: SLOTS, + masks: &masks, + lanes: &[], + }; + let prog = Program::new( + vec![MaskOp::And { + a: Operand::Plane(0), + b: Operand::Plane(1), + dst: 0, + }], + Terminal::Keep { + mask: Operand::Scratch(0), + }, + ); + let mut out = [0u64; SLOT_WORDS]; + execute_into(&prog, &pl, &Foreign::NONE, scratch, Out::Mask(&mut out)).expect("valid"); + let mut ids: Vec = slots_of(&out).into_iter().map(|s| focus.id(s)).collect(); + ids.sort_unstable(); + ids +} + +/// The count of the same intersection, as And then Count (POPCNT). +pub fn intersect_count(a: &SlotSet, b: &SlotSet, scratch: &mut Scratch<'_>) -> usize { + let masks = [a.as_slice(), b.as_slice()]; + let pl = Planes { + n_rows: SLOTS, + masks: &masks, + lanes: &[], + }; + let prog = Program::new( + vec![MaskOp::And { + a: Operand::Plane(0), + b: Operand::Plane(1), + dst: 0, + }], + Terminal::Count { + mask: Operand::Scratch(0), + }, + ); + match execute(&prog, &pl, scratch, None).expect("valid") { + Value::Count(n) => n, + v => unreachable!("{v:?}"), + } +} + +/// Merge-intersect two sorted posting slices (the ordered-plane arm). +pub fn intersect_sorted(a: &[u32], b: &[u32]) -> Vec { + let (mut i, mut j, mut out) = (0, 0, Vec::new()); + while i < a.len() && j < b.len() { + match a[i].cmp(&b[j]) { + std::cmp::Ordering::Equal => { + out.push(a[i]); + i += 1; + j += 1; + } + std::cmp::Ordering::Less => i += 1, + std::cmp::Ordering::Greater => j += 1, + } + } + out +} + +// ---- deterministic fixtures ------------------------------------------------- + +/// SplitMix64. +pub struct Rng(pub u64); + +impl Rng { + pub fn next_u64(&mut self) -> u64 { + self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } + + pub fn below(&mut self, n: u64) -> u64 { + self.next_u64() % n + } +} + +/// A sparse typed graph over `n` identities and `preds` predicates. Identities +/// are scattered: community membership is a hidden permutation of the ids, so +/// a community's members are spread over the whole id space. `local` is the +/// share of edges that stay inside their community (of size `community`). +pub fn graph(n: u32, preds: u32, degree: u32, community: u32, local: f64, seed: u64) -> Vec { + let mut rng = Rng(seed); + // Hidden position <-> id bijection. + let mut pos_to_id: Vec = (0..n).collect(); + for i in (1..n as usize).rev() { + let j = rng.below(i as u64 + 1) as usize; + pos_to_id.swap(i, j); + } + let mut id_to_pos = vec![0u32; n as usize]; + for (p, &id) in pos_to_id.iter().enumerate() { + id_to_pos[id as usize] = p as u32; + } + let mut facts = Vec::new(); + for s in 0..n { + let c = id_to_pos[s as usize] / community; + for p in 0..preds { + for _ in 0..degree { + let o = if (rng.below(1_000_000) as f64) < local * 1e6 { + let pos = c * community + rng.below(community as u64) as u32; + pos_to_id[pos.min(n - 1) as usize] + } else { + rng.below(n as u64) as u32 + }; + if o != s { + facts.push([s, p, o]); + } + } + } + } + facts.sort_unstable(); + facts.dedup(); + // Canonical order is not the index order. + for i in (1..facts.len()).rev() { + let j = rng.below(i as u64 + 1) as usize; + facts.swap(i, j); + } + facts +} + +/// Places with scattered coordinates and 16 classes. +pub fn places(n: usize, seed: u64) -> Places { + let mut rng = Rng(seed); + let mut p = Places { + lat: Vec::with_capacity(n), + lon: Vec::with_capacity(n), + class: Vec::with_capacity(n), + }; + for _ in 0..n { + p.lat.push(rng.below(180_000_000) as i32 - 90_000_000); + p.lon.push(rng.below(360_000_000) as i32 - 180_000_000); + p.class.push(rng.below(16) as u32); + } + p +} diff --git a/crates/mork-pathmap-probe/tests/rmap.rs b/crates/mork-pathmap-probe/tests/rmap.rs new file mode 100644 index 000000000..6a0caea47 --- /dev/null +++ b/crates/mork-pathmap-probe/tests/rmap.rs @@ -0,0 +1,536 @@ +//! D-RMAP focus probe against an independent oracle. +//! +//! The oracle scans the canonical fact slice directly: it uses no index, no +//! focus binding and no prepared operand, so agreement is not one structure +//! agreeing with itself. + +use std::collections::BTreeSet; + +use lance_graph_mask_risc::Scratch; +use mork_pathmap_probe::boundary::PostingTrie; +use mork_pathmap_probe::rmap::*; + +const PREDS: [u32; 3] = [0, 1, 2]; + +/// Scoped reachability by scanning all facts: the induced subgraph of the +/// focus, BFS by levels, escapes collected from reached nodes within budget. +fn oracle( + facts: &[Fact], + focus: &[u32], + preds: &[u32], + seeds: &[u32], + hops: usize, +) -> (BTreeSet, BTreeSet) { + let inf: BTreeSet = focus.iter().copied().collect(); + let mut reached: BTreeSet = seeds.iter().copied().collect(); + let mut frontier = reached.clone(); + let mut escaped = BTreeSet::new(); + for _ in 0..hops { + let mut next = BTreeSet::new(); + for &[s, p, o] in facts { + if frontier.contains(&s) && preds.contains(&p) { + if inf.contains(&o) { + next.insert(o); + } else { + escaped.insert(o); + } + } + } + frontier = next.difference(&reached).copied().collect(); + reached.extend(next); + if frontier.is_empty() { + break; + } + } + (reached, escaped) +} + +fn ids_of(focus: &Focus, set: &SlotSet) -> BTreeSet { + slots_of(set).into_iter().map(|s| focus.id(s)).collect() +} + +struct World { + facts: Vec, +} + +fn world() -> World { + World { + facts: graph(65_536, 3, 2, 512, 0.9, 0x5EED), + } +} + +fn w(seal: u64, focus: u64) -> Witness { + Witness { + facts: 1, + seal, + focus, + scope: 7, + } +} + +/// One hop, two hops, three hops, single and multiple predicates: every +/// prepared arm equals the oracle, and so does the direct walk over both +/// indexes (sorted run and PathMap trie). +#[test] +fn every_arm_matches_the_oracle_on_one_two_and_three_hops() { + let wd = world(); + let run = SortedRun::build(&wd.facts, false, 1); + let trie = SpogTrie::build(&wd.facts); + let focus = discover(&run, &[17], &PREDS, SLOTS); + assert_eq!(focus.len(), SLOTS, "the fixture must yield a full focus"); + let focus_trie = discover(&trie, &[17], &PREDS, SLOTS); + assert_eq!( + focus.ids(), + focus_trie.ids(), + "both indexes discover the same focus in the same order" + ); + let cur = w(1, focus.generation); + let csr = Csr::prepare(&run, &focus, &PREDS, cur); + let tile = Tile::from_csr(&csr); + let mut ex = TileExec::new(); + let mut checked = 0; + for preds in [&PREDS[..1], &PREDS[..]] { + for hops in 1..=3 { + for seed_slot in [0u8, 40, 255] { + let seeds = slot_set([seed_slot]); + let (want_r, want_e) = + oracle(&wd.facts, focus.ids(), preds, &[focus.id(seed_slot)], hops); + let direct = reach_direct(&run, &focus, preds, &seeds, hops); + let via_trie = reach_direct(&trie, &focus, preds, &seeds, hops); + let c = csr.reach(&cur, preds, &seeds, hops).unwrap(); + let t = ex + .reach(&tile, &csr, &cur, None, None, preds, &seeds, hops) + .unwrap(); + for (name, got) in [ + ("direct", &direct), + ("trie", &via_trie), + ("csr", &c), + ("tile", &t), + ] { + assert_eq!( + ids_of(&focus, &got.reached), + want_r, + "{name} reached, preds {preds:?}, hops {hops}" + ); + assert_eq!( + got.escaped, want_e, + "{name} escaped, preds {preds:?}, hops {hops}" + ); + } + // The mask mode gives the same reach, and its escape sources + // turn into exactly the escaped identities. + let cm = csr.reach_sources(&cur, preds, &seeds, hops).unwrap(); + let tm = ex + .reach_sources(&tile, &csr, &cur, None, None, preds, &seeds, hops) + .unwrap(); + assert_eq!(cm, tm); + assert_eq!(reach_direct_sources(&run, &focus, preds, &seeds, hops), cm); + assert_eq!(ids_of(&focus, &cm.reached), want_r); + assert_eq!(csr.escape_ids(&cm.escape_sources, preds), want_e); + assert_eq!(cm.global().is_ok(), want_e.is_empty()); + checked += 1; + } + } + } + assert_eq!(checked, 18); +} + +/// Anti-vacuity: the fixture exercises what the assertions above claim. +/// Three hops reach more than one hop, edges leave the focus, and a reached +/// set is neither empty nor the whole focus. +#[test] +fn the_fixture_is_not_degenerate() { + let wd = world(); + let run = SortedRun::build(&wd.facts, false, 1); + let focus = discover(&run, &[17], &PREDS, SLOTS); + let csr = Csr::prepare(&run, &focus, &PREDS, w(1, 1)); + assert!(csr.edges() > SLOTS, "in-focus edges: {}", csr.edges()); + assert!(csr.escapes() > 0, "some edges must leave the focus"); + let seeds = slot_set([40]); + let one = csr.reach(&w(1, 1), &PREDS, &seeds, 1).unwrap(); + let three = csr.reach(&w(1, 1), &PREDS, &seeds, 3).unwrap(); + let n1 = slots_of(&one.reached).len(); + let n3 = slots_of(&three.reached).len(); + assert!(n1 > 1 && n3 > n1 && n3 < SLOTS, "1 hop {n1}, 3 hops {n3}"); + assert!(!three.escaped.is_empty()); +} + +/// An out-of-focus target is never a relationship: it is reported, and the +/// global question is refused while it exists. +#[test] +fn an_out_of_focus_target_is_reported_never_reached() { + // 0 -> 1 -> 9 -> 2; the focus is {0, 1, 2}. The global answer from 0 in + // three hops contains 2, but only through 9, which is not focused. + let facts: Vec = vec![[0, 0, 1], [1, 0, 9], [9, 0, 2]]; + let run = SortedRun::build(&facts, false, 1); + let focus = Focus::bind(&[0, 1, 2], 1); + let csr = Csr::prepare(&run, &focus, &[0], w(1, 1)); + let r = csr.reach(&w(1, 1), &[0], &slot_set([0]), 3).unwrap(); + assert_eq!( + ids_of(&focus, &r.reached), + BTreeSet::from([0, 1]), + "2 is not reachable INSIDE the focus" + ); + assert_eq!(r.escaped, BTreeSet::from([9])); + assert_eq!(r.global(), Err(Refusal::IncompleteFocus)); + // Silence twin: with 9 focused, nothing escapes and the global answer is given. + let focus = Focus::bind(&[0, 1, 2, 9], 1); + let csr = Csr::prepare(&run, &focus, &[0], w(1, 1)); + let r = csr.reach(&w(1, 1), &[0], &slot_set([0]), 3).unwrap(); + assert!(r.escaped.is_empty()); + assert_eq!( + ids_of(&focus, r.global().unwrap()), + BTreeSet::from([0, 1, 2, 9]) + ); +} + +/// Every in-focus edge keeps the canonical fact it came from. +#[test] +fn prepared_edges_keep_their_canonical_rows() { + let wd = world(); + let run = SortedRun::build(&wd.facts, false, 1); + let focus = discover(&run, &[17], &PREDS, SLOTS); + let csr = Csr::prepare(&run, &focus, &PREDS, w(1, 1)); + let mut n = 0; + for (p, s, t, row) in csr.edge_set() { + assert_eq!(wd.facts[row as usize], [focus.id(s), p, focus.id(t)]); + n += 1; + } + assert_eq!(n, csr.edges()); +} + +/// Each validity dimension refuses on its own, and a matching witness passes. +#[test] +fn a_stale_operand_is_refused_per_dimension() { + let wd = world(); + let run = SortedRun::build(&wd.facts, false, 1); + let focus = discover(&run, &[17], &PREDS, SLOTS); + let bound = w(1, focus.generation); + let csr = Csr::prepare(&run, &focus, &PREDS, bound); + let tile = Tile::from_csr(&csr); + let seeds = slot_set([0]); + let mut ex = TileExec::new(); + assert!(csr.reach(&bound, &PREDS, &seeds, 2).is_ok()); + let cases = [ + (Witness { facts: 2, ..bound }, Refusal::FactsChanged), + (Witness { seal: 2, ..bound }, Refusal::IndexResealed), + ( + Witness { + focus: bound.focus + 1, + ..bound + }, + Refusal::FocusChanged, + ), + (Witness { scope: 8, ..bound }, Refusal::ScopeChanged), + ]; + for (cur, why) in cases { + assert_eq!(csr.reach(&cur, &PREDS, &seeds, 2).err(), Some(why)); + assert_eq!( + ex.reach(&tile, &csr, &cur, None, None, &PREDS, &seeds, 2) + .err(), + Some(why) + ); + } +} + +/// A recycled thought slot: a delta built for one thought incarnation is +/// refused for another, even when the slot number is the same. +#[test] +fn a_thought_delta_is_refused_for_another_incarnation() { + let wd = world(); + let run = SortedRun::build(&wd.facts, false, 1); + let focus = discover(&run, &[17], &PREDS, SLOTS); + let cur = w(1, focus.generation); + let csr = Csr::prepare(&run, &focus, &PREDS, cur); + let tile = Tile::from_csr(&csr); + let delta = ThoughtDelta::new(41, 0, &[(0, 200)], &[]); + let mut ex = TileExec::new(); + let seeds = slot_set([0]); + assert!( + ex.reach(&tile, &csr, &cur, Some(&delta), Some(41), &PREDS, &seeds, 1) + .is_ok() + ); + assert_eq!( + ex.reach(&tile, &csr, &cur, Some(&delta), Some(42), &PREDS, &seeds, 1) + .err(), + Some(Refusal::ThoughtMismatch) + ); + assert_eq!( + ex.reach_sources(&tile, &csr, &cur, Some(&delta), Some(42), &PREDS, &seeds, 1) + .err(), + Some(Refusal::ThoughtMismatch) + ); +} + +/// Thought deltas over one shared tile: added and retracted edges change only +/// that thought's answer, equal an oracle over the edited fact set, and leave +/// the shared tile and other thoughts untouched. +#[test] +fn thought_deltas_change_only_their_own_thought() { + let wd = world(); + let run = SortedRun::build(&wd.facts, false, 1); + let focus = discover(&run, &[17], &PREDS, SLOTS); + let cur = w(1, focus.generation); + let csr = Csr::prepare(&run, &focus, &[0], cur); + let tile = Tile::from_csr(&csr); + let mut ex = TileExec::new(); + let seeds = slot_set([0]); + let base = ex + .reach(&tile, &csr, &cur, None, None, &[0], &seeds, 2) + .unwrap(); + // Retract every edge out of slot 0, add 0 -> 255. + let out_of_0: Vec<(u8, u8)> = csr + .edge_set() + .into_iter() + .filter(|e| e.1 == 0) + .map(|e| (0u8, e.2)) + .collect(); + assert!(!out_of_0.is_empty()); + let delta = ThoughtDelta::new(5, 0, &[(0, 255)], &out_of_0); + let t = ex + .reach(&tile, &csr, &cur, Some(&delta), Some(5), &[0], &seeds, 2) + .unwrap(); + // Oracle over the edited facts (in-focus part only; escapes are the CSR's). + let mut edited: Vec = wd + .facts + .iter() + .copied() + .filter(|f| !(f[0] == focus.id(0) && f[1] == 0 && focus.slot_of(f[2]).is_some())) + .collect(); + edited.push([focus.id(0), 0, focus.id(255)]); + let (want, _) = oracle(&edited, focus.ids(), &[0], &[focus.id(0)], 2); + assert_eq!(ids_of(&focus, &t.reached), want); + assert_ne!( + t.reached, base.reached, + "the delta must change this thought's answer" + ); + let again = ex + .reach(&tile, &csr, &cur, None, None, &[0], &seeds, 2) + .unwrap(); + assert_eq!(again.reached, base.reached, "the shared tile is unchanged"); +} + +/// Rebinding k slots incrementally gives exactly what a full preparation of +/// the new focus gives, for k = 1, 8, 16, 128, 256. +#[test] +fn incremental_rebind_equals_full_preparation() { + let wd = world(); + let fwd = SortedRun::build(&wd.facts, false, 1); + let focus = discover(&fwd, &[17], &PREDS, SLOTS); + let csr = Csr::prepare(&fwd, &focus, &PREDS, w(1, focus.generation)); + // Replacement identities: a second community's focus (some may already be + // in the first, which exercises slot swaps). + let other = discover(&fwd, &[40_000], &PREDS, SLOTS); + // Two sources of replacements: another community (few edges into the + // focus), and the focus's own escape targets (focus expansion: every + // replacement already has an edge INTO it from the focus, which is what + // the escape re-resolution must recover). + let mut escaped: Vec = csr.escape_set().into_iter().map(|e| e.2).collect(); + escaped.sort_unstable(); + escaped.dedup(); + assert!( + escaped.len() >= 256, + "expansion needs 256 escape targets: {}", + escaped.len() + ); + let mut recovered_incoming = 0; + for (mode, source) in [ + ("other community", other.ids().to_vec()), + ("expansion", escaped), + ] { + for k in [1usize, 8, 16, 128, 256] { + let mut taken: BTreeSet = focus.ids().iter().copied().collect(); + let mut changes = Vec::new(); + for (s, &g) in (0..k).zip(source.iter()) { + if taken.insert(g) { + changes.push(((SLOTS - 1 - s) as u8, g)); + } + } + let new = focus.replace(&changes); + let nw = w(1, new.generation); + let (inc, inc_tile) = rebind(&fwd, &focus, &new, &csr, nw); + let full = Csr::prepare(&fwd, &new, &PREDS, nw); + assert_eq!( + inc.edge_set(), + full.edge_set(), + "{mode}: edges after rebinding {k}" + ); + // Edges from an unchanged slot into a changed one: the case only the + // escape re-resolution finds. + let changed: BTreeSet = changes.iter().map(|c| c.0).collect(); + recovered_incoming += full + .edge_set() + .iter() + .filter(|e| !changed.contains(&e.1) && changed.contains(&e.2)) + .count(); + assert_eq!( + inc.escape_set(), + full.escape_set(), + "escapes after rebinding {k}" + ); + let full_tile = Tile::from_csr(&full); + for &p in &PREDS { + assert_eq!( + inc_tile.plane(p), + full_tile.plane(p), + "tile after rebinding {k}" + ); + } + // A changed focus refuses the old preparation. + assert_eq!( + csr.reach(&nw, &PREDS, &slot_set([0]), 1).err(), + Some(Refusal::FocusChanged) + ); + } + } + assert!( + recovered_incoming > 0, + "the fixture must contain edges into replaced slots" + ); +} + +/// PathMap's `restrict` cuts the focus's outgoing facts out of the fact trie: +/// exactly the facts whose subject is focused, and nothing else. +#[test] +fn trie_restrict_yields_exactly_the_focus_facts() { + let wd = world(); + let trie = SpogTrie::build(&wd.facts); + let run = SortedRun::build(&wd.facts, false, 1); + let focus = discover(&run, &[17], &PREDS, SLOTS); + let pairs = PREDS + .iter() + .flat_map(|&p| focus.ids().iter().map(move |&s| (p, s))); + let cut = trie.restrict_to(pairs); + let inf: BTreeSet = focus.ids().iter().copied().collect(); + let want = wd.facts.iter().filter(|f| inf.contains(&f[0])).count(); + assert_eq!(cut.len(), want); + assert!(want > 0 && want < wd.facts.len()); + for &s in focus.ids() { + for &p in &PREDS { + let (mut a, mut b) = (Vec::new(), Vec::new()); + cut.objects(p, s, |o, r| a.push((o, r))); + trie.objects(p, s, |o, r| b.push((o, r))); + assert_eq!(a, b); + } + } +} + +/// Fixture 2: the inverted index. Every arm returns Doc21 and Doc39 as +/// canonical identities, on the small case and at scale. +#[test] +fn inverted_index_intersection_agrees_across_arms() { + let mut postings: Vec> = vec![vec![17, 21, 39], vec![21, 39, 82]]; + // Scale: 254 more terms over 65 536 docs. + let mut rng = Rng(9); + for _ in 0..254 { + let mut v: Vec = (0..40).map(|_| rng.below(65_536) as u32).collect(); + v.sort_unstable(); + v.dedup(); + postings.push(v); + } + let pairs: Vec<[u32; 2]> = postings + .iter() + .enumerate() + .flat_map(|(t, d)| d.iter().map(move |&x| [t as u32, x])) + .collect(); + let trie = PostingTrie::build(&pairs); + // Focus: the union of the two queried terms' docs (the scope). + let mut scope: Vec = postings[0].iter().chain(&postings[1]).copied().collect(); + scope.sort_unstable(); + scope.dedup(); + let focus = Focus::bind(&scope, 1); + let planes = membership_planes(&[&postings[0], &postings[1]], &focus); + let mut scratch = Scratch::new(SLOT_WORDS, 2); + let a = intersect_sorted(&postings[0], &postings[1]); + let b = trie.intersect(0, 1); + let c = intersect_planes(&planes[0], &planes[1], &focus, &mut scratch); + let d = intersect_count(&planes[0], &planes[1], &mut scratch); + assert_eq!(a, vec![21, 39]); + assert_eq!(b, a); + assert_eq!(c, a); + assert_eq!(d, 2); + // At scale, every pair of terms, sorted merge vs trie. + for x in 0..postings.len() as u32 { + let y = (x * 7 + 3) % postings.len() as u32; + assert_eq!( + trie.intersect(x, y), + intersect_sorted(&postings[x as usize], &postings[y as usize]) + ); + } +} + +/// Fixture 1: a focus over 65 536 places is a presence plane over the +/// canonical lanes. The native count equals reading each identity's lanes, +/// and no coordinate is copied. +#[test] +fn place_focus_counts_natively_over_canonical_lanes() { + let places = places(65_536, 3); + let mut rng = Rng(11); + let mut ids: Vec = Vec::new(); + while ids.len() < SLOTS { + let i = rng.below(65_536) as u32; + if !ids.contains(&i) { + ids.push(i); + } + } + let focus = FocusMask::over(65_536, &ids, Ordering::Canonical, 1); + let mut scratch = Scratch::new(lance_graph_mask_risc::words_for(65_536), 2); + let lanes = FocusLanes::load(&places, &ids, 1); + assert_eq!( + lanes.bytes(), + SLOTS * 12, + "the register copy is the only payload copied" + ); + let mut small = Scratch::new(SLOT_WORDS, 2); + let mut nonzero = 0; + for k in 0..64 { + let q = BoxQuery { + lat: (-90_000_000 + k * 2_000_000, 90_000_000), + lon: (-180_000_000, 180_000_000 - k * 4_000_000), + class: (k as u32) % 16, + }; + let native = box_count(&places, &focus, &q, &mut scratch).unwrap(); + assert_eq!(native, box_count_by_id(&places, &ids, &q)); + assert_eq!(lanes.count(1, &q, &mut small).unwrap(), native); + nonzero += usize::from(native > 0); + } + assert!(nonzero > 8, "the queries must select something: {nonzero}"); + // The copy is tied to its focus generation. + let q = BoxQuery { + lat: (-90_000_000, 90_000_000), + lon: (-180_000_000, 180_000_000), + class: 0, + }; + assert_eq!(lanes.count(2, &q, &mut small), Err(Refusal::FocusChanged)); +} + +/// Two row orders both use 16-bit positions. A focus built in Morton order is +/// refused against canonical lanes; read as if canonical, it would select +/// different places (the falsifier for "equal ordinals, equal identities"). +#[test] +fn a_mask_in_another_row_order_is_refused() { + let places = places(65_536, 3); + let morton = morton_order(&places); + let ids: Vec = (0..SLOTS as u32).collect(); // Morton positions 0..256 + let as_morton = FocusMask::over(65_536, &ids, Ordering::Morton(1), 1); + let mut scratch = Scratch::new(lance_graph_mask_risc::words_for(65_536), 2); + let q = BoxQuery { + lat: (-90_000_000, 90_000_000), + lon: (-180_000_000, 180_000_000), + class: 3, + }; + assert_eq!( + box_count(&places, &as_morton, &q, &mut scratch), + Err(Refusal::OrderingMismatch) + ); + // What the refusal prevents: positions 0..256 in Morton order are not + // rows 0..256. + let rows: Vec = morton[..SLOTS].to_vec(); + let right = box_count_by_id(&places, &rows, &q); + let wrong = box_count_by_id(&places, &ids, &q); + assert_ne!(rows, ids); + assert_ne!( + right, wrong, + "the two readings must disagree on this fixture" + ); +} From 9108865b8f9541cd873e26e3957662f14cd1c169 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 00:00:15 +0000 Subject: [PATCH 4/4] Native join over a versioned relationship table MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Schnellverschluss native arm was blocked on "a co-addressed producer for the input shape". This adds one as a probe: JoinTable lays R(x, y) and S(y, z) out by the slot of y in one shared domain, each side tied to its own dataset generation. The leapfrog y-level intersection becomes one mask-risc And over two presence planes; enumeration reads both sides only at the surviving slot, so no operand is read at a position taken from another's values. Not admitted: the global-to-local binding still needs the D-RMAP-1 BIND membrane review, and nothing is wired into Quack or R2IL. - src/join.rs: JoinDomain, Side, JoinTable (prepare, prepare_star, replace_side, check), JoinExec (joint, answers, for_each_answer, bag_count), filter planes per thought. - boundary.rs: Backend::Native answers through the table, for set and bag semantics. - tests: 8 join tests and native arms in the differential suite against the nested-loop oracle; 7 guards disable-verified. - examples/join_cost.rs: native vs PathMap vs MORK on the cost.rs fixtures and a heterogeneous one. Measured at n = 50 000: prepare 2.7 ms (PathMap 14.7, MORK 22.9); set answers 7.4 ms (26.1, 144); bag count 36 us; reseal of one side 0.21 ms vs 2.47 ms for the table. Cyclic bodies are not covered. Recorded in the Schnellverschluss plan §7b, D-RMAP-1A §8.8a, STATUS_BOARD D-MPS-8 and a board entry. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01VyWZYKycy6MoYDxHRVnV5K --- .claude/board/STATUS_BOARD.md | 1 + ...26-10-19-native-join-relationship-table.md | 15 + .claude/board/entries/README.md | 3 +- ...map-separate-tables-thought-palettes-v1.md | 4 + .../mork-pathmap-schnellverschluss-v1.md | 33 +- .../mork-pathmap-probe/examples/join_cost.rs | 219 ++++++++++ crates/mork-pathmap-probe/src/boundary.rs | 30 +- crates/mork-pathmap-probe/src/join.rs | 407 ++++++++++++++++++ crates/mork-pathmap-probe/src/lib.rs | 4 +- .../mork-pathmap-probe/tests/differential.rs | 35 +- crates/mork-pathmap-probe/tests/join.rs | 213 +++++++++ 11 files changed, 941 insertions(+), 23 deletions(-) create mode 100644 .claude/board/entries/2026-10-19-native-join-relationship-table.md create mode 100644 crates/mork-pathmap-probe/examples/join_cost.rs create mode 100644 crates/mork-pathmap-probe/src/join.rs create mode 100644 crates/mork-pathmap-probe/tests/join.rs diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index c13c6dfba..bef9f6044 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -25,6 +25,7 @@ These PRs merged with no board rows. Every row below is a PROPOSAL (documents an | D-RMAP-1A | separate map tables, borrowed projections, thought-selected palettes (`d-rmap-separate-tables-thought-palettes-v1.md`) | PROPOSAL; generation witnesses corrected 2026-10-19 (§6a) | refusal on map regeneration, palette-slot reassignment, focus-slot reuse, incarnation reuse, stale tile, cross-tenant change | | D-RMAP-1A-V | versioning crosswalk: map/palette/binding generations derived from D-TGA `ThoughtCommitId` + #1605 `DurableRef`; focus generation and index seal retained (`d-rmap-separate-tables-thought-palettes-v1.md` §7) | PROPOSED; no code, no type; falsifiers specified, none run | X1–X9 in §7.4 (cross-dataset, compaction, palette ABA, slot reuse, uncommitted change, rebase, authorization, squash+replay, deletion/DRed) | | D-RMAP-1A-P | focus-bound RelationshipMap probe over existing sorted/trie indexes: CSR, 256×256 tile via mask-risc, incremental rebind, thought deltas, place masks, postings (`d-rmap-separate-tables-thought-palettes-v1.md` §8; `crates/mork-pathmap-probe/src/rmap.rs`) | MEASURED, probe only; nothing adopted | 12 tests vs scan oracle; 11 guards disable-verified; break-even 76/11/1.8 reuses (sparse d2 / d8 / dense) | +| D-MPS-8 | native join over a versioned relationship table: R and S laid out by `y` slot, leapfrog intersection as one mask-risc `And`, per-side generations and reseal, per-thought filter planes (`mork-pathmap-schnellverschluss-v1.md` §7b; `crates/mork-pathmap-probe/src/join.rs`) | MEASURED, probe only; not admitted (D-RMAP-1 BIND membrane review) | 8 join tests + native arms in the differential suite vs the oracle; 7 guards disable-verified; cyclic bodies open | | D-RMAP-2 | context-declaration reuse and GraphBLAS/BF16 tile probe (`d-rmap-2-context-tile-algebra-probe-v1.md`) | PROPOSAL; not run | arms 1–5 in the plan; 297× (D-GTM-0j) is not evidence | | D-SRAM-0 | inventory Soufflé AST/ast2ram/RAM passes against D-ASM/D-RA/D-TGA owners | Queued | evidence per mapped pass; no code imported | | D-SRAM-1 | bound AST → RAM-like schedule for pure conjunctive selection | Queued | same answers and read sets on hand-built fixtures; unsupported ops typed-refused | diff --git a/.claude/board/entries/2026-10-19-native-join-relationship-table.md b/.claude/board/entries/2026-10-19-native-join-relationship-table.md new file mode 100644 index 000000000..4a8eb41d2 --- /dev/null +++ b/.claude/board/entries/2026-10-19-native-join-relationship-table.md @@ -0,0 +1,15 @@ +# Native join over a versioned relationship table (2026-10-19) + +**Status:** MEASURED (D-MPS-8), probe only; not admitted. Record: `.claude/plans/mork-pathmap-schnellverschluss-v1.md` §7b. Code: `crates/mork-pathmap-probe/src/join.rs`. + +## Measured + +- `Q(x,y,z) :- R(x,y), S(y,z)` with R and S laid out by the slot of `y` in one shared domain. The leapfrog `y`-level intersection is one mask-risc `And` over two presence planes; enumeration reads both sides at the surviving slot only. No gather. Native now answers in the differential suite, for set and bag semantics. +- n = 50 000: prepare 2.7 ms (PathMap 14.7, MORK 22.9). Set answers into a map 7.4 ms (PathMap 26.1, MORK 144). Streamed answers 0.33 ms; bag count 36 µs; joint plane 0.15 µs. +- One dataset resealed: 0.21 ms to re-prepare its side vs 2.47 ms for the table. 64 thoughts with their own `y` filter planes: 5.2 µs each over one shared table. +- The presence-plane `And` is an answer only as the existence mask; for enumeration and counts it prunes. Seven guards disable-verified. + +## Open + +- Cyclic bodies (triangle) need two variables co-addressed at once; chain joins over large domains would need a forbidden scatter. +- Admission of the global-to-local binding as a membrane (D-RMAP-1 BIND review). diff --git a/.claude/board/entries/README.md b/.claude/board/entries/README.md index 70aa89da5..c96592ef1 100644 --- a/.claude/board/entries/README.md +++ b/.claude/board/entries/README.md @@ -25,11 +25,12 @@ index row, (3) no duplicate entry id. Checks 1 and 2 are deliberately opposite directions; the stranding this convention prevents shows up in exactly one of them, never both. -373 entries, 2026-08-06 .. 2026-10-19. +374 entries, 2026-08-06 .. 2026-10-19. | date | entry id | finding | file | |---|---|---|---| | 2026-10-19 | `post-merge-recovery-1601-1617` | | [2026-10-19-post-merge-recovery-1601-1617.md](2026-10-19-post-merge-recovery-1601-1617.md) | +| 2026-10-19 | `native-join-relationship-table` | | [2026-10-19-native-join-relationship-table.md](2026-10-19-native-join-relationship-table.md) | | 2026-10-19 | `mork-pathmap-schnellverschluss` | | [2026-10-19-mork-pathmap-schnellverschluss.md](2026-10-19-mork-pathmap-schnellverschluss.md) | | 2026-10-19 | `D-RMAP-1A` | | [2026-10-19-d-rmap-thought-git-versioning.md](2026-10-19-d-rmap-thought-git-versioning.md) | | 2026-10-19 | `D-RMAP-1` | | [2026-10-19-d-rmap-implicit-relationship-register.md](2026-10-19-d-rmap-implicit-relationship-register.md) | diff --git a/.claude/plans/d-rmap-separate-tables-thought-palettes-v1.md b/.claude/plans/d-rmap-separate-tables-thought-palettes-v1.md index 47d6dc520..38116717f 100644 --- a/.claude/plans/d-rmap-separate-tables-thought-palettes-v1.md +++ b/.claude/plans/d-rmap-separate-tables-thought-palettes-v1.md @@ -288,6 +288,10 @@ Per-thought delta: 32 B as sparse edits, 16 KiB when lowered to executor planes. On 65 536 places with a focus of 256, a bounding-box + class count through mask-risc over the canonical lanes takes 20–24 µs with 0 B copied, while loading the focus's lanes into a 3 072 B register and counting there takes 0.78 + 0.38–1.27 µs. The canonical-lane path is slow because the `under` gate is word-granular: 228–235 of 1 024 words are live for 256 rows. This is an open tension between zero-copy and a register copy, not a decision; bit-granular gating would remove it. +### 8.8a Joins over heterogeneous relations + +The same preparation idea, applied to a shared-variable join between two datasets, is in the Schnellverschluss plan §7b: a versioned relationship table lays R and S out at the `y` slot, so the leapfrog intersection is one mask-risc `And` and each side reseals on its own. Measured there: prepare 2.7 ms vs PathMap 14.7 ms and MORK 22.9 ms at n = 50 000; bag count 36 µs per query; cyclic bodies not covered. + ### 8.8 Not covered, and blockers Lance-backed `DurableRef` and real stable-row-id seals (the probe's seals are integers); cold-cache and page-fault measurement; MORK matching; any gather-admission ruling. The probe answers §6's economics for in-memory, warm, single-machine fixtures only. diff --git a/.claude/plans/mork-pathmap-schnellverschluss-v1.md b/.claude/plans/mork-pathmap-schnellverschluss-v1.md index c2aefa575..a00d00a51 100644 --- a/.claude/plans/mork-pathmap-schnellverschluss-v1.md +++ b/.claude/plans/mork-pathmap-schnellverschluss-v1.md @@ -1,6 +1,6 @@ # MORK / PathMap Schnellverschluss — v1 -**Status:** D-MPS-0 and D-MPS-1 MEASURED (probe `crates/mork-pathmap-probe`, results only). D-MPS-2..7 PROPOSAL / BLOCKED as listed in §8. No production executor, no CE64 change, no Lance storage, no upstream file edited by this work. +**Status:** D-MPS-0, D-MPS-1 and D-MPS-8 MEASURED (probe `crates/mork-pathmap-probe`, results only; D-MPS-8 is the native arm over a versioned relationship table, §7b, not admitted). D-MPS-2..7 PROPOSAL / BLOCKED as listed in §8. No production executor, no CE64 change, no Lance storage, no upstream file edited by this work. **Ownership rule:** our types stay ours, upstream types stay upstream's, and the one replaceable boundary sits where they meet: `crates/mork-pathmap-probe/src/boundary.rs` plus the `[dependencies]` / `[patch]` block of the probe's `Cargo.toml`. @@ -127,6 +127,8 @@ What it shows, and what it does not: The native arm becomes testable only when a co-addressed producer for this input shape exists. **BLOCKED.** + > **2026-10-19, §7b:** a co-addressed producer for this input shape now exists as a probe (`src/join.rs`), and `Backend::Native` answers through it. The arm is measured, not admitted. + ## 7a. Versioning: what `RelationView.generation` stands for (2026-10-19) `RelationView.generation: u64` is a probe-local stand-in. It is not a design. Inside the probe it is enough: the `Stale { bound, current }` refusal fires on any change. Outside the probe a bare `u64` fails in two ways: @@ -142,6 +144,32 @@ D-RMAP-1A §7 (#1619) replaces the stand-in with a binding witness built from th The Schnellverschluss does not touch the witness. Switching the backend between MORK, PathMap and native changes how the join runs; the witness describes which data was bound. When the witness lands, it replaces `generation` in this crate, and the backend switch stays as it is. Today no `DurableRef` type exists in `crates/`, and CPG-00 has not run. +## 7b. Native arm: a versioned relationship table puts R and S at one coordinate (2026-10-19) + +> **Status: MEASURED, probe only; not admitted.** D-RMAP-1 (§ on BIND) says a pre-fold global-to-local binding is a candidate membrane that needs its own review. Code: `src/join.rs`; tests `tests/join.rs` (8) and the native arms added to `tests/differential.rs`; cost `examples/join_cost.rs`. Nothing is wired into Quack or R2IL. + +**Layout.** `JoinTable::prepare` builds one domain, the sorted distinct values of the shared variable `y` over both datasets, and lays each relation out by `y` slot. Each side gets a presence plane, offsets, and the other column's values. That is the one global-to-local translation (binary search into the domain), and it is timed inside `prepare`. Each side carries its own dataset's generation, so R and S can come from different sources and be resealed independently: `replace_side` re-prepares one side against the same domain, and refuses `OutsideDomain` if the new rows bring a `y` the domain lacks. + +**Evaluation.** Leapfrog triejoin with `y` first intersects R's `y` level with S's `y` level, then enumerates `x` and `z` independently, since they share no variable. Here the intersection is one mask-risc `And` over the two presence planes (`execute_into`, `Terminal::Keep`). Enumeration then reads both sides at each surviving slot. Every read is at the coordinate the mask names, in every operand; the `x` and `z` values are emitted and never used as addresses. That is the `Thought(t)[T_a[i] : T_b[i]]` shape of H14c/H14d, not a gather. For this query (one shared variable) the leapfrog join reduces to `And` plus enumeration. A star of k relations on `y` is an `And` chain. + +**What the plane decides.** Disable runs showed that the presence-plane intersection is an *answer* only when the joint plane itself is read, as the existence mask (which `y` every relation has). For enumeration and counts it only prunes, because a slot where one side is empty emits nothing. Seven guards were each disabled, and each turned its test red: per-side generation, domain refusal, filter shape, filter in the program, `And` (vs `Or`), presence only for non-empty slots, and every side's plane in a star. + +**Measured** (release, medians of 5, `cargo run --release --example join_cost`; MORK product-zipper build, leapfrog build within about 10%): + +| Fixture | prepare/build: native · PathMap · MORK (ms) | set answers into a map: native · PathMap · MORK (ms) | native: stream answers · bag count · joint plane (µs) | +| --- | --- | --- | --- | +| cost.rs, n = 10 000 | 0.61 · 1.86 · 4.26 | 1.55 · 2.44 · 27.8 | 175 · 11.9 · 0.35 | +| cost.rs, n = 50 000 | 2.75 · 14.65 · 22.87 | 7.39 · 26.06 · 144.4 | 332 · 35.5 · 0.15 | +| heterogeneous (ids over `u32`, 50 % `y` overlap, S fan-out 3), \|R\| 50 000, \|S\| 75 000 | 5.78 · 24.27 · 60.46 | 22.80 · 44.72 · 461.0 | 251 · 16.2 · 0.31 | + +- **The output dominates, not the join.** Native set answers into a `BTreeMap` cost 7.4 ms at n = 50 000; streaming the same answers costs 0.33 ms. The PathMap and MORK columns include the same kind of map insertion (MORK also its text decode, §6), so the fair comparison is the middle column. +- **A join amortizes into a fold when the consumer does not need tuples.** Over 1 024 queries at n = 50 000 (arithmetic, prepare + N × query): bag count 39 ms on native, against 7.6 s for native set answers and 26.7 s for PathMap answers. These answer different questions (count vs tuples), so this is a statement about demand, not a like-for-like speed-up. +- **Bags.** The native table keeps multiplicities and answers `Semantics::Bag` exactly. Both upstream indexes are sets and refuse (`SetOnly`). +- **Heterogeneous reseal.** Re-preparing one side costs 0.21 ms vs 2.47 ms for the whole table (n = 50 000); 1.17 ms vs 4.10 ms on the heterogeneous fixture. +- **Thoughts.** 64 thoughts over one shared table (403 KB at n = 50 000), each restricting `y` to its own eighth with a filter plane (1 568 B per thought), take 5.2 µs per thought for a bag count. The table is not copied per thought. + +**What this does not cover.** Cyclic bodies: the triangle `R(a,b), S(b,c), T(a,c)` needs two variables co-addressed at once, which a one-variable domain cannot give. The 256 × 256 tile of D-RMAP-1A §8 does it for a small focus only, and nothing here covers large domains. Chain joins: stepping from `y` results to `z` slots over a large domain would need a scatter, which stays forbidden. Enumeration and the count's `Σ_y |R_y| · |S_y|` are plain loops; only the intersection runs on mask-risc, which delegates to `ndarray::simd`. + ## 8. Deliverables | D-id | Item | Status | Gate / evidence | @@ -154,7 +182,8 @@ The Schnellverschluss does not touch the witness. Switching the backend between | D-MPS-5 | Upstream (MORK): `UnifyJoin` / `query_multi_leapfrog` generic over a cursor provider + reindex hook | PROPOSAL | MORK's own differential corpus unchanged on `PathMap<()>` | | D-MPS-6 | Lance-owned PathMap-trait cursor over a sealed `RelationOrder` view | BLOCKED: `RelationOrder` is not built (D-RMAP PROPOSED) | must show no copy, or state the copy it needs | | D-MPS-7 | Licence: no MORK source moved into or redistributed from lance-graph until MORK declares a licence (PR #148 unmerged) | OPEN | the probe fetches MORK and vendors nothing | -| — | Native lance-graph arm for this join | BLOCKED by the no-gather ruling | a co-addressed producer for the input shape | +| — | Native lance-graph arm for this join | ~~BLOCKED by the no-gather ruling~~ MEASURED as a probe (§7b); not admitted until the D-RMAP-1 BIND membrane review | a co-addressed producer for the input shape (`src/join.rs`); 8 + 2 tests vs the oracle; 7 guards disable-verified | +| D-MPS-8 | Versioned relationship table: one shared-variable domain, per-side generations, side reseal, filter planes per thought, star on one variable | MEASURED, probe | §7b; cyclic bodies and chain joins over large domains open | ## 9. Upstream contribution proposal (short) diff --git a/crates/mork-pathmap-probe/examples/join_cost.rs b/crates/mork-pathmap-probe/examples/join_cost.rs new file mode 100644 index 000000000..b51426896 --- /dev/null +++ b/crates/mork-pathmap-probe/examples/join_cost.rs @@ -0,0 +1,219 @@ +//! Cost of `Q(x, y, z) :- R(x, y), S(y, z)` over a shared, versioned +//! relationship table vs the PathMap and MORK indexes. +//! +//! Run: `cd crates/mork-pathmap-probe && cargo run --release --example join_cost` +//! (add `--features leapfrog` for MORK's leapfrog join). One machine, warm +//! caches, medians of `REPS` runs. Every answer is checked against the +//! nested-loop oracle before anything is timed. + +use std::time::{Duration, Instant}; + +use mork_pathmap_probe::join::{JoinExec, JoinTable, S_KEY, filter_plane}; +use mork_pathmap_probe::{ + Backend, JoinInput, RelationView, Semantics, bind, boundary::mork_join, reference, +}; + +const REPS: usize = 5; + +fn median(mut v: Vec) -> Duration { + v.sort(); + v[v.len() / 2] +} + +fn time(mut f: impl FnMut() -> T) -> (Duration, T) { + let mut ts = Vec::with_capacity(REPS); + let mut last = None; + for _ in 0..REPS { + let t = Instant::now(); + let v = f(); + ts.push(t.elapsed()); + last = Some(v); + } + (median(ts), last.unwrap()) +} + +fn us(d: Duration) -> f64 { + d.as_secs_f64() * 1e6 +} + +/// The `cost.rs` fixture: y in a small domain, one z per y. +fn small_domain(n: usize) -> (Vec<[u32; 2]>, Vec<[u32; 2]>) { + let domain = (n as u32 / 4).max(1); + let mut s = 7u32; + let r = (0..n as u32) + .map(|i| { + s = s.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + [i, s % domain] + }) + .collect(); + let s = (0..domain).map(|y| [y, y + 1_000_000]).collect(); + (r, s) +} + +/// Two datasets with ids spread over `u32`: half of each side's y values are +/// shared, S fans out 3 z per row. +fn heterogeneous(n: usize) -> (Vec<[u32; 2]>, Vec<[u32; 2]>) { + let ny = (n / 4) as u32; + let ys: Vec = (0..ny + ny / 2) + .map(|i| i.wrapping_mul(2_654_435_761).wrapping_add(17)) + .collect(); + let (ry, sy) = (&ys[..ny as usize], &ys[(ny / 2) as usize..]); + let mut st = 99u64; + let mut next = || { + st = st.wrapping_mul(6_364_136_223_846_793_005).wrapping_add(1); + (st >> 33) as u32 + }; + let r = (0..n) + .map(|_| [next(), ry[next() as usize % ry.len()]]) + .collect(); + let s = (0..n / 2) + .flat_map(|_| { + let y = sy[next() as usize % sy.len()]; + let z = next(); + (0..3u32).map(move |k| [y, z.wrapping_add(k)]) + }) + .collect(); + (r, s) +} + +fn section(label: &str, r: &[[u32; 2]], s: &[[u32; 2]]) { + let inp = JoinInput { + r: RelationView { + rows: r, + generation: 1, + }, + s: RelationView { + rows: s, + generation: 1, + }, + }; + let want_set = reference::answers(&inp, Semantics::Set); + let want_bag = reference::answers(&inp, Semantics::Bag); + println!( + "\n## {label}: |R| {} |S| {}, {} set answers, {} bag answers", + r.len(), + s.len(), + want_set.rows.len(), + want_bag.total() + ); + + // Upstream arms: build once, query once. + let mut rows = Vec::new(); + for backend in [Backend::PathMapZipper, Backend::Mork] { + let (build, mut idx) = time(|| bind(backend, &inp, Semantics::Set).unwrap()); + let (query, got) = time(|| idx.answers(&inp).unwrap()); + assert_eq!(got, want_set, "{backend:?}"); + rows.push((format!("{backend:?}"), build, query)); + } + + // Native: prepare, then answer. + let (prep, table) = time(|| JoinTable::prepare(inp.r, inp.s)); + let mut exec = JoinExec::new(&table); + let (q_set, got) = time(|| exec.answers(&table, (1, 1), None, Semantics::Set).unwrap()); + assert_eq!(got, want_set); + let (q_bag, got) = time(|| exec.answers(&table, (1, 1), None, Semantics::Bag).unwrap()); + assert_eq!(got, want_bag); + let (q_count, n) = time(|| exec.bag_count(&table, &[1, 1], None).unwrap()); + assert_eq!(n, want_bag.total()); + let (q_joint, _) = time(|| exec.joint(&table, None).unwrap().len()); + let (q_stream, streamed) = time(|| { + let mut k = 0u64; + exec.for_each_answer(&table, (1, 1), |row| { + k = k.wrapping_add(u64::from(row[0] ^ row[2])) + 1 + }) + .unwrap(); + k + }); + let _ = streamed; + rows.push(("Native set".into(), prep, q_set)); + + println!("arm, build_us, query_us (set answers into a BTreeMap)"); + for (name, b, q) in &rows { + println!("{name}, {:.1}, {:.1}", us(*b), us(*q)); + } + println!( + "native extras: bag answers {:.1} us | answers streamed, no map {:.1} us | bag count (no rows) {:.2} us | joint plane alone (mask-risc And+Keep) {:.2} us", + us(q_bag), + us(q_stream), + us(q_count), + us(q_joint) + ); + println!( + "native table: {} B (domain {} slots, {} B; plane {} B per side)", + table.bytes(), + table.domain.len(), + table.domain.bytes(), + table.words() * 8 + ); + + // Amortization, ARITHMETIC from the medians above: build + N * query. + println!("N, pathmap_ms, mork_ms, native_set_ms, native_count_ms [arithmetic]"); + for n in [1u32, 8, 64, 1024] { + let tot = |b: Duration, q: Duration| (us(b) + f64::from(n) * us(q)) / 1e3; + println!( + "{n}, {:.2}, {:.2}, {:.2}, {:.3}", + tot(rows[0].1, rows[0].2), + tot(rows[1].1, rows[1].2), + tot(prep, q_set), + tot(prep, q_count) + ); + } + + // 64 thoughts, each restricting y to its own slice (1 in 8), over the one + // shared table: measured, not arithmetic. + let filters: Vec> = (0..64u32) + .map(|t| filter_plane(&table, |y| y.wrapping_mul(2_246_822_519) >> 29 == t % 8)) + .collect(); + let (thoughts, total) = time(|| { + filters + .iter() + .map(|f| exec.bag_count(&table, &[1, 1], Some(f)).unwrap()) + .sum::() + }); + let mut check = 0u64; + for t in 0..64u32 { + check += want_bag + .rows + .iter() + .filter(|(row, _)| row[1].wrapping_mul(2_246_822_519) >> 29 == t % 8) + .map(|(_, &c)| u64::from(c)) + .sum::(); + } + assert_eq!(total, check); + println!( + "64 thoughts over one table, each a 1/8 y filter, bag count: {:.1} us total ({:.2} us per thought; filter plane {} B per thought)", + us(thoughts), + us(thoughts) / 64.0, + table.words() * 8 + ); + + // One dataset resealed: re-prepare S only vs the whole table. + let s2: Vec<[u32; 2]> = s.iter().step_by(2).copied().collect(); + let v2 = RelationView { + rows: &s2, + generation: 2, + }; + let (side, _) = time(|| { + let mut t = table.clone(); + t.replace_side(1, v2, S_KEY).unwrap(); + t + }); + let (full, _) = time(|| JoinTable::prepare(inp.r, v2)); + let (clone, _) = time(|| table.clone()); + println!( + "S resealed: replace S side {:.1} us (includes a {:.1} us table clone) | full prepare {:.1} us", + us(side), + us(clone), + us(full) + ); +} + +fn main() { + println!("mork join: {} | release, medians of {REPS}", mork_join()); + for n in [1_000usize, 10_000, 50_000] { + let (r, s) = small_domain(n); + section(&format!("cost.rs fixture, n = {n}"), &r, &s); + } + let (r, s) = heterogeneous(50_000); + section("heterogeneous, ids over u32, S fan-out 3", &r, &s); +} diff --git a/crates/mork-pathmap-probe/src/boundary.rs b/crates/mork-pathmap-probe/src/boundary.rs index 369889ef9..efd88fff0 100644 --- a/crates/mork-pathmap-probe/src/boundary.rs +++ b/crates/mork-pathmap-probe/src/boundary.rs @@ -17,6 +17,7 @@ use mork::space::Space; use pathmap::PathMap; use pathmap::zipper::{ZipperIteration, ZipperPath, ZipperValues}; +use crate::join::{JoinExec, JoinTable}; use crate::{Answers, JoinInput, Semantics}; /// A physical implementation of the fixed query. @@ -28,7 +29,10 @@ pub enum Backend { Mork, /// PathMap byte trie, read through read-only zippers. PathMapZipper, - /// lance-graph's own RelAlg / Quack / mask-risc lowering. + /// The versioned relationship table in [`crate::join`]: both inputs laid + /// out by the shared variable's slot, intersected by one mask-risc `And`. + /// A PROBE of the co-addressed producer §7 of the plan asks for, not an + /// admitted production path. Native, } @@ -39,10 +43,11 @@ pub enum Refusal { /// report multiplicities. Asking it for [`Semantics::Bag`] is refused rather /// than answered with every count collapsed to 1. SetOnly(Backend), - /// No admitted native join exists. The join reads `S` at positions chosen by - /// `R`'s values, the shape the 2026-10-16 no-gather ruling forbids; the - /// admitted form is a co-addressed layout (D-HOLO H14c/H14d), which these - /// inputs are not in. + /// No admitted native join exists for this input shape. Reading `S` at + /// positions chosen by `R`'s values is the shape the 2026-10-16 no-gather + /// ruling forbids. [`Backend::Native`] answers only after + /// [`crate::join::JoinTable::prepare`] has laid both inputs out at one + /// coordinate; this variant stays for shapes that table does not cover. NoAdmittedNativeJoin, /// The index was built from other generations than the caller's current /// views. @@ -73,7 +78,8 @@ type LeapfrogCursor<'a> = /// Answers what a backend can do before anything is built. pub fn capability(backend: Backend, semantics: Semantics) -> Result<(), Refusal> { match (backend, semantics) { - (Backend::Native, _) => Err(Refusal::NoAdmittedNativeJoin), + // Native keeps multiplicities, so it admits both semantics. + (Backend::Native, _) => Ok(()), (_, Semantics::Bag) => Err(Refusal::SetOnly(backend)), (_, Semantics::Set) => Ok(()), } @@ -82,6 +88,7 @@ pub fn capability(backend: Backend, semantics: Semantics) -> Result<(), Refusal> enum Index { Mork(Box), Path(PathMap<()>), + Native(Box<(JoinTable, JoinExec)>, Semantics), } /// An upstream index built from one pair of generations. @@ -107,7 +114,11 @@ pub fn bind( let index = match backend { Backend::Mork => Index::Mork(Box::new(mork_space(input)?)), Backend::PathMapZipper => Index::Path(path_trie(input)), - Backend::Native => return Err(Refusal::NoAdmittedNativeJoin), + Backend::Native => { + let table = JoinTable::prepare(input.r, input.s); + let exec = JoinExec::new(&table); + Index::Native(Box::new((table, exec)), semantics) + } }; Ok(BoundIndex { backend, @@ -136,6 +147,11 @@ impl BoundIndex { match &mut self.index { Index::Mork(space) => mork_answers(space), Index::Path(trie) => Ok(path_answers(trie)), + Index::Native(te, semantics) => { + let (table, exec) = &mut **te; + exec.answers(table, now, None, *semantics) + .map_err(|e| Refusal::Upstream(format!("{e:?}"))) + } } } } diff --git a/crates/mork-pathmap-probe/src/join.rs b/crates/mork-pathmap-probe/src/join.rs new file mode 100644 index 000000000..9ce202f7a --- /dev/null +++ b/crates/mork-pathmap-probe/src/join.rs @@ -0,0 +1,407 @@ +//! A versioned relationship table that puts both inputs of +//! `Q(x, y, z) :- R(x, y), S(y, z)` at one coordinate, so the join runs as a +//! fold. +//! +//! STATUS: PROBE. This is a candidate for the "co-addressed producer" the +//! Schnellverschluss plan (§7) names as the precondition of a native arm. It is +//! not admitted: D-RMAP-1 says a pre-fold global-to-local binding is a candidate +//! membrane that needs its own review, and nothing here is wired into Quack, +//! R2IL or the production mask-risc path. +//! +//! # Layout +//! +//! - [`JoinDomain`]: the sorted distinct values of the shared variable `y` +//! across both datasets. A value's position is its **slot**. This is the one +//! global-to-local translation, done once in [`JoinTable::prepare`] by binary +//! search and timed there. +//! - [`Side`]: one input relation laid out by `y` slot: a presence plane (bit +//! `y` set when the relation has a row with that `y`), `off[y]..off[y + 1]`, +//! and the other column's values in that range. Each side is prepared from +//! its own dataset and carries that dataset's generation, so R and S can come +//! from different sources and be resealed independently. +//! +//! # Evaluation, and why it is not a gather +//! +//! Leapfrog triejoin with `y` first intersects R's `y` level with S's `y` level, +//! then enumerates `x` and `z` independently (they share no variable). Here the +//! intersection is one mask-risc `And` over the two presence planes, which sit +//! over the same slot space. Enumeration then reads `off[y]` and the value range +//! of each side at the surviving slot `y`: every read is at the coordinate the +//! mask names, in every operand. The `x` and `z` values are emitted, never used +//! as addresses. No operand is read at a position taken from another operand's +//! values (2026-10-16 ruling). +//! +//! Multiplicities are kept, so this arm answers [`Semantics::Bag`] as well as +//! [`Semantics::Set`]; both upstream indexes are sets. + +use lance_graph_mask_risc::{ + Foreign, MaskOp, Operand, Out, Planes, Program, Scratch, Terminal, execute_into, words_for, +}; + +use crate::{Answers, RelationView, Semantics}; + +/// Why the table did not answer. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum JoinRefusal { + /// A side was prepared from another generation than the caller holds now. + Stale { + side: usize, + bound: u64, + current: u64, + }, + /// A re-prepared side has a `y` the domain does not contain. The domain is + /// the shared coordinate space; changing it re-addresses every side, so the + /// whole table must be prepared again. + OutsideDomain { side: usize, y: u32 }, + /// The filter plane is not over this table's slot space. + FilterShape { words: usize, expected: usize }, +} + +/// The shared coordinate space: sorted distinct `y` values. +#[derive(Clone, Debug)] +pub struct JoinDomain { + ids: Vec, +} + +impl JoinDomain { + /// The union of every side's `y` values. + pub fn build(ys: impl IntoIterator) -> Self { + let mut ids: Vec = ys.into_iter().collect(); + ids.sort_unstable(); + ids.dedup(); + JoinDomain { ids } + } + + pub fn len(&self) -> usize { + self.ids.len() + } + + pub fn is_empty(&self) -> bool { + self.ids.is_empty() + } + + /// Global `y` to slot. Preparation only. + pub fn slot_of(&self, y: u32) -> Option { + self.ids.binary_search(&y).ok() + } + + /// Slot to global `y`, for emitting answers. + pub fn id(&self, slot: usize) -> u32 { + self.ids[slot] + } + + pub fn bytes(&self) -> usize { + self.ids.len() * 4 + } +} + +/// One relation laid out by `y` slot. +#[derive(Clone, Debug)] +pub struct Side { + pub generation: u64, + present: Vec, + off: Vec, + vals: Vec, +} + +impl Side { + /// Lays `rows` out by the slot of column `key`; the other column is the + /// value. A counting sort over slots, so the value order within one slot + /// is the input order. + pub fn prepare( + domain: &JoinDomain, + rows: &[[u32; 2]], + key: usize, + generation: u64, + side: usize, + ) -> Result { + let n = domain.len(); + let mut slots = Vec::with_capacity(rows.len()); + let mut off = vec![0u32; n + 1]; + for row in rows { + let y = row[key]; + let s = domain + .slot_of(y) + .ok_or(JoinRefusal::OutsideDomain { side, y })?; + slots.push(s as u32); + off[s + 1] += 1; + } + for i in 0..n { + off[i + 1] += off[i]; + } + let mut fill = off.clone(); + let mut vals = vec![0u32; rows.len()]; + for (row, &s) in rows.iter().zip(&slots) { + let at = &mut fill[s as usize]; + vals[*at as usize] = row[1 - key]; + *at += 1; + } + let mut present = vec![0u64; words_for(n)]; + for s in 0..n { + if off[s + 1] > off[s] { + present[s >> 6] |= 1 << (s & 63); + } + } + Ok(Side { + generation, + present, + off, + vals, + }) + } + + /// The values at slot `y`. + pub fn at(&self, y: usize) -> &[u32] { + &self.vals[self.off[y] as usize..self.off[y + 1] as usize] + } + + pub fn present(&self) -> &[u64] { + &self.present + } + + pub fn bytes(&self) -> usize { + (self.present.len() * 8) + (self.off.len() * 4) + (self.vals.len() * 4) + } +} + +/// The versioned relationship table: one domain, one side per relation, each +/// side tied to its own dataset generation. +#[derive(Clone, Debug)] +pub struct JoinTable { + pub domain: JoinDomain, + pub sides: Vec, +} + +/// Where the shared variable sits in each input row: `R(x, y)` keys column 1, +/// `S(y, z)` keys column 0. +pub const R_KEY: usize = 1; +pub const S_KEY: usize = 0; + +impl JoinTable { + /// Prepares the table for `R(x, y), S(y, z)`. + pub fn prepare(r: RelationView<'_>, s: RelationView<'_>) -> JoinTable { + Self::prepare_star(&[(r, R_KEY), (s, S_KEY)]) + } + + /// Prepares a star of relations that all share `y`: every relation is laid + /// out over the same domain. `key` names the column holding `y`. + pub fn prepare_star(inputs: &[(RelationView<'_>, usize)]) -> JoinTable { + let domain = JoinDomain::build( + inputs + .iter() + .flat_map(|(v, key)| v.rows.iter().map(move |r| r[*key])), + ); + let sides = inputs + .iter() + .enumerate() + .map(|(i, (v, key))| { + Side::prepare(&domain, v.rows, *key, v.generation, i).expect("domain is the union") + }) + .collect(); + JoinTable { domain, sides } + } + + /// Re-prepares one side from a resealed dataset, keeping the domain and the + /// other sides. Refuses if the new rows leave the domain. + pub fn replace_side( + &mut self, + side: usize, + view: RelationView<'_>, + key: usize, + ) -> Result<(), JoinRefusal> { + self.sides[side] = Side::prepare(&self.domain, view.rows, key, view.generation, side)?; + Ok(()) + } + + /// Refuses unless every side's generation is the one the caller holds. + pub fn check(&self, current: &[u64]) -> Result<(), JoinRefusal> { + for (i, (side, &now)) in self.sides.iter().zip(current).enumerate() { + if side.generation != now { + return Err(JoinRefusal::Stale { + side: i, + bound: side.generation, + current: now, + }); + } + } + Ok(()) + } + + pub fn bytes(&self) -> usize { + self.domain.bytes() + self.sides.iter().map(Side::bytes).sum::() + } + + pub fn words(&self) -> usize { + words_for(self.domain.len()) + } +} + +/// Runs the presence-plane intersection through mask-risc. Owns its output +/// plane and scratch, so repeated queries allocate nothing in the fold. +pub struct JoinExec { + out: Vec, + scratch: Scratch<'static>, +} + +impl JoinExec { + pub fn new(table: &JoinTable) -> Self { + let words = table.words(); + JoinExec { + out: vec![0; words], + scratch: Scratch::new(words, 1), + } + } + + /// `joint = present_0 ∧ … ∧ present_k [∧ filter]`: the slots where every + /// relation has a row (and the thought admits the slot). + pub fn joint( + &mut self, + table: &JoinTable, + filter: Option<&[u64]>, + ) -> Result<&[u64], JoinRefusal> { + let words = table.words(); + if let Some(f) = filter + && f.len() != words + { + return Err(JoinRefusal::FilterShape { + words: f.len(), + expected: words, + }); + } + assert!(table.sides.len() >= 2, "a join needs two relations"); + let mut masks: Vec<&[u64]> = table.sides.iter().map(|s| s.present()).collect(); + if let Some(f) = filter { + masks.push(f); + } + let mut ops = Vec::with_capacity(masks.len()); + let mut acc = Operand::Plane(0); + for i in 1..masks.len() { + ops.push(MaskOp::And { + a: acc, + b: Operand::Plane(i as u16), + dst: 0, + }); + acc = Operand::Scratch(0); + } + let prog = Program::new( + ops, + Terminal::Keep { + mask: Operand::Scratch(0), + }, + ); + let pl = Planes { + n_rows: table.domain.len(), + masks: &masks, + lanes: &[], + }; + execute_into( + &prog, + &pl, + &Foreign::NONE, + &mut self.scratch, + Out::Mask(&mut self.out), + ) + .expect("join program is valid"); + Ok(&self.out) + } + + /// The answers of `R(x, y), S(y, z)`, enumerated at the joint slots. + pub fn answers( + &mut self, + table: &JoinTable, + current: (u64, u64), + filter: Option<&[u64]>, + semantics: Semantics, + ) -> Result { + table.check(&[current.0, current.1])?; + let n = table.domain.len(); + let joint = self.joint(table, filter)?; + let (r, s) = (&table.sides[0], &table.sides[1]); + let mut out = Answers::default(); + for_each_set(joint, n, |slot| { + let y = table.domain.id(slot); + for &x in r.at(slot) { + for &z in s.at(slot) { + *out.rows.entry([x, y, z]).or_insert(0) += 1; + } + } + }); + Ok(match semantics { + Semantics::Bag => out, + Semantics::Set => out.to_set(), + }) + } + + /// Every bag answer `[x, y, z]`, handed to `f` in slot order, without + /// building a map: the join's own cost, separated from the output's. + pub fn for_each_answer( + &mut self, + table: &JoinTable, + current: (u64, u64), + mut f: impl FnMut([u32; 3]), + ) -> Result<(), JoinRefusal> { + table.check(&[current.0, current.1])?; + let n = table.domain.len(); + let joint = self.joint(table, None)?; + let (r, s) = (&table.sides[0], &table.sides[1]); + for_each_set(joint, n, |slot| { + let y = table.domain.id(slot); + for &x in r.at(slot) { + for &z in s.at(slot) { + f([x, y, z]); + } + } + }); + Ok(()) + } + + /// The number of bag answers of the star, `Σ_y Π_i |side_i at y|`, without + /// enumerating a row. + pub fn bag_count( + &mut self, + table: &JoinTable, + current: &[u64], + filter: Option<&[u64]>, + ) -> Result { + table.check(current)?; + let n = table.domain.len(); + let joint = self.joint(table, filter)?; + let mut total = 0u64; + for_each_set(joint, n, |slot| { + total += table + .sides + .iter() + .map(|s| s.at(slot).len() as u64) + .product::(); + }); + Ok(total) + } +} + +/// Calls `f` for every set bit below `n`, in slot order. +pub fn for_each_set(words: &[u64], n: usize, mut f: impl FnMut(usize)) { + for (wi, &w) in words.iter().enumerate() { + let mut w = w; + while w != 0 { + let b = w.trailing_zeros() as usize; + let slot = wi * 64 + b; + if slot < n { + f(slot); + } + w &= w - 1; + } + } +} + +/// A filter plane over the table's slots from a predicate on the global `y`. +/// Built per thought, from the domain, never from the sides. +pub fn filter_plane(table: &JoinTable, mut keep: impl FnMut(u32) -> bool) -> Vec { + let n = table.domain.len(); + let mut p = vec![0u64; words_for(n)]; + for slot in 0..n { + if keep(table.domain.id(slot)) { + p[slot >> 6] |= 1 << (slot & 63); + } + } + p +} diff --git a/crates/mork-pathmap-probe/src/lib.rs b/crates/mork-pathmap-probe/src/lib.rs index 81d4de382..14e593970 100644 --- a/crates/mork-pathmap-probe/src/lib.rs +++ b/crates/mork-pathmap-probe/src/lib.rs @@ -1,7 +1,8 @@ //! MORK / PathMap differential probe behind one Lance-owned boundary. //! //! STATUS: PROBE (results only). Not a production executor, not wired into -//! Quack, R2IL or mask-risc, and not a fact store. Plan: +//! Quack or R2IL, and not a fact store. The native arm ([`join`]) runs its +//! intersection as a mask-risc program, as a probe only. Plan: //! `.claude/plans/mork-pathmap-schnellverschluss-v1.md`. //! //! The query is fixed: `Q(x, y, z) :- R(x, y), S(y, z)` — two relations that @@ -22,6 +23,7 @@ use std::collections::BTreeMap; pub mod boundary; +pub mod join; pub mod reference; pub mod rmap; diff --git a/crates/mork-pathmap-probe/tests/differential.rs b/crates/mork-pathmap-probe/tests/differential.rs index c9421f0de..f66104faa 100644 --- a/crates/mork-pathmap-probe/tests/differential.rs +++ b/crates/mork-pathmap-probe/tests/differential.rs @@ -18,7 +18,8 @@ fn input<'a>(r: &'a [[u32; 2]], s: &'a [[u32; 2]], gen_r: u64, gen_s: u64) -> Jo } } -const ARMS: [Backend; 2] = [Backend::Mork, Backend::PathMapZipper]; +const ARMS: [Backend; 3] = [Backend::Mork, Backend::PathMapZipper, Backend::Native]; +const SET_ONLY: [Backend; 2] = [Backend::Mork, Backend::PathMapZipper]; fn check_set(r: &[[u32; 2]], s: &[[u32; 2]]) -> usize { let inp = input(r, s, 1, 1); @@ -81,9 +82,15 @@ fn duplicates_are_refused_under_bag_semantics() { reference::answers(&inp, Semantics::Bag).rows[&[1, 10, 5]], 4 ); - for arm in ARMS { + for arm in SET_ONLY { assert!(matches!(bind(arm, &inp, Semantics::Bag), Err(Refusal::SetOnly(a)) if a == arm)); } + // The native table keeps multiplicities and answers the bag exactly. + let mut native = bind(Backend::Native, &inp, Semantics::Bag).unwrap(); + assert_eq!( + native.answers(&inp).unwrap(), + reference::answers(&inp, Semantics::Bag) + ); // Under set semantics they agree with the oracle's set. assert_eq!(check_set(&r, &s), 1); } @@ -130,16 +137,20 @@ fn a_reused_generation_number_is_not_detected() { assert!(!got.rows.contains_key(&[2, 10, 5])); } +/// The native arm answers only through the co-addressed table, for both +/// semantics, and its bag answers carry the oracle's multiplicities. #[test] -fn native_join_is_refused_not_emulated() { - assert_eq!( - capability(Backend::Native, Semantics::Set), - Err(Refusal::NoAdmittedNativeJoin) +fn native_join_answers_bags_from_the_coaddressed_table() { + assert_eq!(capability(Backend::Native, Semantics::Set), Ok(())); + assert_eq!(capability(Backend::Native, Semantics::Bag), Ok(())); + let r = [[1, 10], [1, 10], [2, 10], [3, 11], [4, 99]]; + let s = [[10, 5], [10, 6], [10, 5], [11, 7], [12, 8]]; + let inp = input(&r, &s, 1, 1); + let want = reference::answers(&inp, Semantics::Bag); + assert!( + want.rows.values().any(|&c| c > 1), + "fixture has multiplicities" ); - let r = [[1, 10]]; - let s = [[10, 5]]; - assert!(matches!( - bind(Backend::Native, &input(&r, &s, 1, 1), Semantics::Set), - Err(Refusal::NoAdmittedNativeJoin) - )); + let mut idx = bind(Backend::Native, &inp, Semantics::Bag).unwrap(); + assert_eq!(idx.answers(&inp).unwrap(), want); } diff --git a/crates/mork-pathmap-probe/tests/join.rs b/crates/mork-pathmap-probe/tests/join.rs new file mode 100644 index 000000000..e07e656e2 --- /dev/null +++ b/crates/mork-pathmap-probe/tests/join.rs @@ -0,0 +1,213 @@ +//! The versioned relationship table (`join`) against nested-loop oracles. + +use mork_pathmap_probe::join::{ + JoinExec, JoinRefusal, JoinTable, R_KEY, S_KEY, filter_plane, for_each_set, +}; +use mork_pathmap_probe::{JoinInput, RelationView, Semantics, reference}; + +fn view(rows: &[[u32; 2]], generation: u64) -> RelationView<'_> { + RelationView { rows, generation } +} + +/// Deterministic rows with ids spread over the whole `u32` range: the two +/// datasets share a known fraction of `y` values and nothing else. +fn hetero(n: usize, seed: u64, ys: &[u32], fan: usize) -> Vec<[u32; 2]> { + let mut s = seed; + let mut next = || { + s = s.wrapping_mul(6_364_136_223_846_793_005).wrapping_add(1); + (s >> 33) as u32 + }; + (0..n) + .flat_map(|_| { + let y = ys[next() as usize % ys.len()]; + let a = next(); + (0..fan).map(move |k| [a.wrapping_add(k as u32), y]) + }) + .collect() +} + +fn swap(rows: &[[u32; 2]]) -> Vec<[u32; 2]> { + rows.iter().map(|&[a, b]| [b, a]).collect() +} + +fn ys(n: u32, seed: u32) -> Vec { + (0..n) + .map(|i| i.wrapping_mul(2_654_435_761).wrapping_add(seed)) + .collect() +} + +#[test] +fn heterogeneous_join_matches_the_oracle_in_both_semantics() { + let all = ys(64, 7); + // R uses the first 48 y values, S the last 48: 32 shared. + let r = hetero(400, 1, &all[..48], 1); + let s = swap(&hetero(300, 2, &all[16..], 2)); + let inp = JoinInput { + r: view(&r, 3), + s: view(&s, 11), + }; + let table = JoinTable::prepare(inp.r, inp.s); + let mut exec = JoinExec::new(&table); + for sem in [Semantics::Set, Semantics::Bag] { + let want = reference::answers(&inp, sem); + assert!(!want.rows.is_empty()); + assert_eq!(exec.answers(&table, (3, 11), None, sem).unwrap(), want); + } + // Not degenerate: some y are in one relation only, and are excluded. + assert!(table.domain.len() > 48); + let bag = reference::answers(&inp, Semantics::Bag).total(); + assert_eq!(exec.bag_count(&table, &[3, 11], None).unwrap(), bag); +} + +#[test] +fn a_star_of_three_relations_counts_like_the_oracle() { + let all = ys(40, 3); + let r = hetero(200, 4, &all[..30], 1); + let s = swap(&hetero(150, 5, &all[10..], 1)); + let t = swap(&hetero(120, 6, &all[5..35], 3)); + let table = JoinTable::prepare_star(&[ + (view(&r, 1), R_KEY), + (view(&s, 1), S_KEY), + (view(&t, 1), S_KEY), + ]); + let mut exec = JoinExec::new(&table); + let mut want = 0u64; + for &[_, y] in &r { + let a = s.iter().filter(|row| row[0] == y).count() as u64; + let b = t.iter().filter(|row| row[0] == y).count() as u64; + want += a * b; + } + assert!(want > 0); + assert_eq!(exec.bag_count(&table, &[1, 1, 1], None).unwrap(), want); + // The third relation narrows the joint: dropping it gives a different count. + let two = JoinTable::prepare_star(&[(view(&r, 1), R_KEY), (view(&s, 1), S_KEY)]); + let mut exec2 = JoinExec::new(&two); + assert_ne!(exec2.bag_count(&two, &[1, 1], None).unwrap(), want); +} + +#[test] +fn a_filter_plane_restricts_the_join_to_the_thoughts_slots() { + let all = ys(32, 9); + let r = hetero(200, 7, &all, 1); + let s = swap(&hetero(200, 8, &all, 1)); + let inp = JoinInput { + r: view(&r, 1), + s: view(&s, 1), + }; + let table = JoinTable::prepare(inp.r, inp.s); + let mut exec = JoinExec::new(&table); + let keep = |y: u32| y.is_multiple_of(3); + let filter = filter_plane(&table, keep); + let got = exec + .answers(&table, (1, 1), Some(&filter), Semantics::Bag) + .unwrap(); + let mut want = reference::answers(&inp, Semantics::Bag); + want.rows.retain(|row, _| keep(row[1])); + assert!(!want.rows.is_empty()); + assert_eq!(got, want); + // Silence twin: the filter removes something. + assert!(want.total() < reference::answers(&inp, Semantics::Bag).total()); + // A plane over another slot space is refused, not read past its end. + assert!(matches!( + exec.answers(&table, (1, 1), Some(&[0u64; 1000]), Semantics::Set), + Err(JoinRefusal::FilterShape { .. }) + )); +} + +#[test] +fn each_side_is_refused_against_its_own_generation() { + let r = [[1, 10], [2, 11]]; + let s = [[10, 5], [11, 6]]; + let table = JoinTable::prepare(view(&r, 4), view(&s, 9)); + let mut exec = JoinExec::new(&table); + assert!(exec.answers(&table, (4, 9), None, Semantics::Set).is_ok()); + assert_eq!( + exec.answers(&table, (5, 9), None, Semantics::Set), + Err(JoinRefusal::Stale { + side: 0, + bound: 4, + current: 5 + }) + ); + assert_eq!( + exec.answers(&table, (4, 8), None, Semantics::Set), + Err(JoinRefusal::Stale { + side: 1, + bound: 9, + current: 8 + }) + ); +} + +#[test] +fn replacing_one_side_equals_preparing_the_table_again() { + let all = ys(48, 1); + let r = hetero(300, 1, &all, 1); + let s1 = swap(&hetero(300, 2, &all, 2)); + let s2 = swap(&hetero(250, 3, &all[..40], 1)); + let mut table = JoinTable::prepare(view(&r, 1), view(&s1, 1)); + table.replace_side(1, view(&s2, 2), S_KEY).unwrap(); + let mut exec = JoinExec::new(&table); + let got = exec.answers(&table, (1, 2), None, Semantics::Bag).unwrap(); + let inp = JoinInput { + r: view(&r, 1), + s: view(&s2, 2), + }; + assert_eq!(got, reference::answers(&inp, Semantics::Bag)); + // The old S answer differs, so the replacement did something. + let old = JoinInput { + r: view(&r, 1), + s: view(&s1, 1), + }; + assert_ne!(got, reference::answers(&old, Semantics::Bag)); +} + +#[test] +fn a_side_leaving_the_domain_is_refused() { + let r = [[1, 10]]; + let s = [[10, 5]]; + let mut table = JoinTable::prepare(view(&r, 1), view(&s, 1)); + let s2 = [[10, 5], [77, 6]]; + assert_eq!( + table.replace_side(1, view(&s2, 2), S_KEY), + Err(JoinRefusal::OutsideDomain { side: 1, y: 77 }) + ); + // The table is unchanged: the old generation still answers. + let mut exec = JoinExec::new(&table); + assert!(exec.answers(&table, (1, 1), None, Semantics::Set).is_ok()); +} + +#[test] +fn the_joint_plane_has_exactly_the_shared_slots() { + let r = [[0, 1], [0, 3], [0, 5]]; + let s = [[3, 0], [4, 0], [5, 0]]; + let table = JoinTable::prepare(view(&r, 1), view(&s, 1)); + let mut exec = JoinExec::new(&table); + let mut shared = Vec::new(); + let n = table.domain.len(); + let joint = exec.joint(&table, None).unwrap().to_vec(); + for_each_set(&joint, n, |slot| shared.push(table.domain.id(slot))); + assert_eq!(shared, vec![3, 5]); +} + +/// The joint plane is itself an answer: the `y` values that every relation of +/// the star has (the existence half of the join). Enumeration and counts do not +/// need it to be exact, since an empty side emits nothing at its slot; this +/// test is what pins it. +#[test] +fn the_star_joint_is_the_existence_mask() { + let r = [[0, 1], [0, 2], [0, 3], [0, 4]]; + let s = [[2, 0], [3, 0], [4, 0], [5, 0]]; + let t = [[3, 0], [4, 0], [6, 0]]; + let table = JoinTable::prepare_star(&[ + (view(&r, 1), R_KEY), + (view(&s, 1), S_KEY), + (view(&t, 1), S_KEY), + ]); + let mut exec = JoinExec::new(&table); + let n = table.domain.len(); + let joint = exec.joint(&table, None).unwrap().to_vec(); + let mut shared = Vec::new(); + for_each_set(&joint, n, |slot| shared.push(table.domain.id(slot))); + assert_eq!(shared, vec![3, 4]); +}