From 67929546692ee604ceee27286001d27ea446f051 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 21 Oct 2026 10:37:35 +0000 Subject: [PATCH 01/14] mork-pathmap-probe: a relation is two lanes; a side is an index map; answers are folds - RelationView holds two lanes (arrow ScalarBuffer, shared by reference count) instead of [u32; 2] pairs. from_rows builds the lanes once from authored fixture rows. - Side keeps present + off and holds pos, the row positions per slot (the global-to-local index map), instead of a copy of the value column. at(slot) returns a Run read in place (lane[pos[k]]); counts come from the offsets. - JoinExec: answers() (BTreeMap rows) is gone. bag_count reads only offsets; matching(pattern) is a fold (a pinned y reads one slot); for_each_answer hands out a JoinRow read in place. The next slot's runs are prefetched through ndarray::simd while the current one is read. - reference is a counting oracle over the lanes; it builds no row. - MORK and PathMap keep their own representation; their row sets are checked against the folds in both directions. The native BoundIndex answers through native(), and answers() refuses with FoldOnly. - Examples report microseconds, not milliseconds. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01G22yT6htkcdyXsihxxXdrg --- crates/mork-pathmap-probe/Cargo.toml | 3 + crates/mork-pathmap-probe/examples/cost.rs | 33 +- .../mork-pathmap-probe/examples/join_cost.rs | 70 ++--- crates/mork-pathmap-probe/src/boundary.rs | 74 +++-- crates/mork-pathmap-probe/src/join.rs | 295 +++++++++++++----- crates/mork-pathmap-probe/src/lib.rs | 76 ++++- crates/mork-pathmap-probe/src/reference.rs | 38 ++- .../mork-pathmap-probe/tests/differential.rs | 115 +++++-- crates/mork-pathmap-probe/tests/join.rs | 164 +++++++--- 9 files changed, 594 insertions(+), 274 deletions(-) diff --git a/crates/mork-pathmap-probe/Cargo.toml b/crates/mork-pathmap-probe/Cargo.toml index 7498f61d3..ab1d1899c 100644 --- a/crates/mork-pathmap-probe/Cargo.toml +++ b/crates/mork-pathmap-probe/Cargo.toml @@ -38,6 +38,9 @@ lance-graph-mask-risc = { path = "../lance-graph-mask-risc" } # through the `ndarray::simd` facade. Same coordinate as mask-risc's, so the # graph keeps one ndarray package. ndarray = { git = "https://github.com/AdaWorldAPI/ndarray", branch = "master", default-features = false, features = ["std"] } +# A relation is two lanes; a lane is Arrow's reference-counted buffer, so a +# lane read from Lance is shared, never copied. Same arrow major as Lance 13. +arrow-buffer = "58" # MORK's workspace names PathMap as `path = "../PathMap/"`, outside its own # repository, so a git dependency on MORK cannot resolve it. This patch points diff --git a/crates/mork-pathmap-probe/examples/cost.rs b/crates/mork-pathmap-probe/examples/cost.rs index 6457e0f1d..031a5ac48 100644 --- a/crates/mork-pathmap-probe/examples/cost.rs +++ b/crates/mork-pathmap-probe/examples/cost.rs @@ -29,7 +29,7 @@ fn rows(n: usize, seed: u32, domain: u32) -> Vec<[u32; 2]> { fn main() { println!("mork join: {}", mork_join()); println!( - "backend,n_rows,answers,build_ms,build_ns_per_row,query_ms,shared_per_thought_ms_k8,shared_per_thought_ms_k64,rebuilt_per_thought_ms,shared_total_ms_k64,rebuilt_total_ms_k64" + "backend,n_rows,answers,build_us,build_ns_per_row,query_us,shared_per_thought_us_k8,shared_per_thought_us_k64,rebuilt_per_thought_us,shared_total_us_k64,rebuilt_total_us_k64" ); for &n in &[1_000usize, 10_000, 50_000] { let domain = (n as u32 / 4).max(1); @@ -37,16 +37,9 @@ fn main() { // S(y, z): one z per y, so |answers| ~ |R|. let s: Vec<[u32; 2]> = (0..domain).map(|y| [y, y + 1_000_000]).collect(); let inp = JoinInput { - r: RelationView { - rows: &r, - generation: 1, - }, - s: RelationView { - rows: &s, - generation: 1, - }, + r: RelationView::from_rows(&r, 1), + s: RelationView::from_rows(&s, 1), }; - let want = reference::answers(&inp, Semantics::Set); for backend in [Backend::PathMapZipper, Backend::Mork] { let t = Instant::now(); let mut idx = bind(backend, &inp, Semantics::Set).unwrap(); @@ -54,21 +47,27 @@ fn main() { let t = Instant::now(); let got = idx.answers(&inp).unwrap(); let query = t.elapsed(); - assert_eq!(got, want, "{backend:?} n={n}"); - let ms = |d: std::time::Duration| d.as_secs_f64() * 1e3; + // Every row the engine returns is a derivation the oracle counts. + for row in got.rows.keys() { + assert!( + reference::matching(&inp, row.map(Some)) > 0, + "{backend:?} n={n}: {row:?}" + ); + } + let us = |d: std::time::Duration| d.as_secs_f64() * 1e6; // One index built once and queried by k thoughts, per thought. - let shared_per_thought = |k: f64| ms(build) / k + ms(query); - let rebuilt_per_thought = ms(build) + ms(query); + let shared_per_thought = |k: f64| us(build) / k + us(query); + let rebuilt_per_thought = us(build) + us(query); println!( "{backend:?},{n},{},{:.3},{:.0},{:.3},{:.3},{:.3},{:.3},{:.3},{:.3}", got.rows.len(), - ms(build), + us(build), build.as_nanos() as f64 / (r.len() + s.len()) as f64, - ms(query), + us(query), shared_per_thought(8.0), shared_per_thought(64.0), rebuilt_per_thought, - ms(build) + 64.0 * ms(query), + us(build) + 64.0 * us(query), 64.0 * rebuilt_per_thought, ); } diff --git a/crates/mork-pathmap-probe/examples/join_cost.rs b/crates/mork-pathmap-probe/examples/join_cost.rs index b51426896..308b34949 100644 --- a/crates/mork-pathmap-probe/examples/join_cost.rs +++ b/crates/mork-pathmap-probe/examples/join_cost.rs @@ -78,23 +78,15 @@ fn heterogeneous(n: usize) -> (Vec<[u32; 2]>, Vec<[u32; 2]>) { fn section(label: &str, r: &[[u32; 2]], s: &[[u32; 2]]) { let inp = JoinInput { - r: RelationView { - rows: r, - generation: 1, - }, - s: RelationView { - rows: s, - generation: 1, - }, + r: RelationView::from_rows(r, 1), + s: RelationView::from_rows(s, 1), }; - let want_set = reference::answers(&inp, Semantics::Set); - let want_bag = reference::answers(&inp, Semantics::Bag); + let want_bag = reference::bag_count(&inp); println!( - "\n## {label}: |R| {} |S| {}, {} set answers, {} bag answers", + "\n## {label}: |R| {} |S| {}, {} bag answers", r.len(), s.len(), - want_set.rows.len(), - want_bag.total() + want_bag ); // Upstream arms: build once, query once. @@ -102,38 +94,38 @@ fn section(label: &str, r: &[[u32; 2]], s: &[[u32; 2]]) { for backend in [Backend::PathMapZipper, Backend::Mork] { let (build, mut idx) = time(|| bind(backend, &inp, Semantics::Set).unwrap()); let (query, got) = time(|| idx.answers(&inp).unwrap()); - assert_eq!(got, want_set, "{backend:?}"); + for row in got.rows.keys() { + assert!( + reference::matching(&inp, row.map(Some)) > 0, + "{backend:?}: {row:?}" + ); + } rows.push((format!("{backend:?}"), build, query)); } // Native: prepare, then answer. - let (prep, table) = time(|| JoinTable::prepare(inp.r, inp.s)); + let (prep, table) = time(|| JoinTable::prepare(&inp.r, &inp.s)); let mut exec = JoinExec::new(&table); - let (q_set, got) = time(|| exec.answers(&table, (1, 1), None, Semantics::Set).unwrap()); - assert_eq!(got, want_set); - let (q_bag, got) = time(|| exec.answers(&table, (1, 1), None, Semantics::Bag).unwrap()); - assert_eq!(got, want_bag); let (q_count, n) = time(|| exec.bag_count(&table, &[1, 1], None).unwrap()); - assert_eq!(n, want_bag.total()); + assert_eq!(n, want_bag); let (q_joint, _) = time(|| exec.joint(&table, None).unwrap().len()); let (q_stream, streamed) = time(|| { let mut k = 0u64; exec.for_each_answer(&table, (1, 1), |row| { - k = k.wrapping_add(u64::from(row[0] ^ row[2])) + 1 + k = k.wrapping_add(u64::from(row.x() ^ row.z())) + 1 }) .unwrap(); k }); let _ = streamed; - rows.push(("Native set".into(), prep, q_set)); + rows.push(("Native bag count".into(), prep, q_count)); - println!("arm, build_us, query_us (set answers into a BTreeMap)"); + println!("arm, build_us, query_us (upstream: set answers into a BTreeMap; native: a fold)"); for (name, b, q) in &rows { println!("{name}, {:.1}, {:.1}", us(*b), us(*q)); } println!( - "native extras: bag answers {:.1} us | answers streamed, no map {:.1} us | bag count (no rows) {:.2} us | joint plane alone (mask-risc And+Keep) {:.2} us", - us(q_bag), + "native extras: answers read in place, no map {:.1} us | bag count (no rows) {:.2} us | joint plane alone (mask-risc And+Keep) {:.2} us", us(q_stream), us(q_count), us(q_joint) @@ -147,14 +139,13 @@ fn section(label: &str, r: &[[u32; 2]], s: &[[u32; 2]]) { ); // Amortization, ARITHMETIC from the medians above: build + N * query. - println!("N, pathmap_ms, mork_ms, native_set_ms, native_count_ms [arithmetic]"); + println!("N, pathmap_us, mork_us, native_count_us [arithmetic]"); for n in [1u32, 8, 64, 1024] { - let tot = |b: Duration, q: Duration| (us(b) + f64::from(n) * us(q)) / 1e3; + let tot = |b: Duration, q: Duration| us(b) + f64::from(n) * us(q); println!( - "{n}, {:.2}, {:.2}, {:.2}, {:.3}", + "{n}, {:.1}, {:.1}, {:.1}", tot(rows[0].1, rows[0].2), tot(rows[1].1, rows[1].2), - tot(prep, q_set), tot(prep, q_count) ); } @@ -172,12 +163,12 @@ fn section(label: &str, r: &[[u32; 2]], s: &[[u32; 2]]) { }); let mut check = 0u64; for t in 0..64u32 { - check += want_bag - .rows - .iter() - .filter(|(row, _)| row[1].wrapping_mul(2_246_822_519) >> 29 == t % 8) - .map(|(_, &c)| u64::from(c)) - .sum::(); + for slot in 0..table.domain.len() { + let y = table.domain.id(slot); + if y.wrapping_mul(2_246_822_519) >> 29 == t % 8 { + check += reference::matching(&inp, [None, Some(y), None]); + } + } } assert_eq!(total, check); println!( @@ -189,16 +180,13 @@ fn section(label: &str, r: &[[u32; 2]], s: &[[u32; 2]]) { // One dataset resealed: re-prepare S only vs the whole table. let s2: Vec<[u32; 2]> = s.iter().step_by(2).copied().collect(); - let v2 = RelationView { - rows: &s2, - generation: 2, - }; + let v2 = RelationView::from_rows(&s2, 2); let (side, _) = time(|| { let mut t = table.clone(); - t.replace_side(1, v2, S_KEY).unwrap(); + t.replace_side(1, &v2, S_KEY).unwrap(); t }); - let (full, _) = time(|| JoinTable::prepare(inp.r, v2)); + let (full, _) = time(|| JoinTable::prepare(&inp.r, &v2)); let (clone, _) = time(|| table.clone()); println!( "S resealed: replace S side {:.1} us (includes a {:.1} us table clone) | full prepare {:.1} us", diff --git a/crates/mork-pathmap-probe/src/boundary.rs b/crates/mork-pathmap-probe/src/boundary.rs index efd88fff0..8d577b806 100644 --- a/crates/mork-pathmap-probe/src/boundary.rs +++ b/crates/mork-pathmap-probe/src/boundary.rs @@ -57,6 +57,9 @@ pub enum Refusal { }, /// The upstream index rejected the encoded input. Upstream(String), + /// The native arm answers by folds ([`crate::join::JoinExec`]); it builds + /// no rows. Compare it with [`BoundIndex::native`]. + FoldOnly, } /// Which join the compiled MORK runs for a conjunctive body. @@ -88,7 +91,7 @@ pub fn capability(backend: Backend, semantics: Semantics) -> Result<(), Refusal> enum Index { Mork(Box), Path(PathMap<()>), - Native(Box<(JoinTable, JoinExec)>, Semantics), + Native(Box<(JoinTable, JoinExec)>), } /// An upstream index built from one pair of generations. @@ -106,7 +109,7 @@ pub struct BoundIndex { /// translation cost is paid. pub fn bind( backend: Backend, - input: &JoinInput<'_>, + input: &JoinInput, semantics: Semantics, ) -> Result { capability(backend, semantics)?; @@ -115,16 +118,16 @@ pub fn bind( Backend::Mork => Index::Mork(Box::new(mork_space(input)?)), Backend::PathMapZipper => Index::Path(path_trie(input)), Backend::Native => { - let table = JoinTable::prepare(input.r, input.s); + let table = JoinTable::prepare(&input.r, &input.s); let exec = JoinExec::new(&table); - Index::Native(Box::new((table, exec)), semantics) + Index::Native(Box::new((table, exec))) } }; Ok(BoundIndex { backend, generations: input.generations(), index, - rows_in: input.r.rows.len() + input.s.rows.len(), + rows_in: input.r.len() + input.s.len(), build_time: start.elapsed(), }) } @@ -136,7 +139,32 @@ impl BoundIndex { /// Answers the query against the views the caller holds now. Refuses if /// they are not the generations this index was built from. - pub fn answers(&mut self, current: &JoinInput<'_>) -> Result { + pub fn answers(&mut self, current: &JoinInput) -> Result { + self.check(current)?; + match &mut self.index { + Index::Mork(space) => mork_answers(space), + Index::Path(trie) => Ok(path_answers(trie)), + Index::Native(_) => Err(Refusal::FoldOnly), + } + } + + /// The native arm's table and executor, for its folds. `None` for the + /// upstream arms. Refuses stale generations like [`Self::answers`]. + pub fn native( + &mut self, + current: &JoinInput, + ) -> Result, Refusal> { + self.check(current)?; + Ok(match &mut self.index { + Index::Native(te) => { + let (table, exec) = &mut **te; + Some((&*table, exec)) + } + _ => None, + }) + } + + fn check(&self, current: &JoinInput) -> Result<(), Refusal> { let now = current.generations(); if now != self.generations { return Err(Refusal::Stale { @@ -144,25 +172,17 @@ impl BoundIndex { current: now, }); } - match &mut self.index { - Index::Mork(space) => mork_answers(space), - Index::Path(trie) => Ok(path_answers(trie)), - Index::Native(te, semantics) => { - let (table, exec) = &mut **te; - exec.answers(table, now, None, *semantics) - .map_err(|e| Refusal::Upstream(format!("{e:?}"))) - } - } + Ok(()) } } // ---- MORK ------------------------------------------------------------------ /// Symbols are `n`, so the text round trip is exact. -fn mork_space(input: &JoinInput<'_>) -> Result { - let mut text = Vec::with_capacity(16 * (input.r.rows.len() + input.s.rows.len())); - for (name, rows) in [("R", input.r.rows), ("S", input.s.rows)] { - for &[a, b] in rows { +fn mork_space(input: &JoinInput) -> Result { + let mut text = Vec::with_capacity(16 * (input.r.len() + input.s.len())); + for (name, rel) in [("R", &input.r), ("S", &input.s)] { + for (a, b) in rel.cols[0].iter().zip(rel.cols[1].iter()) { text.extend_from_slice(format!("({name} n{a} n{b})\n").as_bytes()); } } @@ -218,12 +238,12 @@ const S_TAG: u8 = b'S'; /// `R(x, y)` is keyed `R ‖ y ‖ x` and `S(y, z)` is keyed `S ‖ y ‖ z` /// (big-endian), so both are ordered by the shared variable. -fn path_trie(input: &JoinInput<'_>) -> PathMap<()> { +fn path_trie(input: &JoinInput) -> PathMap<()> { let mut trie = PathMap::new(); let mut key = [0u8; 9]; - for (tag, rows, swap) in [(R_TAG, input.r.rows, true), (S_TAG, input.s.rows, false)] { + for (tag, rel, swap) in [(R_TAG, &input.r, true), (S_TAG, &input.s, false)] { key[0] = tag; - for &[a, b] in rows { + for (&a, &b) in rel.cols[0].iter().zip(rel.cols[1].iter()) { let (first, second) = if swap { (b, a) } else { (a, b) }; key[1..5].copy_from_slice(&first.to_be_bytes()); key[5..9].copy_from_slice(&second.to_be_bytes()); @@ -396,14 +416,8 @@ mod tests { let r = [[1, 10], [2, 10]]; let s = [[10, 5]]; let inp = JoinInput { - r: RelationView { - rows: &r, - generation: 1, - }, - s: RelationView { - rows: &s, - generation: 1, - }, + r: RelationView::from_rows(&r, 1), + s: RelationView::from_rows(&s, 1), }; let mut idx = bind(Backend::Mork, &inp, Semantics::Set).unwrap(); let Index::Mork(space) = &idx.index else { diff --git a/crates/mork-pathmap-probe/src/join.rs b/crates/mork-pathmap-probe/src/join.rs index c9f3cd3c5..0a265a335 100644 --- a/crates/mork-pathmap-probe/src/join.rs +++ b/crates/mork-pathmap-probe/src/join.rs @@ -14,31 +14,41 @@ //! across both datasets. A value's position is its **slot**. This is the one //! global-to-local translation, done once in [`JoinTable::prepare`] by binary //! search and timed there. -//! - [`Side`]: one input relation laid out by `y` slot: a presence plane (bit -//! `y` set when the relation has a row with that `y`), `off[y]..off[y + 1]`, -//! and the other column's values in that range. Each side is prepared from -//! its own dataset and carries that dataset's generation, so R and S can come -//! from different sources and be resealed independently. +//! - [`Side`]: one input relation laid out by `y` slot, as an index map only: +//! a presence plane (bit `y` set when the relation has a row with that `y`), +//! `off[y]..off[y + 1]`, and in that range the POSITIONS of those rows in +//! the relation's own lanes. The values are never copied: a slot's rows are +//! read in place as `lane[pos[k]]` (a [`Run`]), the template +//! `"{0}{1}" -f $laneA[$i],$laneB[$i]` with the positions as `$i`. Each side +//! is prepared from its own dataset and carries that dataset's generation, +//! so R and S can come from different sources and be resealed +//! independently. +//! - Answers are folds: [`JoinExec::bag_count`] reads only the offsets, +//! [`JoinExec::matching`] counts the derivations of a pattern, and +//! [`JoinExec::for_each_answer`] hands out rows read in place. While one slot +//! is read the next slot's runs are prefetched (`ndarray::simd`), because +//! the lanes are other tables' memory than the plane that chose the slot. //! //! # Evaluation, and why it is not a gather //! //! Leapfrog triejoin with `y` first intersects R's `y` level with S's `y` level, //! then enumerates `x` and `z` independently (they share no variable). Here the //! intersection is one mask-risc `And` over the two presence planes, which sit -//! over the same slot space. Enumeration then reads `off[y]` and the value range -//! of each side at the surviving slot `y`: every read is at the coordinate the -//! mask names, in every operand. The `x` and `z` values are emitted, never used +//! over the same slot space. Enumeration then reads `off[y]` and the positions +//! of each side at the surviving slot `y`, and reads the lane at them: every +//! read is at the coordinate the mask names or at a position the index map +//! holds, never at a position taken from another lane's values. The `x` and `z` values are emitted, never used //! as addresses. No operand is read at a position taken from another operand's //! values (2026-10-16 ruling). //! -//! Multiplicities are kept, so this arm answers [`Semantics::Bag`] as well as -//! [`Semantics::Set`]; both upstream indexes are sets. +//! Multiplicities are kept, so this arm answers bag counts; both upstream +//! indexes are sets. use lance_graph_mask_risc::{ Foreign, MaskOp, Operand, Out, Planes, Program, Scratch, Terminal, execute_into, words_for, }; -use crate::{Answers, RelationView, Semantics}; +use crate::{Lane, RelationView}; /// Why the table did not answer. #[derive(Clone, Debug, PartialEq, Eq)] @@ -95,31 +105,78 @@ impl JoinDomain { } } -/// One relation laid out by `y` slot. +/// One relation laid out by `y` slot, as an index map only: a presence plane, +/// `off[y]..off[y + 1]`, and in that range `pos`, the positions of the rows +/// with that `y` in the relation's own lanes. The values stay in the lane; +/// nothing is copied out of it. This map is the global-to-local inversion, +/// the one thing preparation materializes. #[derive(Clone, Debug)] pub struct Side { pub generation: u64, present: Vec, off: Vec, - vals: Vec, + pos: Vec, + /// The non-key lane, shared with the relation (a reference count, not a + /// copy). Read at `pos`. + lane: Lane, +} + +/// The rows of one side at one slot, read in place: `lane[pos[k]]`. A +/// borrowed view; it holds positions and the lane, never values. +#[derive(Clone, Copy, Debug)] +pub struct Run<'a> { + lane: &'a [u32], + pos: &'a [u32], +} + +impl<'a> Run<'a> { + pub fn len(&self) -> usize { + self.pos.len() + } + + pub fn is_empty(&self) -> bool { + self.pos.is_empty() + } + + /// The `k`-th row's value, read from the lane at its position. + pub fn get(&self, k: usize) -> u32 { + self.lane[self.pos[k] as usize] + } + + pub fn iter(&self) -> impl Iterator + 'a { + let lane = self.lane; + self.pos.iter().map(move |&p| lane[p as usize]) + } + + /// Rows whose value equals `v`. + pub fn count_eq(&self, v: u32) -> usize { + self.iter().filter(|&x| x == v).count() + } + + /// Asks for the run's first lane cache line ahead of its read: the lane + /// sits in another table's memory than the plane that chose the slot. + pub fn prefetch(&self) { + if let Some(&p) = self.pos.first() { + ndarray::simd::prefetch_read_t0(self.lane[p as usize..].as_ptr().cast()); + } + } } impl Side { - /// Lays `rows` out by the slot of column `key`; the other column is the - /// value. A counting sort over slots, so the value order within one slot - /// is the input order. + /// Lays the relation out by the slot of lane `key`; the other lane holds + /// the value. A counting sort over slots, so positions within one slot + /// are in row order. pub fn prepare( domain: &JoinDomain, - rows: &[[u32; 2]], + view: &RelationView, key: usize, - generation: u64, side: usize, ) -> Result { let n = domain.len(); - let mut slots = Vec::with_capacity(rows.len()); + let keys = &view.cols[key]; + let mut slots = Vec::with_capacity(keys.len()); let mut off = vec![0u32; n + 1]; - for row in rows { - let y = row[key]; + for &y in keys.iter() { let s = domain .slot_of(y) .ok_or(JoinRefusal::OutsideDomain { side, y })?; @@ -130,10 +187,10 @@ impl Side { off[i + 1] += off[i]; } let mut fill = off.clone(); - let mut vals = vec![0u32; rows.len()]; - for (row, &s) in rows.iter().zip(&slots) { + let mut pos = vec![0u32; keys.len()]; + for (row, &s) in slots.iter().enumerate() { let at = &mut fill[s as usize]; - vals[*at as usize] = row[1 - key]; + pos[*at as usize] = row as u32; *at += 1; } let mut present = vec![0u64; words_for(n)]; @@ -143,35 +200,52 @@ impl Side { } } Ok(Side { - generation, + generation: view.generation, present, off, - vals, + pos, + lane: view.cols[1 - key].clone(), }) } - /// The values at slot `y`. - pub fn at(&self, y: usize) -> &[u32] { - &self.vals[self.off[y] as usize..self.off[y + 1] as usize] + /// The rows at slot `y`, read in place. + pub fn at(&self, y: usize) -> Run<'_> { + Run { + lane: &self.lane, + pos: &self.pos[self.off[y] as usize..self.off[y + 1] as usize], + } } - pub fn present(&self) -> &[u64] { - &self.present + /// Rows at slot `y`, from the offsets alone (no lane read). + pub fn count(&self, y: usize) -> usize { + (self.off[y + 1] - self.off[y]) as usize } - /// The other column, in slot order: the side's own row order. Row `i` - /// sits in slot `s` for `offsets()[s] <= i < offsets()[s + 1]`. - pub fn values(&self) -> &[u32] { - &self.vals + pub fn present(&self) -> &[u64] { + &self.present } /// `off`: the start of each slot's run, `domain.len() + 1` entries. + /// Row `k` of the side sits in slot `s` for `off[s] <= k < off[s + 1]`. pub fn offsets(&self) -> &[u32] { &self.off } + /// Row positions into the relation's lanes, in slot order. + pub fn positions(&self) -> &[u32] { + &self.pos + } + + /// The non-key lane, shared with the relation. The side's `k`-th value + /// is `lane()[positions()[k]]`, read in place. + pub fn lane(&self) -> &[u32] { + &self.lane + } + + /// Bytes the index map adds; the lane is shared with the relation and + /// not counted. pub fn bytes(&self) -> usize { - (self.present.len() * 8) + (self.off.len() * 4) + (self.vals.len() * 4) + (self.present.len() * 8) + (self.off.len() * 4) + (self.pos.len() * 4) } } @@ -190,24 +264,22 @@ pub const S_KEY: usize = 0; impl JoinTable { /// Prepares the table for `R(x, y), S(y, z)`. - pub fn prepare(r: RelationView<'_>, s: RelationView<'_>) -> JoinTable { + pub fn prepare(r: &RelationView, s: &RelationView) -> JoinTable { Self::prepare_star(&[(r, R_KEY), (s, S_KEY)]) } /// Prepares a star of relations that all share `y`: every relation is laid /// out over the same domain. `key` names the column holding `y`. - pub fn prepare_star(inputs: &[(RelationView<'_>, usize)]) -> JoinTable { + pub fn prepare_star(inputs: &[(&RelationView, usize)]) -> JoinTable { let domain = JoinDomain::build( inputs .iter() - .flat_map(|(v, key)| v.rows.iter().map(move |r| r[*key])), + .flat_map(|(v, key)| v.cols[*key].iter().copied()), ); let sides = inputs .iter() .enumerate() - .map(|(i, (v, key))| { - Side::prepare(&domain, v.rows, *key, v.generation, i).expect("domain is the union") - }) + .map(|(i, (v, key))| Side::prepare(&domain, v, *key, i).expect("domain is the union")) .collect(); JoinTable { domain, sides } } @@ -217,10 +289,10 @@ impl JoinTable { pub fn replace_side( &mut self, side: usize, - view: RelationView<'_>, + view: &RelationView, key: usize, ) -> Result<(), JoinRefusal> { - self.sides[side] = Side::prepare(&self.domain, view.rows, key, view.generation, side)?; + self.sides[side] = Side::prepare(&self.domain, view, key, side)?; Ok(()) } @@ -316,58 +388,87 @@ impl JoinExec { Ok(&self.out) } - /// The answers of `R(x, y), S(y, z)`, enumerated at the joint slots. - pub fn answers( + /// Derivations reading as `[x, y, z]` (`None`: any value): a fold. A + /// pinned `y` is inverted to its slot and only that slot is read; + /// otherwise every joint slot. Per slot it is `count_R(x) · count_S(z)`, + /// with a free column counted from the offsets alone. + pub fn matching( &mut self, table: &JoinTable, current: (u64, u64), filter: Option<&[u64]>, - semantics: Semantics, - ) -> Result { + pattern: [Option; 3], + ) -> Result { table.check(&[current.0, current.1])?; let n = table.domain.len(); let joint = self.joint(table, filter)?; let (r, s) = (&table.sides[0], &table.sides[1]); - let mut out = Answers::default(); - for_each_set(joint, n, |slot| { - let y = table.domain.id(slot); - for &x in r.at(slot) { - for &z in s.at(slot) { - *out.rows.entry([x, y, z]).or_insert(0) += 1; - } - } - }); - Ok(match semantics { - Semantics::Bag => out, - Semantics::Set => out.to_set(), - }) + let at = |slot: usize| -> u64 { + let cr = pattern[0].map_or(r.count(slot), |x| r.at(slot).count_eq(x)); + let cs = pattern[2].map_or(s.count(slot), |z| s.at(slot).count_eq(z)); + cr as u64 * cs as u64 + }; + if let Some(y) = pattern[1] { + let Some(slot) = table.domain.slot_of(y) else { + return Ok(0); + }; + let kept = joint[slot >> 6] >> (slot & 63) & 1 == 1; + return Ok(if kept { at(slot) } else { 0 }); + } + let mut total = 0u64; + for_each_set_ahead( + joint, + n, + |next| { + r.at(next).prefetch(); + s.at(next).prefetch(); + }, + |slot| total += at(slot), + ); + Ok(total) } - /// Every bag answer `[x, y, z]`, handed to `f` in slot order, without - /// building a map: the join's own cost, separated from the output's. + /// Every bag derivation, in slot order, as a row read in place: `f` gets + /// a [`JoinRow`] whose columns are lane reads at the row positions. The + /// next slot's runs are prefetched while this one is read. pub fn for_each_answer( &mut self, table: &JoinTable, current: (u64, u64), - mut f: impl FnMut([u32; 3]), + mut f: impl FnMut(&JoinRow<'_>), ) -> Result<(), JoinRefusal> { table.check(&[current.0, current.1])?; let n = table.domain.len(); let joint = self.joint(table, None)?; let (r, s) = (&table.sides[0], &table.sides[1]); - for_each_set(joint, n, |slot| { - let y = table.domain.id(slot); - for &x in r.at(slot) { - for &z in s.at(slot) { - f([x, y, z]); + for_each_set_ahead( + joint, + n, + |next| { + r.at(next).prefetch(); + s.at(next).prefetch(); + }, + |slot| { + let (rr, sr) = (r.at(slot), s.at(slot)); + let y = table.domain.id(slot); + for i in 0..rr.len() { + for j in 0..sr.len() { + f(&JoinRow { + y, + r: rr, + s: sr, + i, + j, + }); + } } - } - }); + }, + ); Ok(()) } - /// The number of bag answers of the star, `Σ_y Π_i |side_i at y|`, without - /// enumerating a row. + /// The number of bag answers of the star, `Σ_y Π_i |side_i at y|`, from + /// the offsets alone: no lane is read. pub fn bag_count( &mut self, table: &JoinTable, @@ -382,13 +483,59 @@ impl JoinExec { total += table .sides .iter() - .map(|s| s.at(slot).len() as u64) + .map(|s| s.count(slot) as u64) .product::(); }); Ok(total) } } +/// One derivation of `R(x, y), S(y, z)`, read in place: `x` is R's run at +/// row `i`, `z` is S's run at row `j`. The template `"{x}{y}{z}"` over the +/// lanes; nothing is copied. +#[derive(Clone, Copy, Debug)] +pub struct JoinRow<'a> { + y: u32, + r: Run<'a>, + s: Run<'a>, + i: usize, + j: usize, +} + +impl JoinRow<'_> { + pub fn x(&self) -> u32 { + self.r.get(self.i) + } + + pub fn y(&self) -> u32 { + self.y + } + + pub fn z(&self) -> u32 { + self.s.get(self.j) + } +} + +/// Calls `ahead(next)` for the next set bit before `f(slot)` for the current +/// one: the next slot's reads are requested while this one is processed. +pub fn for_each_set_ahead( + words: &[u64], + n: usize, + mut ahead: impl FnMut(usize), + mut f: impl FnMut(usize), +) { + let mut pending: Option = None; + for_each_set(words, n, |slot| { + ahead(slot); + if let Some(prev) = pending.replace(slot) { + f(prev); + } + }); + if let Some(last) = pending { + f(last); + } +} + /// Calls `f` for every set bit below `n`, in slot order. pub fn for_each_set(words: &[u64], n: usize, mut f: impl FnMut(usize)) { for (wi, &w) in words.iter().enumerate() { diff --git a/crates/mork-pathmap-probe/src/lib.rs b/crates/mork-pathmap-probe/src/lib.rs index 79d301ae9..c27d5406f 100644 --- a/crates/mork-pathmap-probe/src/lib.rs +++ b/crates/mork-pathmap-probe/src/lib.rs @@ -11,14 +11,18 @@ //! //! Ownership, as the module layout states it: //! -//! - [`RelationView`] is the Lance side: borrowed rows plus the generation of the -//! sealed view they came from. It is the probe's stand-in for a sealed -//! `RelationOrder` view, which does not exist in code yet (D-RMAP is PROPOSED). +//! - [`RelationView`] is the Lance side: a relation as two lanes (Arrow +//! buffers, shared by reference count, never copied) plus the generation of +//! the sealed view they came from. A row is the template over the lanes at +//! one position; it is never held as a pair. +//! - [`join`] is the native arm: an index map per side and folds over it. +//! It builds no row (`"{0}{1}" -f $laneA[$i],$laneB[$i]`). //! - [`boundary`] is the only module that names `mork` or `pathmap`. Replacing //! or removing a backend touches that module and `Cargo.toml`, nothing else. -//! - No upstream type crosses the boundary in either direction: callers pass -//! `[u32; 2]` rows in and get `[u32; 3]` rows out. A MORK `Expr` is a raw -//! pointer with no lifetime, so it never leaves the boundary module. +//! Those two engines store their own encoded copies (a trie of paths) and +//! answer in rows ([`Answers`]); that is their representation, read only to +//! compare them against the native folds. A MORK `Expr` is a raw pointer +//! with no lifetime, so it never leaves the boundary module. use std::collections::BTreeMap; @@ -31,22 +35,57 @@ pub mod tile_gemm; pub use boundary::{Backend, BoundIndex, Refusal, bind, capability}; -/// Borrowed rows of one binary relation and the generation of the sealed view -/// they were read from. Row order and duplicates are the caller's: a bag. -#[derive(Clone, Copy, Debug)] -pub struct RelationView<'a> { - pub rows: &'a [[u32; 2]], +/// One column of a relation: Arrow's reference-counted buffer. Cloning a lane +/// shares its bytes; a lane read from Lance is the column buffer itself. +pub type Lane = arrow_buffer::ScalarBuffer; + +/// One binary relation as two lanes and the generation of the sealed view +/// they were read from. Row `i` is `(cols[0][i], cols[1][i])`, read in place +/// at `i`; it is never held as a pair. Row order and duplicates are the +/// caller's: a bag. +#[derive(Clone, Debug)] +pub struct RelationView { + pub cols: [Lane; 2], pub generation: u64, } +impl RelationView { + pub fn new(c0: impl Into, c1: impl Into, generation: u64) -> Self { + let cols = [c0.into(), c1.into()]; + assert_eq!( + cols[0].len(), + cols[1].len(), + "a relation's lanes have one length" + ); + RelationView { cols, generation } + } + + /// Builds the two lanes from authored fixture rows, once. For tests and + /// examples that write their data as literals; a stored relation arrives + /// as lanes. + pub fn from_rows(rows: &[[u32; 2]], generation: u64) -> Self { + let c0: Vec = rows.iter().map(|r| r[0]).collect(); + let c1: Vec = rows.iter().map(|r| r[1]).collect(); + RelationView::new(c0, c1, generation) + } + + pub fn len(&self) -> usize { + self.cols[0].len() + } + + pub fn is_empty(&self) -> bool { + self.cols[0].is_empty() + } +} + /// The two inputs of `Q(x, y, z) :- R(x, y), S(y, z)`. -#[derive(Clone, Copy, Debug)] -pub struct JoinInput<'a> { - pub r: RelationView<'a>, - pub s: RelationView<'a>, +#[derive(Clone, Debug)] +pub struct JoinInput { + pub r: RelationView, + pub s: RelationView, } -impl JoinInput<'_> { +impl JoinInput { pub fn generations(&self) -> (u64, u64) { (self.r.generation, self.s.generation) } @@ -64,7 +103,10 @@ pub enum Semantics { Bag, } -/// Answers with their multiplicities. Under [`Semantics::Set`] every count is 1. +/// The upstream engines' answers, in their own representation (rows): what +/// MORK and PathMap return, kept only to compare them against the native +/// folds. The native arm never builds one. Under [`Semantics::Set`] every +/// count is 1. #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct Answers { pub rows: BTreeMap, diff --git a/crates/mork-pathmap-probe/src/reference.rs b/crates/mork-pathmap-probe/src/reference.rs index 497baf9fe..9179617ac 100644 --- a/crates/mork-pathmap-probe/src/reference.rs +++ b/crates/mork-pathmap-probe/src/reference.rs @@ -1,20 +1,30 @@ -//! Independent oracle: a nested loop over the borrowed rows. Names no upstream -//! type and shares no code with [`crate::boundary`]. +//! Independent oracle: a nested loop over the two relations' lanes. Names no +//! upstream type and shares no code with [`crate::join`] or +//! [`crate::boundary`]. It builds no row either: it counts. -use crate::{Answers, JoinInput, Semantics}; +use crate::JoinInput; -/// `Q(x, y, z) :- R(x, y), S(y, z)` by nested loop. -pub fn answers(input: &JoinInput<'_>, semantics: Semantics) -> Answers { - let mut out = Answers::default(); - for &[x, y] in input.r.rows { - for &[y2, z] in input.s.rows { - if y == y2 { - *out.rows.entry([x, y, z]).or_insert(0) += 1; +/// Derivations of `Q(x, y, z) :- R(x, y), S(y, z)` reading as `pattern` +/// (`None`: any value), by nested loop over `R`'s and `S`'s lanes. +pub fn matching(input: &JoinInput, pattern: [Option; 3]) -> u64 { + let hit = |p: Option, v: u32| p.is_none_or(|w| w == v); + let (rx, ry) = (&input.r.cols[0], &input.r.cols[1]); + let (sy, sz) = (&input.s.cols[0], &input.s.cols[1]); + let mut n = 0u64; + for i in 0..rx.len() { + if !hit(pattern[0], rx[i]) || !hit(pattern[1], ry[i]) { + continue; + } + for j in 0..sy.len() { + if sy[j] == ry[i] && hit(pattern[2], sz[j]) { + n += 1; } } } - match semantics { - Semantics::Bag => out, - Semantics::Set => out.to_set(), - } + n +} + +/// Every derivation, counted. +pub fn bag_count(input: &JoinInput) -> u64 { + matching(input, [None; 3]) } diff --git a/crates/mork-pathmap-probe/tests/differential.rs b/crates/mork-pathmap-probe/tests/differential.rs index f66104faa..1909367fb 100644 --- a/crates/mork-pathmap-probe/tests/differential.rs +++ b/crates/mork-pathmap-probe/tests/differential.rs @@ -5,38 +5,62 @@ use mork_pathmap_probe::{ reference, }; -fn input<'a>(r: &'a [[u32; 2]], s: &'a [[u32; 2]], gen_r: u64, gen_s: u64) -> JoinInput<'a> { +fn input(r: &[[u32; 2]], s: &[[u32; 2]], gen_r: u64, gen_s: u64) -> JoinInput { JoinInput { - r: RelationView { - rows: r, - generation: gen_r, - }, - s: RelationView { - rows: s, - generation: gen_s, - }, + r: RelationView::from_rows(r, gen_r), + s: RelationView::from_rows(s, gen_s), } } const ARMS: [Backend; 3] = [Backend::Mork, Backend::PathMapZipper, Backend::Native]; const SET_ONLY: [Backend; 2] = [Backend::Mork, Backend::PathMapZipper]; +fn pattern(row: &[u32; 3]) -> [Option; 3] { + row.map(Some) +} + +/// The native arm against the counting oracle, then each upstream engine's +/// set against both: every row it returns has a derivation, and every native +/// derivation (read in place) is in its set. Returns the set's size. fn check_set(r: &[[u32; 2]], s: &[[u32; 2]]) -> usize { let inp = input(r, s, 1, 1); - let want = reference::answers(&inp, Semantics::Set); - for arm in ARMS { + let mut nat = bind(Backend::Native, &inp, Semantics::Set).expect("native binds"); + assert_eq!( + nat.answers(&inp), + Err(Refusal::FoldOnly), + "native builds no rows" + ); + let (table, exec) = nat.native(&inp).unwrap().expect("native arm"); + assert_eq!( + exec.bag_count(table, &[1, 1], None).unwrap(), + reference::bag_count(&inp) + ); + let mut size = None; + for arm in SET_ONLY { let mut idx = bind(arm, &inp, Semantics::Set).expect("set semantics is admitted"); let got = idx.answers(&inp).expect("fresh generations"); - assert_eq!( - got, - want, - "{arm:?} ({}) disagrees with the oracle", - mork_join() - ); + for row in got.rows.keys() { + let want = reference::matching(&inp, pattern(row)); + assert!(want > 0, "{arm:?} ({}) returned {row:?}", mork_join()); + assert_eq!( + exec.matching(table, (1, 1), None, pattern(row)).unwrap(), + want, + "native vs oracle at {row:?}" + ); + } + exec.for_each_answer(table, (1, 1), |d| { + assert!( + got.rows.contains_key(&[d.x(), d.y(), d.z()]), + "{arm:?} misses {:?}", + (d.x(), d.y(), d.z()) + ); + }) + .unwrap(); // Asking twice gives the same answer: the MORK arm removes its `Q` facts. - assert_eq!(idx.answers(&inp).unwrap(), want, "{arm:?} second call"); + assert_eq!(idx.answers(&inp).unwrap(), got, "{arm:?} second call"); + assert_eq!(*size.get_or_insert(got.rows.len()), got.rows.len()); } - want.rows.len() + size.unwrap() } #[test] @@ -78,19 +102,19 @@ fn duplicates_are_refused_under_bag_semantics() { let r = [[1, 10], [1, 10]]; let s = [[10, 5], [10, 5]]; let inp = input(&r, &s, 1, 1); - assert_eq!( - reference::answers(&inp, Semantics::Bag).rows[&[1, 10, 5]], - 4 - ); + assert_eq!(reference::matching(&inp, pattern(&[1, 10, 5])), 4); for arm in SET_ONLY { assert!(matches!(bind(arm, &inp, Semantics::Bag), Err(Refusal::SetOnly(a)) if a == arm)); } - // The native table keeps multiplicities and answers the bag exactly. + // The native table keeps multiplicities and counts the bag exactly. let mut native = bind(Backend::Native, &inp, Semantics::Bag).unwrap(); + let (table, exec) = native.native(&inp).unwrap().unwrap(); assert_eq!( - native.answers(&inp).unwrap(), - reference::answers(&inp, Semantics::Bag) + exec.matching(table, (1, 1), None, pattern(&[1, 10, 5])) + .unwrap(), + 4 ); + assert_eq!(exec.bag_count(table, &[1, 1], None).unwrap(), 4); // Under set semantics they agree with the oracle's set. assert_eq!(check_set(&r, &s), 1); } @@ -112,7 +136,11 @@ fn a_rebound_generation_is_refused() { }) ); // Silence twin: the matching generations still answer. - assert!(idx.answers(&bound).is_ok()); + if arm == Backend::Native { + assert!(idx.native(&bound).is_ok()); + } else { + assert!(idx.answers(&bound).is_ok()); + } } } @@ -138,7 +166,7 @@ fn a_reused_generation_number_is_not_detected() { } /// The native arm answers only through the co-addressed table, for both -/// semantics, and its bag answers carry the oracle's multiplicities. +/// semantics, and its bag counts carry the oracle's multiplicities. #[test] fn native_join_answers_bags_from_the_coaddressed_table() { assert_eq!(capability(Backend::Native, Semantics::Set), Ok(())); @@ -146,11 +174,36 @@ fn native_join_answers_bags_from_the_coaddressed_table() { let r = [[1, 10], [1, 10], [2, 10], [3, 11], [4, 99]]; let s = [[10, 5], [10, 6], [10, 5], [11, 7], [12, 8]]; let inp = input(&r, &s, 1, 1); - let want = reference::answers(&inp, Semantics::Bag); assert!( - want.rows.values().any(|&c| c > 1), + reference::matching(&inp, pattern(&[1, 10, 5])) > 1, "fixture has multiplicities" ); let mut idx = bind(Backend::Native, &inp, Semantics::Bag).unwrap(); - assert_eq!(idx.answers(&inp).unwrap(), want); + let (table, exec) = idx.native(&inp).unwrap().unwrap(); + assert_eq!( + exec.bag_count(table, &[1, 1], None).unwrap(), + reference::bag_count(&inp) + ); + // Every derivation read in place is one the oracle has; visited once each. + let mut visited = 0u64; + exec.for_each_answer(table, (1, 1), |d| { + assert!(reference::matching(&inp, [Some(d.x()), Some(d.y()), Some(d.z())]) > 0); + visited += 1; + }) + .unwrap(); + assert_eq!(visited, reference::bag_count(&inp)); + // Multiplicities per pattern, pinned and free columns alike. + for p in [ + [Some(1), Some(10), Some(5)], + [None, Some(10), None], + [None, None, Some(5)], + [Some(3), None, None], + [None, Some(99), None], + ] { + assert_eq!( + exec.matching(table, (1, 1), None, p).unwrap(), + reference::matching(&inp, p), + "{p:?}" + ); + } } diff --git a/crates/mork-pathmap-probe/tests/join.rs b/crates/mork-pathmap-probe/tests/join.rs index e07e656e2..e9a1c5869 100644 --- a/crates/mork-pathmap-probe/tests/join.rs +++ b/crates/mork-pathmap-probe/tests/join.rs @@ -3,10 +3,66 @@ use mork_pathmap_probe::join::{ JoinExec, JoinRefusal, JoinTable, R_KEY, S_KEY, filter_plane, for_each_set, }; -use mork_pathmap_probe::{JoinInput, RelationView, Semantics, reference}; +use mork_pathmap_probe::{JoinInput, RelationView, reference}; -fn view(rows: &[[u32; 2]], generation: u64) -> RelationView<'_> { - RelationView { rows, generation } +fn view(rows: &[[u32; 2]], generation: u64) -> RelationView { + RelationView::from_rows(rows, generation) +} + +/// The native table agrees with the counting oracle: the bag count, the +/// derivations visited in place, and per-pattern counts that pin each R +/// row's `(x, y)` and each S row's `(y, z)`, so a misread lane cannot pass. +/// `keep` restricts both to the filter's `y` values. +fn agrees( + exec: &mut JoinExec, + table: &JoinTable, + gens: (u64, u64), + filter: Option<&[u64]>, + inp: &JoinInput, + keep: impl Fn(u32) -> bool, +) -> u64 { + let oracle = |p: [Option; 3]| { + if p[1].is_some_and(|y| !keep(y)) { + return 0; + } + match p[1] { + Some(_) => reference::matching(inp, p), + None => (0..table.domain.len()) + .map(|slot| table.domain.id(slot)) + .filter(|&y| keep(y)) + .map(|y| reference::matching(inp, [p[0], Some(y), p[2]])) + .sum(), + } + }; + let total = oracle([None; 3]); + assert_eq!( + exec.bag_count(table, &[gens.0, gens.1], filter).unwrap(), + total + ); + let (rx, ry) = (&inp.r.cols[0], &inp.r.cols[1]); + for i in 0..rx.len() { + let p = [Some(rx[i]), Some(ry[i]), None]; + assert_eq!( + exec.matching(table, gens, filter, p).unwrap(), + oracle(p), + "{p:?}" + ); + } + let (sy, sz) = (&inp.s.cols[0], &inp.s.cols[1]); + for j in 0..sy.len() { + let p = [None, Some(sy[j]), Some(sz[j])]; + assert_eq!( + exec.matching(table, gens, filter, p).unwrap(), + oracle(p), + "{p:?}" + ); + } + if filter.is_none() { + let mut visited = 0u64; + exec.for_each_answer(table, gens, |_| visited += 1).unwrap(); + assert_eq!(visited, total); + } + total } /// Deterministic rows with ids spread over the whole `u32` range: the two @@ -46,17 +102,12 @@ fn heterogeneous_join_matches_the_oracle_in_both_semantics() { r: view(&r, 3), s: view(&s, 11), }; - let table = JoinTable::prepare(inp.r, inp.s); + let table = JoinTable::prepare(&inp.r, &inp.s); let mut exec = JoinExec::new(&table); - for sem in [Semantics::Set, Semantics::Bag] { - let want = reference::answers(&inp, sem); - assert!(!want.rows.is_empty()); - assert_eq!(exec.answers(&table, (3, 11), None, sem).unwrap(), want); - } + let total = agrees(&mut exec, &table, (3, 11), None, &inp, |_| true); + assert!(total > 0); // Not degenerate: some y are in one relation only, and are excluded. assert!(table.domain.len() > 48); - let bag = reference::answers(&inp, Semantics::Bag).total(); - assert_eq!(exec.bag_count(&table, &[3, 11], None).unwrap(), bag); } #[test] @@ -66,9 +117,9 @@ fn a_star_of_three_relations_counts_like_the_oracle() { let s = swap(&hetero(150, 5, &all[10..], 1)); let t = swap(&hetero(120, 6, &all[5..35], 3)); let table = JoinTable::prepare_star(&[ - (view(&r, 1), R_KEY), - (view(&s, 1), S_KEY), - (view(&t, 1), S_KEY), + (&view(&r, 1), R_KEY), + (&view(&s, 1), S_KEY), + (&view(&t, 1), S_KEY), ]); let mut exec = JoinExec::new(&table); let mut want = 0u64; @@ -80,7 +131,7 @@ fn a_star_of_three_relations_counts_like_the_oracle() { assert!(want > 0); assert_eq!(exec.bag_count(&table, &[1, 1, 1], None).unwrap(), want); // The third relation narrows the joint: dropping it gives a different count. - let two = JoinTable::prepare_star(&[(view(&r, 1), R_KEY), (view(&s, 1), S_KEY)]); + let two = JoinTable::prepare_star(&[(&view(&r, 1), R_KEY), (&view(&s, 1), S_KEY)]); let mut exec2 = JoinExec::new(&two); assert_ne!(exec2.bag_count(&two, &[1, 1], None).unwrap(), want); } @@ -94,22 +145,17 @@ fn a_filter_plane_restricts_the_join_to_the_thoughts_slots() { r: view(&r, 1), s: view(&s, 1), }; - let table = JoinTable::prepare(inp.r, inp.s); + let table = JoinTable::prepare(&inp.r, &inp.s); let mut exec = JoinExec::new(&table); let keep = |y: u32| y.is_multiple_of(3); let filter = filter_plane(&table, keep); - let got = exec - .answers(&table, (1, 1), Some(&filter), Semantics::Bag) - .unwrap(); - let mut want = reference::answers(&inp, Semantics::Bag); - want.rows.retain(|row, _| keep(row[1])); - assert!(!want.rows.is_empty()); - assert_eq!(got, want); + let kept = agrees(&mut exec, &table, (1, 1), Some(&filter), &inp, keep); + assert!(kept > 0); // Silence twin: the filter removes something. - assert!(want.total() < reference::answers(&inp, Semantics::Bag).total()); + assert!(kept < reference::bag_count(&inp)); // A plane over another slot space is refused, not read past its end. assert!(matches!( - exec.answers(&table, (1, 1), Some(&[0u64; 1000]), Semantics::Set), + exec.bag_count(&table, &[1, 1], Some(&[0u64; 1000])), Err(JoinRefusal::FilterShape { .. }) )); } @@ -118,11 +164,11 @@ fn a_filter_plane_restricts_the_join_to_the_thoughts_slots() { fn each_side_is_refused_against_its_own_generation() { let r = [[1, 10], [2, 11]]; let s = [[10, 5], [11, 6]]; - let table = JoinTable::prepare(view(&r, 4), view(&s, 9)); + let table = JoinTable::prepare(&view(&r, 4), &view(&s, 9)); let mut exec = JoinExec::new(&table); - assert!(exec.answers(&table, (4, 9), None, Semantics::Set).is_ok()); + assert!(exec.bag_count(&table, &[4, 9], None).is_ok()); assert_eq!( - exec.answers(&table, (5, 9), None, Semantics::Set), + exec.bag_count(&table, &[5, 9], None), Err(JoinRefusal::Stale { side: 0, bound: 4, @@ -130,7 +176,7 @@ fn each_side_is_refused_against_its_own_generation() { }) ); assert_eq!( - exec.answers(&table, (4, 8), None, Semantics::Set), + exec.bag_count(&table, &[4, 8], None), Err(JoinRefusal::Stale { side: 1, bound: 9, @@ -145,49 +191,45 @@ fn replacing_one_side_equals_preparing_the_table_again() { let r = hetero(300, 1, &all, 1); let s1 = swap(&hetero(300, 2, &all, 2)); let s2 = swap(&hetero(250, 3, &all[..40], 1)); - let mut table = JoinTable::prepare(view(&r, 1), view(&s1, 1)); - table.replace_side(1, view(&s2, 2), S_KEY).unwrap(); + let mut table = JoinTable::prepare(&view(&r, 1), &view(&s1, 1)); + table.replace_side(1, &view(&s2, 2), S_KEY).unwrap(); let mut exec = JoinExec::new(&table); - let got = exec.answers(&table, (1, 2), None, Semantics::Bag).unwrap(); let inp = JoinInput { r: view(&r, 1), s: view(&s2, 2), }; - assert_eq!(got, reference::answers(&inp, Semantics::Bag)); + let got = agrees(&mut exec, &table, (1, 2), None, &inp, |_| true); // The old S answer differs, so the replacement did something. let old = JoinInput { r: view(&r, 1), s: view(&s1, 1), }; - assert_ne!(got, reference::answers(&old, Semantics::Bag)); + assert_ne!(got, reference::bag_count(&old)); } #[test] fn a_side_leaving_the_domain_is_refused() { let r = [[1, 10]]; let s = [[10, 5]]; - let mut table = JoinTable::prepare(view(&r, 1), view(&s, 1)); + let mut table = JoinTable::prepare(&view(&r, 1), &view(&s, 1)); let s2 = [[10, 5], [77, 6]]; assert_eq!( - table.replace_side(1, view(&s2, 2), S_KEY), + table.replace_side(1, &view(&s2, 2), S_KEY), Err(JoinRefusal::OutsideDomain { side: 1, y: 77 }) ); // The table is unchanged: the old generation still answers. let mut exec = JoinExec::new(&table); - assert!(exec.answers(&table, (1, 1), None, Semantics::Set).is_ok()); + assert_eq!(exec.bag_count(&table, &[1, 1], None), Ok(1)); } #[test] fn the_joint_plane_has_exactly_the_shared_slots() { let r = [[0, 1], [0, 3], [0, 5]]; let s = [[3, 0], [4, 0], [5, 0]]; - let table = JoinTable::prepare(view(&r, 1), view(&s, 1)); + let table = JoinTable::prepare(&view(&r, 1), &view(&s, 1)); let mut exec = JoinExec::new(&table); - let mut shared = Vec::new(); - let n = table.domain.len(); - let joint = exec.joint(&table, None).unwrap().to_vec(); - for_each_set(&joint, n, |slot| shared.push(table.domain.id(slot))); - assert_eq!(shared, vec![3, 5]); + assert_eq!(joint_ids(&mut exec, &table), (2, true)); + assert!(in_joint(&mut exec, &table, &[3, 5])); } /// The joint plane is itself an answer: the `y` values that every relation of @@ -200,14 +242,36 @@ fn the_star_joint_is_the_existence_mask() { let s = [[2, 0], [3, 0], [4, 0], [5, 0]]; let t = [[3, 0], [4, 0], [6, 0]]; let table = JoinTable::prepare_star(&[ - (view(&r, 1), R_KEY), - (view(&s, 1), S_KEY), - (view(&t, 1), S_KEY), + (&view(&r, 1), R_KEY), + (&view(&s, 1), S_KEY), + (&view(&t, 1), S_KEY), ]); let mut exec = JoinExec::new(&table); + assert_eq!(joint_ids(&mut exec, &table), (2, true)); + assert!(in_joint(&mut exec, &table, &[3, 4])); +} + +/// The joint's population and whether its slots are in ascending `y` order. +fn joint_ids(exec: &mut JoinExec, table: &JoinTable) -> (usize, bool) { let n = table.domain.len(); - let joint = exec.joint(&table, None).unwrap().to_vec(); - let mut shared = Vec::new(); - for_each_set(&joint, n, |slot| shared.push(table.domain.id(slot))); - assert_eq!(shared, vec![3, 4]); + let joint = exec.joint(table, None).unwrap(); + let (mut count, mut last, mut ordered) = (0, None, true); + for_each_set(joint, n, |slot| { + let y = table.domain.id(slot); + ordered &= last.is_none_or(|l| l < y); + last = Some(y); + count += 1; + }); + (count, ordered) +} + +/// Whether every `y` in `ys` is a joint slot. +fn in_joint(exec: &mut JoinExec, table: &JoinTable, ys: &[u32]) -> bool { + let joint = exec.joint(table, None).unwrap(); + ys.iter().all(|&y| { + table + .domain + .slot_of(y) + .is_some_and(|s| joint[s >> 6] >> (s & 63) & 1 == 1) + }) } From 1ff6b8bd06c443a0a7d23707a49ecb7fbddda17c Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 21 Oct 2026 10:38:51 +0000 Subject: [PATCH 02/14] mork-pathmap-probe: join tests check the rows read in place (catches a Run::get misread) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01G22yT6htkcdyXsihxxXdrg --- crates/mork-pathmap-probe/tests/join.rs | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/crates/mork-pathmap-probe/tests/join.rs b/crates/mork-pathmap-probe/tests/join.rs index e9a1c5869..a70e2ef62 100644 --- a/crates/mork-pathmap-probe/tests/join.rs +++ b/crates/mork-pathmap-probe/tests/join.rs @@ -58,9 +58,24 @@ fn agrees( ); } if filter.is_none() { - let mut visited = 0u64; - exec.for_each_answer(table, gens, |_| visited += 1).unwrap(); + // The rows read in place, folded: count and a checksum over x and z. + let mix = |x: u32, z: u32| u64::from(x).wrapping_mul(31).wrapping_add(u64::from(z)); + let (mut visited, mut sum) = (0u64, 0u64); + exec.for_each_answer(table, gens, |d| { + visited += 1; + sum = sum.wrapping_add(mix(d.x(), d.z())); + }) + .unwrap(); + let mut want = 0u64; + for i in 0..rx.len() { + for j in 0..sy.len() { + if ry[i] == sy[j] { + want = want.wrapping_add(mix(rx[i], sz[j])); + } + } + } assert_eq!(visited, total); + assert_eq!(sum, want, "rows read in place"); } total } From 8c88de1c578a62fb4f20e0bbe370d733e013860b Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 21 Oct 2026 10:49:40 +0000 Subject: [PATCH 03/14] mork-pathmap-probe: a restriction selects row positions (prepare_where), never copies rows Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01G22yT6htkcdyXsihxxXdrg --- crates/mork-pathmap-probe/src/join.rs | 58 ++++++++++++++++++++----- crates/mork-pathmap-probe/tests/join.rs | 34 +++++++++++++++ 2 files changed, 81 insertions(+), 11 deletions(-) diff --git a/crates/mork-pathmap-probe/src/join.rs b/crates/mork-pathmap-probe/src/join.rs index 0a265a335..4da68495f 100644 --- a/crates/mork-pathmap-probe/src/join.rs +++ b/crates/mork-pathmap-probe/src/join.rs @@ -171,25 +171,45 @@ impl Side { view: &RelationView, key: usize, side: usize, + ) -> Result { + Self::prepare_where(domain, view, key, side, |_| true) + } + + /// [`Self::prepare`] over the rows `keep` admits, by row position. A + /// restriction is a selection of positions, never a copy of rows: the + /// index map simply leaves the other positions out. + pub fn prepare_where( + domain: &JoinDomain, + view: &RelationView, + key: usize, + side: usize, + keep: impl Fn(usize) -> bool, ) -> Result { let n = domain.len(); let keys = &view.cols[key]; - let mut slots = Vec::with_capacity(keys.len()); let mut off = vec![0u32; n + 1]; - for &y in keys.iter() { + let mut kept = 0usize; + for (row, &y) in keys.iter().enumerate() { + if !keep(row) { + continue; + } let s = domain .slot_of(y) .ok_or(JoinRefusal::OutsideDomain { side, y })?; - slots.push(s as u32); off[s + 1] += 1; + kept += 1; } for i in 0..n { off[i + 1] += off[i]; } let mut fill = off.clone(); - let mut pos = vec![0u32; keys.len()]; - for (row, &s) in slots.iter().enumerate() { - let at = &mut fill[s as usize]; + let mut pos = vec![0u32; kept]; + for (row, &y) in keys.iter().enumerate() { + if !keep(row) { + continue; + } + let s = domain.slot_of(y).expect("checked above"); + let at = &mut fill[s]; pos[*at as usize] = row as u32; *at += 1; } @@ -257,6 +277,10 @@ pub struct JoinTable { pub sides: Vec, } +/// One input of [`JoinTable::prepare_star_where`]: the relation, the lane +/// holding the shared variable, and which row positions to keep. +pub type SelectedSide<'a> = (&'a RelationView, usize, &'a dyn Fn(usize) -> bool); + /// Where the shared variable sits in each input row: `R(x, y)` keys column 1, /// `S(y, z)` keys column 0. pub const R_KEY: usize = 1; @@ -271,15 +295,27 @@ impl JoinTable { /// Prepares a star of relations that all share `y`: every relation is laid /// out over the same domain. `key` names the column holding `y`. pub fn prepare_star(inputs: &[(&RelationView, usize)]) -> JoinTable { - let domain = JoinDomain::build( - inputs + let all = |_: usize| true; + let with: Vec> = inputs.iter().map(|&(v, k)| (v, k, &all as _)).collect(); + Self::prepare_star_where(&with) + } + + /// [`Self::prepare_star`] over the rows each side's `keep` admits (by row + /// position). The domain is the union of the kept rows' keys only. + pub fn prepare_star_where(inputs: &[SelectedSide<'_>]) -> JoinTable { + let domain = JoinDomain::build(inputs.iter().flat_map(|&(v, key, keep)| { + v.cols[key] .iter() - .flat_map(|(v, key)| v.cols[*key].iter().copied()), - ); + .enumerate() + .filter(move |&(row, _)| keep(row)) + .map(|(_, &y)| y) + })); let sides = inputs .iter() .enumerate() - .map(|(i, (v, key))| Side::prepare(&domain, v, *key, i).expect("domain is the union")) + .map(|(i, &(v, key, keep))| { + Side::prepare_where(&domain, v, key, i, keep).expect("domain is the union") + }) .collect(); JoinTable { domain, sides } } diff --git a/crates/mork-pathmap-probe/tests/join.rs b/crates/mork-pathmap-probe/tests/join.rs index a70e2ef62..bc49c211a 100644 --- a/crates/mork-pathmap-probe/tests/join.rs +++ b/crates/mork-pathmap-probe/tests/join.rs @@ -290,3 +290,37 @@ fn in_joint(exec: &mut JoinExec, table: &JoinTable, ys: &[u32]) -> bool { .is_some_and(|s| joint[s >> 6] >> (s & 63) & 1 == 1) }) } + +/// FAILS IF: a position selection is not the same table as preparing from +/// only the kept rows, or the selection does not exclude anything. +#[test] +fn a_position_selection_equals_preparing_the_kept_rows() { + let all = ys(40, 5); + let r = hetero(300, 11, &all, 1); + let s = swap(&hetero(300, 12, &all, 2)); + let (rv, sv) = (view(&r, 1), view(&s, 1)); + // Keep R rows whose x is even, and S rows at even positions. + let keep_r = |row: usize| rv.cols[0][row].is_multiple_of(2); + let keep_s = |row: usize| row.is_multiple_of(2); + let selected = JoinTable::prepare_star_where(&[(&rv, R_KEY, &keep_r), (&sv, S_KEY, &keep_s)]); + let r2: Vec<[u32; 2]> = r + .iter() + .copied() + .filter(|p| p[0].is_multiple_of(2)) + .collect(); + let s2: Vec<[u32; 2]> = s.iter().step_by(2).copied().collect(); + let inp = JoinInput { + r: view(&r2, 1), + s: view(&s2, 1), + }; + let mut exec = JoinExec::new(&selected); + let kept = agrees(&mut exec, &selected, (1, 1), None, &inp, |_| true); + let full = JoinInput { + r: rv.clone(), + s: sv.clone(), + }; + assert!( + kept > 0 && kept < reference::bag_count(&full), + "the selection excludes something" + ); +} From 1f5fd59633b6f2b19f2c5d5ae0c747472ca125e1 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 21 Oct 2026 10:57:56 +0000 Subject: [PATCH 04/14] crepe-plan-probe: sides read lanes through the index map Store holds RelationView lanes (arrow ScalarBuffer, shared with Lance's column buffers, no to_vec). Restriction and semijoin centers select row positions (prepare_views); restricted reads are no longer trimmed by retain. Derivations and slot views read Run (lane[pos[k]]) in place, with a one-slot prefetch lookahead. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01G22yT6htkcdyXsihxxXdrg --- crates/crepe-plan-probe/Cargo.toml | 1 + crates/crepe-plan-probe/examples/measure.rs | 22 ++- crates/crepe-plan-probe/examples/rule_cse.rs | 20 +-- crates/crepe-plan-probe/src/addressed.rs | 62 +++---- crates/crepe-plan-probe/src/cse.rs | 8 +- crates/crepe-plan-probe/src/lance_store.rs | 64 +++---- crates/crepe-plan-probe/src/native.rs | 170 +++++++++++-------- crates/crepe-plan-probe/src/pipeline.rs | 4 +- crates/crepe-plan-probe/tests/pipeline.rs | 115 +++++-------- crates/mork-pathmap-probe/src/join.rs | 7 +- 10 files changed, 222 insertions(+), 251 deletions(-) diff --git a/crates/crepe-plan-probe/Cargo.toml b/crates/crepe-plan-probe/Cargo.toml index 85f695fec..73e748886 100644 --- a/crates/crepe-plan-probe/Cargo.toml +++ b/crates/crepe-plan-probe/Cargo.toml @@ -30,6 +30,7 @@ lance-graph-contract = { path = "../lance-graph-contract" } lance = { version = "13.*", default-features = false } lancedb = { version = "0.40.*", default-features = false, features = ["remote"] } arrow-array = "58" +arrow-buffer = "58" arrow-schema = "58" futures = "0.3" tokio = { version = "1.37", features = ["macros", "rt-multi-thread"] } diff --git a/crates/crepe-plan-probe/examples/measure.rs b/crates/crepe-plan-probe/examples/measure.rs index 703bc602f..df5179d66 100644 --- a/crates/crepe-plan-probe/examples/measure.rs +++ b/crates/crepe-plan-probe/examples/measure.rs @@ -100,13 +100,11 @@ fn main() { .into_iter() .map(|r| r.unwrap()) .collect(); - let st = Store { - relations: vec![ - (labels.id("Parent").unwrap(), &parent, 1), - (labels.id("Allowed").unwrap(), &allowed, 1), - (labels.id("Sponsor").unwrap(), &sponsor, 1), - ], - }; + let st = Store::from_rows(vec![ + (labels.id("Parent").unwrap(), &parent, 1), + (labels.id("Allowed").unwrap(), &allowed, 1), + (labels.id("Sponsor").unwrap(), &sponsor, 1), + ]); let demand = Demand { goal: labels.id("Answer").unwrap(), bound: vec![Some(17), None], @@ -117,12 +115,12 @@ fn main() { let df = datafrog_answer(&parent, &allowed); let reference = (n <= 10_000).then(|| { let rows = |rel| { - let (r, _) = st.view(rel); - r.iter() - .map(|x| { - x[..labels.arity(rel)] + let v = st.view(rel); + (0..v.len()) + .map(|i| { + v.cols[..labels.arity(rel)] .iter() - .map(|&v| u64::from(v)) + .map(|c| u64::from(c[i])) .collect() }) .collect() diff --git a/crates/crepe-plan-probe/examples/rule_cse.rs b/crates/crepe-plan-probe/examples/rule_cse.rs index 2fc0fbb90..069510123 100644 --- a/crates/crepe-plan-probe/examples/rule_cse.rs +++ b/crates/crepe-plan-probe/examples/rule_cse.rs @@ -87,14 +87,14 @@ impl Workload { RelId(2 * self.cores.len() as u16 + 1) } - pub fn store(&self) -> Store<'_> { + pub fn store(&self) -> Store { let mut relations = Vec::new(); for (g, (r, s)) in self.cores.iter().enumerate() { relations.push((RelId(2 * g as u16), r.as_slice(), 1)); relations.push((RelId(2 * g as u16 + 1), s.as_slice(), 1)); } relations.push((self.t_rel(), self.t.as_slice(), 1)); - Store { relations } + Store::from_rows(relations) } } @@ -442,11 +442,9 @@ mod tests { fn reference_bag(w: &Workload) -> Vec<(Vec, u64)> { let store = w.store(); let rows = |r: Rid| -> Vec> { - store - .view(r) - .0 - .iter() - .map(|p| p.iter().map(|&v| u64::from(v)).collect()) + let v = store.view(r); + (0..v.len()) + .map(|i| v.cols.iter().map(|c| u64::from(c[i])).collect()) .collect() }; eval(&w.rules, w.out_rel(), &[None, None, None], &rows) @@ -534,11 +532,9 @@ mod tests { ); let store = w.store(); let rows = |r: Rid| -> Vec> { - store - .view(r) - .0 - .iter() - .map(|p| p.iter().map(|&v| u64::from(v)).collect()) + let v = store.view(r); + (0..v.len()) + .map(|i| v.cols.iter().map(|c| u64::from(c[i])).collect()) .collect() }; let want: Vec<(Vec, u64)> = eval(&w.rules, w.out_rel(), &bound, &rows) diff --git a/crates/crepe-plan-probe/src/addressed.rs b/crates/crepe-plan-probe/src/addressed.rs index a072b0db9..595f656c0 100644 --- a/crates/crepe-plan-probe/src/addressed.rs +++ b/crates/crepe-plan-probe/src/addressed.rs @@ -33,7 +33,8 @@ use lance_graph_contract::ordered_lane::{OrderedLaneWitness, SealedFacetLane, Wi use crate::lance_store::{Io, LanceStore, LanceStoreError, SideIo, read_rows}; use crate::logic::RelId; -use crate::native::{PreparedStar, StarPlan, prepare_from_rows}; +use crate::native::{PreparedStar, StarPlan, prepare_views}; +use mork_pathmap_probe::RelationView; /// The key of row `(a, b)`. pub fn key(a: u32, b: u32) -> FacetCascade { @@ -307,26 +308,24 @@ async fn current_lane<'a>( Ok((a, ds)) } -/// Reads one side's rows for the located `ranges` of the leading values in -/// `wanted` (sorted, deduplicated), as `policy` decides. A coalesced or full -/// read returns rows of other leading values too; they are dropped by the -/// row's own leading value, so the result equals the exact read. +/// Reads one side's lanes for the located `ranges`, as `policy` decides. A +/// coalesced or full read returns rows of other leading values too; they are +/// left out by position when the index map is built +/// ([`crate::native::prepare_views`]: the restriction and the centers), so +/// the prepared table equals the exact read's. async fn read_side( ds: &Dataset, a: &AddressedRelation, ranges: &[(u32, u32)], - wanted: &[u32], policy: ReadPolicy, -) -> Result<(Read, usize, Vec<[u32; 2]>, Io), AddressedRefusal> { +) -> Result<(Read, usize, RelationView, Io), AddressedRefusal> { let (how, r) = plan_read(ranges, a.lane.n_rows(), policy); let mask = match how { Read::Full => None, Read::Ranges | Read::Coalesced => Some(range_mask(&r)), }; let (mut got, rows_io) = read_rows(ds, mask).await?; - if how != Read::Ranges { - got.retain(|row| wanted.binary_search(&row[0]).is_ok()); - } + got.generation = a.lane.version(); Ok((how, r.len(), got, rows_io)) } @@ -351,7 +350,7 @@ pub async fn prepare_addressed_with( policy: ReadPolicy, ) -> Result<(PreparedStar, Vec, Vec), AddressedRefusal> { let n_sides = plan.sides.len(); - let mut rows: Vec, u64)>> = vec![None; n_sides]; + let mut rows: Vec> = vec![None; n_sides]; let mut io = vec![SideIo::default(); n_sides]; let mut lookups = vec![Lookup::default(); n_sides]; let restricted = |i: usize| { @@ -372,10 +371,7 @@ pub async fn prepare_addressed_with( Ok(k) => ranges(a, &[k])?, Err(_) => (Vec::new(), 0), }; - let wanted: Vec = u32::try_from(restricted(i).expect("restricted")) - .into_iter() - .collect(); - let (how, read_ranges, got, rows_io) = read_side(&ds, a, &r, &wanted, policy).await?; + let (how, read_ranges, got, rows_io) = read_side(&ds, a, &r, policy).await?; lookups[i] = Lookup { keys: 1, rows_in_ranges: n, @@ -387,17 +383,18 @@ pub async fn prepare_addressed_with( mask_rows: Some(n), rows: rows_io, }; - rows[i] = Some((got, a.lane.version())); + rows[i] = Some(got); } let mut centers: Option> = None; for i in (0..n_sides).filter(|&i| restricted(i).is_some()) { - let mut c: Vec = rows[i] - .as_ref() - .expect("read") - .0 - .iter() - .map(|r| r[1]) + // Rows of the restricted side whose leading value is the constant; + // a coalesced read holds others too. + let v = rows[i].as_ref().expect("read"); + let k = restricted(i).expect("restricted"); + let mut c: Vec = (0..v.len()) + .filter(|&row| u64::from(v.cols[0][row]) == k) + .map(|row| v.cols[1][row]) .collect(); c.sort_unstable(); c.dedup(); @@ -410,14 +407,8 @@ pub async fn prepare_addressed_with( }); } - // Restricted sides keep only the intersected centers (rows already read). - if let Some(c) = ¢ers { - for i in (0..n_sides).filter(|&i| restricted(i).is_some()) { - if let Some((r, _)) = rows[i].as_mut() { - r.retain(|row| c.binary_search(&row[1]).is_ok()); - } - } - } + // Restricted sides keep only the intersected centers: by position, in + // `prepare_views`; nothing is trimmed here. for i in (0..n_sides).filter(|&i| restricted(i).is_none()) { let key = plan.sides[i].key; @@ -430,11 +421,12 @@ pub async fn prepare_addressed_with( } Some(c) => { let (r, n) = ranges(a, c)?; - let (how, rr, got, rows_io) = read_side(&ds, a, &r, c, policy).await?; + let (how, rr, got, rows_io) = read_side(&ds, a, &r, policy).await?; (n, c.len(), how, rr, got, rows_io) } None => { - let (got, rows_io) = read_rows(&ds, None).await?; + let (mut got, rows_io) = read_rows(&ds, None).await?; + got.generation = a.lane.version(); (a.lane.n_rows() as usize, 0, Read::Full, 0, got, rows_io) } }; @@ -449,14 +441,14 @@ pub async fn prepare_addressed_with( mask_rows: centers.as_ref().map(|_| n), rows: rows_io, }; - rows[i] = Some((got, a.lane.version())); + rows[i] = Some(got); } - let owned = rows + let views = rows .into_iter() .map(|r| r.expect("every side read")) .collect(); - Ok((prepare_from_rows(plan, owned), io, lookups)) + Ok((prepare_views(plan, views, true), io, lookups)) } #[cfg(test)] diff --git a/crates/crepe-plan-probe/src/cse.rs b/crates/crepe-plan-probe/src/cse.rs index 86d401993..5486620b7 100644 --- a/crates/crepe-plan-probe/src/cse.rs +++ b/crates/crepe-plan-probe/src/cse.rs @@ -122,7 +122,7 @@ fn partition(stars: &[StarPlan], layout: Layout) -> Vec> { /// Lays out the distinct sides of each group's stars over the union of that /// group's centers. -pub fn prepare_shared(stars: &[StarPlan], store: &Store<'_>, layout: Layout) -> SharedStars { +pub fn prepare_shared(stars: &[StarPlan], store: &Store, layout: Layout) -> SharedStars { let groups = partition(stars, layout) .into_iter() .map(|members| { @@ -175,7 +175,7 @@ pub fn rule_sets(group: &Group) -> Vec { } /// The current generation of every side of every group, in table order. -pub fn generations(shared: &SharedStars, store: &Store<'_>) -> Vec> { +pub fn generations(shared: &SharedStars, store: &Store) -> Vec> { shared .groups .iter() @@ -183,7 +183,7 @@ pub fn generations(shared: &SharedStars, store: &Store<'_>) -> Vec> { g.prepared .keys .iter() - .map(|k| store.view(k.rel).1) + .map(|k| store.view(k.rel).generation) .collect() }) .collect() @@ -278,7 +278,7 @@ pub fn evaluate_shared( demand: &Demand, kind: ResultKind, opts: Options, - store: &Store<'_>, + store: &Store, layout: Layout, sharing: bool, ) -> Result<(Outcome, SharedReport), CseRefusal> { diff --git a/crates/crepe-plan-probe/src/lance_store.rs b/crates/crepe-plan-probe/src/lance_store.rs index 1d33253d0..4734da839 100644 --- a/crates/crepe-plan-probe/src/lance_store.rs +++ b/crates/crepe-plan-probe/src/lance_store.rs @@ -25,7 +25,9 @@ use lance_graph_relalg::{Measure, Query, Rel, RelError, run_with_program}; use mork_pathmap_probe::join::for_each_set; use crate::logic::RelId; -use crate::native::{PreparedStar, StarPlan, prepare_from_rows}; +use crate::native::{PreparedStar, StarPlan, prepare_views}; +use arrow_buffer::ScalarBuffer; +use mork_pathmap_probe::{Lane, RelationView}; #[derive(Debug)] pub enum LanceStoreError { @@ -177,11 +179,12 @@ fn u32_col<'b>(b: &'b RecordBatch, name: &str) -> &'b UInt32Array { .expect("u32 column") } -/// One column, every row, with each row's address. +/// One column, every row, with each row's address. Both are Arrow's +/// buffers as the scan returned them, shared by reference count. async fn read_column( ds: &Dataset, col: usize, -) -> Result<(Vec, Vec, Io), LanceStoreError> { +) -> Result<(Lane, ScalarBuffer, Io), LanceStoreError> { let name = format!("c{col}"); let (stats, cb) = stats_sink(); let mut s = ds.scan(); @@ -189,23 +192,24 @@ async fn read_column( s.with_row_address(); s.scan_stats_callback(cb); let b = s.try_into_batch().await.map_err(lance_err)?; - let values = u32_col(&b, &name).values().to_vec(); + let values = u32_col(&b, &name).values().clone(); let addr = b .column_by_name("_rowaddr") .and_then(|c| c.as_any().downcast_ref::()) .expect("_rowaddr") .values() - .to_vec(); + .clone(); let mut io = *stats.lock().expect("stats lock"); io.rows_returned = b.num_rows(); Ok((values, addr, io)) } -/// Both columns, under `mask` when given. +/// Both columns, under `mask` when given, as the relation's two lanes: the +/// scan's own buffers, never zipped into rows. pub(crate) async fn read_rows( ds: &Dataset, mask: Option, -) -> Result<(Vec<[u32; 2]>, Io), LanceStoreError> { +) -> Result<(RelationView, Io), LanceStoreError> { let (stats, cb) = stats_sink(); let mut s = ds.scan(); s.project(&["c0", "c1"]).map_err(lance_err)?; @@ -214,14 +218,14 @@ pub(crate) async fn read_rows( } s.scan_stats_callback(cb); let b = s.try_into_batch().await.map_err(lance_err)?; - let c0 = u32_col(&b, "c0"); - let c1 = u32_col(&b, "c1"); - let rows = (0..b.num_rows()) - .map(|i| [c0.value(i), c1.value(i)]) - .collect(); + let view = RelationView::new( + u32_col(&b, "c0").values().clone(), + u32_col(&b, "c1").values().clone(), + ds.version().version, + ); let mut io = *stats.lock().expect("stats lock"); io.rows_returned = b.num_rows(); - Ok((rows, io)) + Ok((view, io)) } /// `rel` over `values` as lane 0: the relalg description, lowered by @@ -273,7 +277,7 @@ pub async fn prepare_lance( semijoin: bool, ) -> Result<(PreparedStar, Vec), LanceStoreError> { let n_sides = plan.sides.len(); - let mut rows: Vec, u64)>> = vec![None; n_sides]; + let mut rows: Vec> = vec![None; n_sides]; let mut io = vec![SideIo::default(); n_sides]; let restricted = |i: usize| { let k = plan.sides[i].key; @@ -299,20 +303,14 @@ pub async fn prepare_lance( mask_rows: Some(r), rows: rows_io, }; - rows[i] = Some((got, ds.version().version)); + rows[i] = Some(got); } let centers: Option> = if semijoin { let mut acc: Option> = None; for i in (0..n_sides).filter(|&i| restricted(i).is_some()) { let col = plan.sides[i].key.key_col as usize; - let mut c: Vec = rows[i] - .as_ref() - .expect("read") - .0 - .iter() - .map(|r| r[col]) - .collect(); + let mut c: Vec = rows[i].as_ref().expect("read").cols[col].to_vec(); c.sort_unstable(); c.dedup(); acc = Some(match acc { @@ -323,17 +321,8 @@ pub async fn prepare_lance( .collect(), }); } - // A restricted side keeps only the centers every restricted side has, - // as in `native::prepare_semijoin`. Its rows were already read by its - // restriction; this trims them, it reads nothing. - if let Some(c) = &acc { - for i in (0..n_sides).filter(|&i| restricted(i).is_some()) { - let col = plan.sides[i].key.key_col as usize; - if let Some((r, _)) = rows[i].as_mut() { - r.retain(|row| c.binary_search(&row[col]).is_ok()); - } - } - } + // The restricted sides keep only these centers in `prepare_views`, + // by row position; their rows were already read, nothing is trimmed. acc } else { None @@ -342,7 +331,6 @@ pub async fn prepare_lance( for i in (0..n_sides).filter(|&i| restricted(i).is_none()) { let key = plan.sides[i].key; let ds = store.dataset(key.rel).await?; - let version = ds.version().version; match ¢ers { Some(c) => { let (values, addr, col_io) = read_column(&ds, key.key_col as usize).await?; @@ -354,7 +342,7 @@ pub async fn prepare_lance( mask_rows: Some(r), rows: rows_io, }; - rows[i] = Some((got, version)); + rows[i] = Some(got); } None => { let (got, rows_io) = read_rows(&ds, None).await?; @@ -363,14 +351,14 @@ pub async fn prepare_lance( mask_rows: None, rows: rows_io, }; - rows[i] = Some((got, version)); + rows[i] = Some(got); } } } - let owned = rows + let views = rows .into_iter() .map(|r| r.expect("every side read")) .collect(); - Ok((prepare_from_rows(plan, owned), io)) + Ok((prepare_views(plan, views, semijoin), io)) } diff --git a/crates/crepe-plan-probe/src/native.rs b/crates/crepe-plan-probe/src/native.rs index e1568efd6..02d80876b 100644 --- a/crates/crepe-plan-probe/src/native.rs +++ b/crates/crepe-plan-probe/src/native.rs @@ -38,7 +38,10 @@ use lance_graph_mask_risc::{MaskOp, Planes, Pred, Program, Terminal, words_for}; use lance_graph_relalg::{Answer, Measure, Query, Rel, RelError, Run, run_with_program}; use mork_pathmap_probe::RelationView; -use mork_pathmap_probe::join::{JoinDomain, JoinRefusal, JoinTable, Side, for_each_set}; +use mork_pathmap_probe::join::{ + JoinDomain, JoinRefusal, JoinTable, Run as SideRun, SelectedSide, Side, for_each_set, + for_each_set_ahead, +}; use crate::logic::{RelId, VarId}; use crate::mork_api::{Expr, QueryMulti}; @@ -192,20 +195,33 @@ pub fn star(q: &QueryMulti) -> Result { }) } -/// The relations a plan reads, by id, with their generations. -pub struct Store<'a> { - /// `(relation, rows, generation)`. A unary relation's rows are `[v, v]`. - pub relations: Vec<(RelId, &'a [[u32; 2]], u64)>, +/// The relations a plan reads, by id. Each is two lanes and a generation +/// ([`RelationView`]); a unary relation's lanes are equal. +pub struct Store { + pub relations: Vec<(RelId, RelationView)>, } -impl<'a> Store<'a> { - pub fn view(&self, rel: RelId) -> (&'a [[u32; 2]], u64) { - let (_, rows, g) = self +impl Store { + /// Builds each relation's lanes once from authored fixture rows. For + /// tests and examples; a stored relation arrives as lanes. + pub fn from_rows>( + relations: impl IntoIterator, + ) -> Store { + Store { + relations: relations + .into_iter() + .map(|(r, rows, g)| (r, RelationView::from_rows(rows.as_ref(), g))) + .collect(), + } + } + + pub fn view(&self, rel: RelId) -> &RelationView { + &self .relations .iter() - .find(|(r, _, _)| *r == rel) - .expect("relation in store"); - (rows, *g) + .find(|(r, _)| *r == rel) + .expect("relation in store") + .1 } } @@ -223,8 +239,8 @@ pub struct PreparedStar { /// Lays the plan's sides out over one shared domain. A side's restriction is /// applied to the rows it is prepared from. -pub fn prepare(plan: &StarPlan, store: &Store<'_>) -> PreparedStar { - prepare_with(plan, store, false) +pub fn prepare(plan: &StarPlan, store: &Store) -> PreparedStar { + prepare_views(plan, side_views(plan, store), false) } /// [`prepare`] with semijoin reduction: when at least one side is restricted @@ -232,36 +248,47 @@ pub fn prepare(plan: &StarPlan, store: &Store<'_>) -> PreparedStar { /// sides all have. A center outside that set cannot reach the joint mask, so /// the answer and its multiplicities are unchanged; the domain shrinks to the /// reduced centers. Without a restricted side nothing changes. -pub fn prepare_semijoin(plan: &StarPlan, store: &Store<'_>) -> PreparedStar { - prepare_with(plan, store, true) +pub fn prepare_semijoin(plan: &StarPlan, store: &Store) -> PreparedStar { + prepare_views(plan, side_views(plan, store), true) } -fn prepare_with(plan: &StarPlan, store: &Store<'_>, semijoin: bool) -> PreparedStar { - let mut owned: Vec<(Vec<[u32; 2]>, u64)> = plan - .sides +/// Each plan side's relation, as lanes shared with the store (a reference +/// count each, no copy). +fn side_views(plan: &StarPlan, store: &Store) -> Vec { + plan.sides .iter() - .map(|s| { - let (rows, g) = store.view(s.key.rel); - let other = 1 - s.key.key_col as usize; - let kept = rows - .iter() - .filter(|r| s.key.unary || s.key.restrict.is_none_or(|k| u64::from(r[other]) == k)) - .copied() + .map(|s| store.view(s.key.rel).clone()) + .collect() +} + +/// Lays the sides out over one shared domain, in plan side order, from their +/// relations' lanes. A side's restriction, and with `semijoin` the centers +/// every restricted side has, select row positions: the index map leaves the +/// other rows out, and no row is copied. +pub fn prepare_views(plan: &StarPlan, views: Vec, semijoin: bool) -> PreparedStar { + let restricted = |i: usize| { + let k = plan.sides[i].key; + !k.unary && k.restrict.is_some() + }; + let restrict_ok = |i: usize, row: usize| { + let k = plan.sides[i].key; + let other = 1 - k.key_col as usize; + k.unary + || k.restrict + .is_none_or(|c| u64::from(views[i].cols[other][row]) == c) + }; + // Centers every restricted side has: sorted, deduplicated, intersected. + let centers: Option> = if semijoin { + let mut acc: Option> = None; + for i in (0..views.len()).filter(|&i| restricted(i)) { + let key = &views[i].cols[plan.sides[i].key.key_col as usize]; + let mut c: Vec = (0..key.len()) + .filter(|&row| restrict_ok(i, row)) + .map(|row| key[row]) .collect(); - (kept, g) - }) - .collect(); - if semijoin { - // Centers every restricted side has: sorted, deduplicated, intersected. - let mut centers: Option> = None; - for (s, (rows, _)) in plan.sides.iter().zip(&owned) { - if s.key.unary || s.key.restrict.is_none() { - continue; - } - let mut c: Vec = rows.iter().map(|r| r[s.key.key_col as usize]).collect(); c.sort_unstable(); c.dedup(); - centers = Some(match centers { + acc = Some(match acc { None => c, Some(prev) => prev .into_iter() @@ -269,37 +296,30 @@ fn prepare_with(plan: &StarPlan, store: &Store<'_>, semijoin: bool) -> PreparedS .collect(), }); } - if let Some(centers) = centers { - for (s, (rows, _)) in plan.sides.iter().zip(owned.iter_mut()) { - let col = s.key.key_col as usize; - rows.retain(|r| centers.binary_search(&r[col]).is_ok()); - } - } - } - prepare_from_rows(plan, owned) -} - -/// Lays out rows already selected per side (`(rows, generation)`, in plan -/// side order) over one shared domain. -pub fn prepare_from_rows(plan: &StarPlan, owned: Vec<(Vec<[u32; 2]>, u64)>) -> PreparedStar { - let inputs: Vec<(RelationView<'_>, usize)> = owned + acc + } else { + None + }; + let keeps: Vec bool + '_>> = (0..views.len()) + .map(|i| { + let key = &views[i].cols[plan.sides[i].key.key_col as usize]; + let centers = centers.as_deref(); + Box::new(move |row: usize| { + restrict_ok(i, row) && centers.is_none_or(|c| c.binary_search(&key[row]).is_ok()) + }) as Box bool + '_> + }) + .collect(); + let inputs: Vec> = views .iter() .zip(&plan.sides) - .map(|((rows, g), s)| { - ( - RelationView { - rows, - generation: *g, - }, - s.key.key_col as usize, - ) - }) + .zip(&keeps) + .map(|((v, s), keep)| (v, s.key.key_col as usize, &**keep)) .collect(); - let table = JoinTable::prepare_star(&inputs); + let table = JoinTable::prepare_star_where(&inputs); PreparedStar { domain_fp: fingerprint(&table), keys: plan.sides.iter().map(|s| s.key).collect(), - rows_prepared: owned.iter().map(|(r, _)| r.len()).sum(), + rows_prepared: table.sides.iter().map(Side::rows).sum(), table, } } @@ -461,7 +481,7 @@ fn side_count(view: &StarView<'_>, filter: &RuleFilter, slot: usize, i: usize) - let vals = view.sides[i].at(slot); match filter.side[i] { None => vals.len() as u64, - Some(k) => vals.iter().filter(|&&v| u64::from(v) == k).count() as u64, + Some(k) => vals.iter().filter(|&v| u64::from(v) == k).count() as u64, } } @@ -611,7 +631,7 @@ pub fn add_distinct( .filter(|&j| j != i) .all(|j| side_count(view, filter, slot, j) > 0); if others { - for &v in view.sides[i].at(slot) { + for v in view.sides[i].at(slot).iter() { set_bit(values, u64::from(v)); } } @@ -657,7 +677,7 @@ pub struct RuleCoordinates { pub struct SlotView<'a> { pub slot: usize, pub center: u32, - pub sides: &'a [&'a [u32]], + pub sides: &'a [SideRun<'a>], pub filter: &'a RuleFilter, pub derivations: u64, } @@ -665,8 +685,9 @@ pub struct SlotView<'a> { /// Calls `f` for every kept slot with at least one derivation, in slot /// order, with the sides' rows borrowed in place. pub fn for_each_slot(view: &StarView<'_>, rc: &RuleCoordinates, mut f: impl FnMut(&SlotView<'_>)) { - let mut sides: Vec<&[u32]> = vec![&[]; view.sides.len()]; - for_each_set(&rc.kept, view.domain.len(), |slot| { + let mut sides: Vec> = vec![SideRun::default(); view.sides.len()]; + let ahead = |next: usize| view.sides.iter().for_each(|s| s.at(next).prefetch()); + for_each_set_ahead(&rc.kept, view.domain.len(), ahead, |slot| { let derivations = slot_derivations(view, &rc.filter, slot); if derivations == 0 { return; @@ -724,7 +745,7 @@ pub fn matching( let vals = view.sides[i].at(slot); match want[i] { None => vals.len() as u64, - Some(t) => vals.iter().filter(|&&v| u64::from(v) == t).count() as u64, + Some(t) => vals.iter().filter(|&v| u64::from(v) == t).count() as u64, } }) .product() @@ -750,7 +771,7 @@ pub fn matching( pub struct DerivationView<'a> { pub center: u32, head: &'a [HeadTerm], - sides: &'a [&'a [u32]], + sides: &'a [SideRun<'a>], at: &'a [usize], } @@ -763,7 +784,7 @@ impl DerivationView<'_> { match self.head[c] { HeadTerm::Const(k) => k, HeadTerm::Center => u64::from(self.center), - HeadTerm::Side(i) => u64::from(self.sides[i][self.at[i]]), + HeadTerm::Side(i) => u64::from(self.sides[i].get(self.at[i])), } } } @@ -781,9 +802,10 @@ pub fn for_each_derivation( .filter(|&i| view.star.sides[i].emit.is_some()) .collect(); let mut at = vec![0usize; k]; - let mut sides: Vec<&[u32]> = vec![&[]; k]; + let mut sides: Vec> = vec![SideRun::default(); k]; let keeps = |i: usize, v: u32| rc.filter.side[i].is_none_or(|t| t == u64::from(v)); - for_each_set(&rc.kept, view.domain.len(), |slot| { + let ahead = |next: usize| view.sides.iter().for_each(|s| s.at(next).prefetch()); + for_each_set_ahead(&rc.kept, view.domain.len(), ahead, |slot| { let center = view.domain.id(slot); if rc.filter.center.is_some_and(|t| t != u64::from(center)) { return; @@ -793,7 +815,7 @@ pub fn for_each_derivation( } let weight: u64 = (0..k) .filter(|i| !emitting.contains(i)) - .map(|i| sides[i].iter().filter(|&&v| keeps(i, v)).count() as u64) + .map(|i| sides[i].iter().filter(|&v| keeps(i, v)).count() as u64) .product(); if weight == 0 || emitting.iter().any(|&i| sides[i].is_empty()) { return; @@ -802,7 +824,7 @@ pub fn for_each_derivation( at[i] = 0; } loop { - if emitting.iter().all(|&i| keeps(i, sides[i][at[i]])) { + if emitting.iter().all(|&i| keeps(i, sides[i].get(at[i]))) { f( &DerivationView { center, diff --git a/crates/crepe-plan-probe/src/pipeline.rs b/crates/crepe-plan-probe/src/pipeline.rs index e73ac4c82..841e5c71f 100644 --- a/crates/crepe-plan-probe/src/pipeline.rs +++ b/crates/crepe-plan-probe/src/pipeline.rs @@ -227,7 +227,7 @@ pub fn evaluate( demand: &Demand, kind: ResultKind, opts: Options, - store: &Store<'_>, + store: &Store, ) -> Result<(Outcome, Report), PipelineRefusal> { let planned = plan(rules, demand, kind, opts)?; let t = Instant::now(); @@ -250,7 +250,7 @@ pub fn evaluate( .map(|s| { s.sides .iter() - .map(|side| store.view(side.key.rel).1) + .map(|side| store.view(side.key.rel).generation) .collect() }) .collect(); diff --git a/crates/crepe-plan-probe/tests/pipeline.rs b/crates/crepe-plan-probe/tests/pipeline.rs index ea5d9f3de..97c190db2 100644 --- a/crates/crepe-plan-probe/tests/pipeline.rs +++ b/crates/crepe-plan-probe/tests/pipeline.rs @@ -106,7 +106,7 @@ fn evaluate( demand: &Demand, kind: ResultKind, opts: Options, - store: &Store<'_>, + store: &Store, ) -> Result<(Norm, Report), PipelineRefusal> { crepe_plan_probe::pipeline::evaluate(rules, demand, kind, opts, store) .map(|(o, rep)| (norm(&o, &rep.tables), rep)) @@ -213,25 +213,23 @@ fn resolved(plan: &Plan) -> (Labels, Vec) { (labels, rules) } -fn store<'a>(labels: &Labels, f: &'a Family, gens: [u64; 3]) -> Store<'a> { - Store { - relations: vec![ - (labels.id("Parent").unwrap(), &f.parent, gens[0]), - (labels.id("Allowed").unwrap(), &f.allowed, gens[1]), - (labels.id("Sponsor").unwrap(), &f.sponsor, gens[2]), - ], - } +fn store(labels: &Labels, f: &Family, gens: [u64; 3]) -> Store { + Store::from_rows(vec![ + (labels.id("Parent").unwrap(), &f.parent, gens[0]), + (labels.id("Allowed").unwrap(), &f.allowed, gens[1]), + (labels.id("Sponsor").unwrap(), &f.sponsor, gens[2]), + ]) } /// Raw rows of a relation at its declared arity, for the reference. -fn raw<'a>(labels: &'a Labels, store: &'a Store<'a>) -> impl Fn(RelId) -> Vec> + 'a { +fn raw<'a>(labels: &'a Labels, store: &'a Store) -> impl Fn(RelId) -> Vec> + 'a { move |rel| { - let (rows, _) = store.view(rel); - rows.iter() - .map(|r| { - r[..labels.arity(rel)] + let v = store.view(rel); + (0..v.len()) + .map(|i| { + v.cols[..labels.arity(rel)] .iter() - .map(|&v| u64::from(v)) + .map(|c| u64::from(c[i])) .collect() }) .collect() @@ -248,7 +246,7 @@ fn answer_demand(labels: &Labels, x: u64) -> Demand { fn reference_bag( labels: &Labels, rules: &[Rule], - store: &Store<'_>, + store: &Store, demand: &Demand, ) -> BTreeMap, u64> { reference::eval(rules, demand.goal, &demand.bound, &raw(labels, store)) @@ -466,13 +464,7 @@ fn d_a_side_from_another_domain_is_not_anded() { // Same schema, other ordinal domain: Parent keyed by column 0 over a // domain built from a different relation. let other_rows: Vec<[u32; 2]> = (0..50).map(|i| [i * 7, i]).collect(); - let other = JoinTable::prepare_star(&[( - RelationView { - rows: &other_rows, - generation: 1, - }, - 0, - )]); + let other = JoinTable::prepare_star(&[(&RelationView::from_rows(&other_rows, 1), 0)]); let foreign = other.sides[0].clone(); let key = prep.keys[1]; assert_eq!( @@ -520,14 +512,9 @@ fn e_a_stale_generation_and_a_value_outside_the_domain_are_refused() { ); // A reseal of Allowed that names a y the domain never had. let resealed: Vec<[u32; 2]> = vec![[1, 1], [999_999, 999_999]]; - let r = prep.table.replace_side( - 2, - RelationView { - rows: &resealed, - generation: 2, - }, - 0, - ); + let r = prep + .table + .replace_side(2, &RelationView::from_rows(&resealed, 2), 0); assert_eq!( r, Err(JoinRefusal::OutsideDomain { @@ -619,9 +606,7 @@ fn g_the_pipeline_refuses_a_chain() { .collect(); let e = labels.id("E").unwrap(); let rows: Vec<[u32; 2]> = vec![[1, 2], [2, 3], [3, 4]]; - let st = Store { - relations: vec![(e, &rows, 1), (labels.id("B").unwrap(), &rows, 1)], - }; + let st = Store::from_rows(vec![(e, &rows, 1), (labels.id("B").unwrap(), &rows, 1)]); let d = Demand { goal: labels.id("Chain").unwrap(), bound: vec![None, None], @@ -682,13 +667,11 @@ fn i_the_star_mask_is_the_intersection_of_all_three_planes() { let r: Vec<[u32; 2]> = (0..300).map(|v| [v, v]).collect(); let s: Vec<[u32; 2]> = (0..300).filter(|v| v % 2 == 0).map(|v| [v, v]).collect(); let q: Vec<[u32; 2]> = (0..300).filter(|v| v % 3 == 0).map(|v| [v, v]).collect(); - let st = Store { - relations: vec![ - (labels.id("R").unwrap(), &r, 1), - (labels.id("S").unwrap(), &s, 1), - (labels.id("Q").unwrap(), &q, 1), - ], - }; + let st = Store::from_rows(vec![ + (labels.id("R").unwrap(), &r, 1), + (labels.id("S").unwrap(), &s, 1), + (labels.id("Q").unwrap(), &q, 1), + ]); let sp = star(&query_multi(&rules[0])).unwrap(); assert_eq!(sp.sides.len(), 3); let prep = prepare(&sp, &st); @@ -709,12 +692,10 @@ fn j_two_way_join_equals_mork_and_the_reference() { let (labels, rules) = resolved(&plan); let r: Vec<[u32; 2]> = (0..120u32).map(|i| [i, i % 17]).collect(); let s: Vec<[u32; 2]> = (0..90u32).map(|i| [i % 23, 500 + i]).collect(); - let st = Store { - relations: vec![ - (labels.id("R").unwrap(), &r, 1), - (labels.id("S").unwrap(), &s, 1), - ], - }; + let st = Store::from_rows(vec![ + (labels.id("R").unwrap(), &r, 1), + (labels.id("S").unwrap(), &s, 1), + ]); let d = Demand { goal: labels.id("T").unwrap(), bound: vec![None, None, None], @@ -740,14 +721,8 @@ fn j_two_way_join_equals_mork_and_the_reference() { assert_eq!(bag, want_bag.clone().into_iter().collect::>()); let input = JoinInput { - r: RelationView { - rows: &r, - generation: 1, - }, - s: RelationView { - rows: &s, - generation: 1, - }, + r: RelationView::from_rows(&r, 1), + s: RelationView::from_rows(&s, 1), }; let mut mork = bind(Backend::Mork, &input, SET).expect("MORK builds"); // MORK answers the set: exactly the bag's distinct rows, each answered. @@ -772,12 +747,10 @@ fn j2_a_post_filter_on_the_center_narrows_the_kept_slots() { let (labels, rules) = resolved(&plan); let r: Vec<[u32; 2]> = (0..120u32).map(|i| [i, i % 17]).collect(); let s: Vec<[u32; 2]> = (0..90u32).map(|i| [i % 23, 500 + i]).collect(); - let st = Store { - relations: vec![ - (labels.id("R").unwrap(), &r, 1), - (labels.id("S").unwrap(), &s, 1), - ], - }; + let st = Store::from_rows(vec![ + (labels.id("R").unwrap(), &r, 1), + (labels.id("S").unwrap(), &s, 1), + ]); let goal = labels.id("T").unwrap(); let no_push = Options { propagate: false, @@ -1034,12 +1007,10 @@ fn n_semijoin_intersects_two_restricted_sides() { r.push([100 + i, 200 + i]); s.push([200 + i, 300 + i]); } - let st = Store { - relations: vec![ - (labels.id("R").unwrap(), &r, 1), - (labels.id("S").unwrap(), &s, 1), - ], - }; + let st = Store::from_rows(vec![ + (labels.id("R").unwrap(), &r, 1), + (labels.id("S").unwrap(), &s, 1), + ]); let d = Demand { goal: labels.id("T").unwrap(), bound: vec![Some(1), None, Some(9)], @@ -1185,12 +1156,10 @@ mod stage3 { ) .await .unwrap(); - let mem = Store { - relations: vec![ - (labels.id("R").unwrap(), &r, 1), - (labels.id("S").unwrap(), &s, 1), - ], - }; + let mem = Store::from_rows(vec![ + (labels.id("R").unwrap(), &r, 1), + (labels.id("S").unwrap(), &s, 1), + ]); let d = Demand { goal: labels.id("T").unwrap(), bound: vec![Some(1), None, Some(9)], diff --git a/crates/mork-pathmap-probe/src/join.rs b/crates/mork-pathmap-probe/src/join.rs index 4da68495f..ed8c4f1c8 100644 --- a/crates/mork-pathmap-probe/src/join.rs +++ b/crates/mork-pathmap-probe/src/join.rs @@ -123,7 +123,7 @@ pub struct Side { /// The rows of one side at one slot, read in place: `lane[pos[k]]`. A /// borrowed view; it holds positions and the lane, never values. -#[derive(Clone, Copy, Debug)] +#[derive(Clone, Copy, Debug, Default)] pub struct Run<'a> { lane: &'a [u32], pos: &'a [u32], @@ -236,6 +236,11 @@ impl Side { } } + /// Rows the index map holds, over all slots. + pub fn rows(&self) -> usize { + self.pos.len() + } + /// Rows at slot `y`, from the offsets alone (no lane read). pub fn count(&self, y: usize) -> usize { (self.off[y + 1] - self.off[y]) as usize From 1fac47fedbe84b7dcec7c18403a3046662774dbf Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 21 Oct 2026 11:08:42 +0000 Subject: [PATCH 05/14] join: the domain is a presence mask with ranks; a restriction is a lane compare into a mask JoinDomain lays a dense value set (span <= 8x count) out as a presence mask plus a running count per word: slot = rank + popcount, in value order, with no sort and no search; sparse sets keep the sorted list. A side's keep is a mask over row positions, and the index map walks its set bits. In crepe-plan-probe a restriction is one eq_u32_to_mask over the lane, and the semijoin centers are a domain probed per row, not a binary search. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01G22yT6htkcdyXsihxxXdrg --- crates/crepe-plan-probe/Cargo.toml | 8 +- crates/crepe-plan-probe/src/native.rs | 72 +++++---- crates/mork-pathmap-probe/src/join.rs | 201 +++++++++++++++++++----- crates/mork-pathmap-probe/tests/join.rs | 49 +++++- 4 files changed, 259 insertions(+), 71 deletions(-) diff --git a/crates/crepe-plan-probe/Cargo.toml b/crates/crepe-plan-probe/Cargo.toml index 73e748886..cd4e00072 100644 --- a/crates/crepe-plan-probe/Cargo.toml +++ b/crates/crepe-plan-probe/Cargo.toml @@ -33,13 +33,13 @@ arrow-array = "58" arrow-buffer = "58" arrow-schema = "58" futures = "0.3" +# The SIMD facade: a side's restriction is one compare over its lane into a +# mask (`eq_u32_to_mask`). Same coordinate as mask-risc and +# mork-pathmap-probe, so the graph keeps one ndarray package. +ndarray = { git = "https://github.com/AdaWorldAPI/ndarray", branch = "master", default-features = false, features = ["std"] } tokio = { version = "1.37", features = ["macros", "rt-multi-thread"] } [dev-dependencies] -# The SIMD facade, for the row-filter comparison in examples/row_filter.rs. -# Same coordinate as mask-risc and mork-pathmap-probe, so the graph keeps one -# ndarray package. -ndarray = { git = "https://github.com/AdaWorldAPI/ndarray", branch = "master", default-features = false, features = ["std"] } tempfile = "3" datafrog = { git = "https://github.com/AdaWorldAPI/datafrog", branch = "master" } # ascent, as a second oracle and baseline for examples/walk_mask.rs. diff --git a/crates/crepe-plan-probe/src/native.rs b/crates/crepe-plan-probe/src/native.rs index 02d80876b..6f31bd978 100644 --- a/crates/crepe-plan-probe/src/native.rs +++ b/crates/crepe-plan-probe/src/native.rs @@ -40,8 +40,9 @@ use lance_graph_relalg::{Answer, Measure, Query, Rel, RelError, Run, run_with_pr use mork_pathmap_probe::RelationView; use mork_pathmap_probe::join::{ JoinDomain, JoinRefusal, JoinTable, Run as SideRun, SelectedSide, Side, for_each_set, - for_each_set_ahead, + for_each_set_ahead, rows, }; +use ndarray::simd::eq_u32_to_mask; use crate::logic::{RelId, VarId}; use crate::mork_api::{Expr, QueryMulti}; @@ -270,50 +271,65 @@ pub fn prepare_views(plan: &StarPlan, views: Vec, semijoin: bool) let k = plan.sides[i].key; !k.unary && k.restrict.is_some() }; - let restrict_ok = |i: usize, row: usize| { - let k = plan.sides[i].key; - let other = 1 - k.key_col as usize; - k.unary - || k.restrict - .is_none_or(|c| u64::from(views[i].cols[other][row]) == c) - }; - // Centers every restricted side has: sorted, deduplicated, intersected. - let centers: Option> = if semijoin { - let mut acc: Option> = None; - for i in (0..views.len()).filter(|&i| restricted(i)) { + // A restricted side's rows: one compare over the lane holding the + // constant, into a mask. + let restriction: Vec>> = (0..views.len()) + .map(|i| { + let k = plan.sides[i].key; + let c = k.restrict.filter(|_| restricted(i))?; + let lane = &views[i].cols[1 - k.key_col as usize]; + let mut m = vec![0u64; words_for(lane.len())]; + if let Ok(c) = u32::try_from(c) { + eq_u32_to_mask(lane, c, &mut m); + } + Some(m) + }) + .collect(); + // Centers every restricted side has: one domain per restricted side over + // its kept rows, intersected. + let centers: Option = if semijoin { + let mut acc: Option = None; + for (i, m) in restriction.iter().enumerate() { + let Some(m) = m else { continue }; let key = &views[i].cols[plan.sides[i].key.key_col as usize]; - let mut c: Vec = (0..key.len()) - .filter(|&row| restrict_ok(i, row)) - .map(|row| key[row]) - .collect(); - c.sort_unstable(); - c.dedup(); + let c = JoinDomain::build(rows(key.len(), Some(m)).map(|row| key[row])); acc = Some(match acc { None => c, - Some(prev) => prev - .into_iter() - .filter(|v| c.binary_search(v).is_ok()) - .collect(), + Some(prev) => { + let both: Vec = (0..prev.len()) + .map(|s| prev.id(s)) + .filter(|&v| c.contains(v)) + .collect(); + JoinDomain::build(both.iter().copied()) + } }); } acc } else { None }; - let keeps: Vec bool + '_>> = (0..views.len()) + // Each side's keep mask: its restriction, and with centers the rows whose + // center every restricted side has. `None`: every row. + let keeps: Vec>> = (0..views.len()) .map(|i| { + let Some(c) = ¢ers else { + return restriction[i].clone(); + }; let key = &views[i].cols[plan.sides[i].key.key_col as usize]; - let centers = centers.as_deref(); - Box::new(move |row: usize| { - restrict_ok(i, row) && centers.is_none_or(|c| c.binary_search(&key[row]).is_ok()) - }) as Box bool + '_> + let mut m = vec![0u64; words_for(key.len())]; + for row in rows(key.len(), restriction[i].as_deref()) { + if c.contains(key[row]) { + m[row >> 6] |= 1 << (row & 63); + } + } + Some(m) }) .collect(); let inputs: Vec> = views .iter() .zip(&plan.sides) .zip(&keeps) - .map(|((v, s), keep)| (v, s.key.key_col as usize, &**keep)) + .map(|((v, s), keep)| (v, s.key.key_col as usize, keep.as_deref())) .collect(); let table = JoinTable::prepare_star_where(&inputs); PreparedStar { diff --git a/crates/mork-pathmap-probe/src/join.rs b/crates/mork-pathmap-probe/src/join.rs index ed8c4f1c8..13aba51d2 100644 --- a/crates/mork-pathmap-probe/src/join.rs +++ b/crates/mork-pathmap-probe/src/join.rs @@ -68,18 +68,97 @@ pub enum JoinRefusal { } /// The shared coordinate space: sorted distinct `y` values. +/// +/// When the values span a range no wider than [`DENSE_SPAN`] times their +/// count, the space is a presence mask over that range plus a running count +/// per mask word: a value's slot is the word's count plus the popcount of the +/// bits below it. No sort, no search, and slots stay in value order. A +/// sparser set falls back to a sorted list and binary search. #[derive(Clone, Debug)] pub struct JoinDomain { ids: Vec, + dense: Option, +} + +/// A value range at most this many times wider than the values in it is laid +/// out as a mask. +pub const DENSE_SPAN: u64 = 8; + +#[derive(Clone, Debug)] +struct Dense { + base: u32, + span: u64, + words: Vec, + rank: Vec, +} + +impl Dense { + fn slot_of(&self, y: u32) -> Option { + let d = u64::from(y.wrapping_sub(self.base)); + if y < self.base || d >= self.span { + return None; + } + let (w, b) = ((d >> 6) as usize, d & 63); + let word = self.words[w]; + if word >> b & 1 == 0 { + return None; + } + Some(self.rank[w] as usize + (word & ((1u64 << b) - 1)).count_ones() as usize) + } } impl JoinDomain { - /// The union of every side's `y` values. - pub fn build(ys: impl IntoIterator) -> Self { - let mut ids: Vec = ys.into_iter().collect(); - ids.sort_unstable(); - ids.dedup(); - JoinDomain { ids } + /// The union of every side's `y` values. The iterator is walked twice: + /// once for the range, once for the mask. + pub fn build(ys: I) -> Self + where + I: IntoIterator, + I::IntoIter: Clone, + { + let it = ys.into_iter(); + let (mut lo, mut hi, mut n) = (u32::MAX, 0u32, 0u64); + for y in it.clone() { + lo = lo.min(y); + hi = hi.max(y); + n += 1; + } + if n == 0 { + return JoinDomain { + ids: Vec::new(), + dense: None, + }; + } + let span = u64::from(hi - lo) + 1; + if span > DENSE_SPAN * n.max(64) { + let mut ids: Vec = it.collect(); + ids.sort_unstable(); + ids.dedup(); + return JoinDomain { ids, dense: None }; + } + let mut words = vec![0u64; span.div_ceil(64) as usize]; + for y in it { + let d = u64::from(y - lo); + words[(d >> 6) as usize] |= 1 << (d & 63); + } + let mut rank = Vec::with_capacity(words.len()); + let mut ids = Vec::new(); + for (w, &word) in words.iter().enumerate() { + rank.push(ids.len() as u32); + let mut m = word; + while m != 0 { + ids.push(lo + (w as u32) * 64 + m.trailing_zeros()); + m &= m - 1; + } + } + JoinDomain { + ids, + dense: Some(Dense { + base: lo, + span, + words, + rank, + }), + } } pub fn len(&self) -> usize { @@ -92,7 +171,20 @@ impl JoinDomain { /// Global `y` to slot. Preparation only. pub fn slot_of(&self, y: u32) -> Option { - self.ids.binary_search(&y).ok() + match &self.dense { + Some(d) => d.slot_of(y), + None => self.ids.binary_search(&y).ok(), + } + } + + /// Whether the space is laid out as a mask over its value range. + pub fn is_dense(&self) -> bool { + self.dense.is_some() + } + + /// Whether `y` is in the space. + pub fn contains(&self, y: u32) -> bool { + self.slot_of(y).is_some() } /// Slot to global `y`, for emitting answers. @@ -101,7 +193,7 @@ impl JoinDomain { } pub fn bytes(&self) -> usize { - self.ids.len() * 4 + self.ids.len() * 4 + self.dense.as_ref().map_or(0, |d| d.words.len() * 12) } } @@ -172,44 +264,40 @@ impl Side { key: usize, side: usize, ) -> Result { - Self::prepare_where(domain, view, key, side, |_| true) + Self::prepare_where(domain, view, key, side, None) } - /// [`Self::prepare`] over the rows `keep` admits, by row position. A - /// restriction is a selection of positions, never a copy of rows: the - /// index map simply leaves the other positions out. + /// [`Self::prepare`] over the rows the `keep` mask (one bit per row + /// position; `None`: every row) admits. A restriction is a selection of + /// positions, never a copy of rows: the index map walks the kept bits and + /// leaves the other positions out. pub fn prepare_where( domain: &JoinDomain, view: &RelationView, key: usize, side: usize, - keep: impl Fn(usize) -> bool, + keep: Option<&[u64]>, ) -> Result { let n = domain.len(); let keys = &view.cols[key]; let mut off = vec![0u32; n + 1]; - let mut kept = 0usize; - for (row, &y) in keys.iter().enumerate() { - if !keep(row) { - continue; - } + // Each kept row's slot, once (scratch), in kept-row order. + let mut slots = Vec::new(); + for row in rows(keys.len(), keep) { + let y = keys[row]; let s = domain .slot_of(y) .ok_or(JoinRefusal::OutsideDomain { side, y })?; + slots.push(s as u32); off[s + 1] += 1; - kept += 1; } for i in 0..n { off[i + 1] += off[i]; } let mut fill = off.clone(); - let mut pos = vec![0u32; kept]; - for (row, &y) in keys.iter().enumerate() { - if !keep(row) { - continue; - } - let s = domain.slot_of(y).expect("checked above"); - let at = &mut fill[s]; + let mut pos = vec![0u32; slots.len()]; + for (row, &s) in rows(keys.len(), keep).zip(&slots) { + let at = &mut fill[s as usize]; pos[*at as usize] = row as u32; *at += 1; } @@ -283,8 +371,55 @@ pub struct JoinTable { } /// One input of [`JoinTable::prepare_star_where`]: the relation, the lane -/// holding the shared variable, and which row positions to keep. -pub type SelectedSide<'a> = (&'a RelationView, usize, &'a dyn Fn(usize) -> bool); +/// holding the shared variable, and which row positions to keep: a mask, one +/// bit per row (`None`: every row). +pub type SelectedSide<'a> = (&'a RelationView, usize, Option<&'a [u64]>); + +/// The row positions `keep` admits, in row order: every row, or the set bits. +#[derive(Clone, Debug)] +pub enum Rows<'a> { + All(std::ops::Range), + Set { + words: &'a [u64], + n: usize, + wi: usize, + w: u64, + }, +} + +/// The kept row positions of a relation with `n` rows. +pub fn rows(n: usize, keep: Option<&[u64]>) -> Rows<'_> { + match keep { + None => Rows::All(0..n), + Some(words) => Rows::Set { + words, + n, + wi: 0, + w: words.first().copied().unwrap_or(0), + }, + } +} + +impl Iterator for Rows<'_> { + type Item = usize; + fn next(&mut self) -> Option { + match self { + Rows::All(r) => r.next(), + Rows::Set { words, n, wi, w } => loop { + if *w != 0 { + let row = *wi * 64 + w.trailing_zeros() as usize; + *w &= *w - 1; + if row < *n { + return Some(row); + } + continue; + } + *wi += 1; + *w = *words.get(*wi)?; + }, + } + } +} /// Where the shared variable sits in each input row: `R(x, y)` keys column 1, /// `S(y, z)` keys column 0. @@ -300,8 +435,7 @@ impl JoinTable { /// Prepares a star of relations that all share `y`: every relation is laid /// out over the same domain. `key` names the column holding `y`. pub fn prepare_star(inputs: &[(&RelationView, usize)]) -> JoinTable { - let all = |_: usize| true; - let with: Vec> = inputs.iter().map(|&(v, k)| (v, k, &all as _)).collect(); + let with: Vec> = inputs.iter().map(|&(v, k)| (v, k, None)).collect(); Self::prepare_star_where(&with) } @@ -309,11 +443,8 @@ impl JoinTable { /// position). The domain is the union of the kept rows' keys only. pub fn prepare_star_where(inputs: &[SelectedSide<'_>]) -> JoinTable { let domain = JoinDomain::build(inputs.iter().flat_map(|&(v, key, keep)| { - v.cols[key] - .iter() - .enumerate() - .filter(move |&(row, _)| keep(row)) - .map(|(_, &y)| y) + let lane = &v.cols[key]; + rows(lane.len(), keep).map(move |row| lane[row]) })); let sides = inputs .iter() diff --git a/crates/mork-pathmap-probe/tests/join.rs b/crates/mork-pathmap-probe/tests/join.rs index bc49c211a..e413cacf7 100644 --- a/crates/mork-pathmap-probe/tests/join.rs +++ b/crates/mork-pathmap-probe/tests/join.rs @@ -1,7 +1,7 @@ //! The versioned relationship table (`join`) against nested-loop oracles. use mork_pathmap_probe::join::{ - JoinExec, JoinRefusal, JoinTable, R_KEY, S_KEY, filter_plane, for_each_set, + JoinDomain, JoinExec, JoinRefusal, JoinTable, R_KEY, S_KEY, filter_plane, for_each_set, }; use mork_pathmap_probe::{JoinInput, RelationView, reference}; @@ -300,9 +300,17 @@ fn a_position_selection_equals_preparing_the_kept_rows() { let s = swap(&hetero(300, 12, &all, 2)); let (rv, sv) = (view(&r, 1), view(&s, 1)); // Keep R rows whose x is even, and S rows at even positions. - let keep_r = |row: usize| rv.cols[0][row].is_multiple_of(2); - let keep_s = |row: usize| row.is_multiple_of(2); - let selected = JoinTable::prepare_star_where(&[(&rv, R_KEY, &keep_r), (&sv, S_KEY, &keep_s)]); + let mask = |n: usize, keep: &dyn Fn(usize) -> bool| { + let mut m = vec![0u64; n.div_ceil(64)]; + for row in (0..n).filter(|&r| keep(r)) { + m[row >> 6] |= 1 << (row & 63); + } + m + }; + let keep_r = mask(rv.len(), &|row| rv.cols[0][row].is_multiple_of(2)); + let keep_s = mask(sv.len(), &|row| row.is_multiple_of(2)); + let selected = + JoinTable::prepare_star_where(&[(&rv, R_KEY, Some(&keep_r)), (&sv, S_KEY, Some(&keep_s))]); let r2: Vec<[u32; 2]> = r .iter() .copied() @@ -324,3 +332,36 @@ fn a_position_selection_equals_preparing_the_kept_rows() { "the selection excludes something" ); } + +/// FAILS IF: the mask layout of a dense domain gives any value another slot +/// (or membership) than the sorted list does, around word edges and both ends +/// of the range, or a sparse set is laid out as a mask. +#[test] +fn a_dense_domain_assigns_the_slots_a_sorted_list_does() { + // 300 values in a span of ~1200: dense. Gaps and runs cross word edges. + let vals: Vec = (0..300u32) + .map(|i| 5_000 + i * 4 - (i % 3)) + .chain([5_000 + 63, 5_000 + 64, 5_000 + 127, 5_000 + 128]) + .collect(); + let dense = JoinDomain::build(vals.iter().copied()); + assert!(dense.is_dense(), "fixture is dense"); + let mut sorted = vals.clone(); + sorted.sort_unstable(); + sorted.dedup(); + assert_eq!(dense.len(), sorted.len()); + let (lo, hi) = (sorted[0], *sorted.last().unwrap()); + let mut members = 0; + for y in lo - 70..=hi + 70 { + assert_eq!(dense.slot_of(y), sorted.binary_search(&y).ok(), "y = {y}"); + members += usize::from(dense.contains(y)); + } + assert_eq!(members, sorted.len()); + for (slot, &y) in sorted.iter().enumerate() { + assert_eq!(dense.id(slot), y); + } + // Two values far apart: a sorted list, same answers. + let sparse = JoinDomain::build([0, u32::MAX, 7]); + assert!(!sparse.is_dense()); + assert_eq!(sparse.slot_of(u32::MAX), Some(2)); + assert_eq!(sparse.slot_of(6), None); +} From 77c8cd1d621cf127f418ce034d963aa596a296b9 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 21 Oct 2026 11:11:05 +0000 Subject: [PATCH 06/14] mork-pathmap-probe: test the kept-row iterator at word edges and past the end Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01G22yT6htkcdyXsihxxXdrg --- crates/mork-pathmap-probe/tests/join.rs | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/crates/mork-pathmap-probe/tests/join.rs b/crates/mork-pathmap-probe/tests/join.rs index e413cacf7..1fcc5c264 100644 --- a/crates/mork-pathmap-probe/tests/join.rs +++ b/crates/mork-pathmap-probe/tests/join.rs @@ -1,7 +1,7 @@ //! The versioned relationship table (`join`) against nested-loop oracles. use mork_pathmap_probe::join::{ - JoinDomain, JoinExec, JoinRefusal, JoinTable, R_KEY, S_KEY, filter_plane, for_each_set, + JoinDomain, JoinExec, JoinRefusal, JoinTable, R_KEY, S_KEY, filter_plane, for_each_set, rows, }; use mork_pathmap_probe::{JoinInput, RelationView, reference}; @@ -365,3 +365,19 @@ fn a_dense_domain_assigns_the_slots_a_sorted_list_does() { assert_eq!(sparse.slot_of(u32::MAX), Some(2)); assert_eq!(sparse.slot_of(6), None); } + +/// FAILS IF: the kept-row iterator drops a set bit at a word edge or the last +/// row, yields a bit past the relation's end, or `None` is not every row. +#[test] +fn kept_rows_are_exactly_the_set_bits_below_n() { + let n = 130; + let want = [0usize, 1, 63, 64, 127, 128, 129]; + let mut m = vec![0u64; 3]; + for &r in &want { + m[r >> 6] |= 1 << (r & 63); + } + m[2] |= 1 << 10; // row 138: past the end + assert_eq!(rows(n, Some(&m)).collect::>(), want); + assert_eq!(rows(5, None).collect::>(), [0, 1, 2, 3, 4]); + assert_eq!(rows(n, Some(&[0, 0, 0])).count(), 0); +} From 392f7b21c7414b1e271ce1efe1e63fc807b112e8 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 21 Oct 2026 11:12:15 +0000 Subject: [PATCH 07/14] board: zero-copy join index map entry (one gate open: no-pruning arms at n=50k) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01G22yT6htkcdyXsihxxXdrg --- .../2026-10-21-zero-copy-join-index-map.md | 19 +++++++++++++++++++ .claude/board/entries/README.md | 3 ++- 2 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 .claude/board/entries/2026-10-21-zero-copy-join-index-map.md diff --git a/.claude/board/entries/2026-10-21-zero-copy-join-index-map.md b/.claude/board/entries/2026-10-21-zero-copy-join-index-map.md new file mode 100644 index 000000000..d172cce32 --- /dev/null +++ b/.claude/board/entries/2026-10-21-zero-copy-join-index-map.md @@ -0,0 +1,19 @@ +# 2026-10-21 — Zero-copy joins: the index map `$i` over lanes + +**Status:** OPEN (one failed gate, below). Lab only (mork-pathmap-probe, crepe-plan-probe). + +## What changed + +- A relation is two lanes (`ScalarBuffer`, shared with Lance's column buffers, no copy). A join side is an index map only: presence plane, per-slot offsets, and row positions into the lanes. Values are read in place (`lane[pos[k]]`), with a one-slot prefetch lookahead. +- Answers are folds: counts read offsets, a pinned pattern inverts to one slot, enumeration reads `JoinRow` in place. The upstream MORK/PathMap row sets are only outside comparators. +- The domain (global id → local slot) is a presence mask with a running count per word when the ids are dense (slot = rank + popcount, value order kept); a sparse id set keeps the sorted list. +- A restriction is one `eq_u32_to_mask` over the lane; semijoin centers are a domain probed per row. The index map walks the kept bits. Nothing trims or copies rows. + +## Measured (crepe-plan-probe `examples/measure.rs`, release, loaded 4-core host) + +- With pushdown and semijoin (36 rows prepared), prepare is µs at every n up to 50 000; mask and terminal stay ns / low µs. Parity ok against every oracle. + +## Failed gate (OPEN) + +- The ablation arms without pruning (`no-semijoin`, `none`) at n = 50 000 index 56 k and 119 k rows and leave the µs budget. Cause: building `$i` from unordered lanes is a counting sort whose count and scatter passes cost about 10 ns per row here. A rank-free dense counting sort was tried and did not help (bound by the random writes). +- Candidate fix, not decided: lanes sealed in key order at write time (the `OrderedLaneWitness` path `addressed.rs` already attests), so `$i` is run offsets and the scatter disappears. A relation keyed by two columns needs two orders or keeps a permutation. Write-path decision; the operator chooses. diff --git a/.claude/board/entries/README.md b/.claude/board/entries/README.md index 795e9cffe..bffdc03cb 100644 --- a/.claude/board/entries/README.md +++ b/.claude/board/entries/README.md @@ -25,10 +25,11 @@ index row, (3) no duplicate entry id. Checks 1 and 2 are deliberately opposite directions; the stranding this convention prevents shows up in exactly one of them, never both. -410 entries, 2026-08-06 .. 2026-10-21. +411 entries, 2026-08-06 .. 2026-10-21. | date | entry id | finding | file | |---|---|---|---| +| 2026-10-21 | `zero-copy-join-index-map` | | [2026-10-21-zero-copy-join-index-map.md](2026-10-21-zero-copy-join-index-map.md) | | 2026-10-21 | `D-RU-4` | | [2026-10-21-u4-coaddressed-join.md](2026-10-21-u4-coaddressed-join.md) | | 2026-10-21 | `D-RU-3` | | [2026-10-21-u3-cross-rule-cse.md](2026-10-21-u3-cross-rule-cse.md) | | 2026-10-21 | `D-RU-2` | | [2026-10-21-u2-let-planes.md](2026-10-21-u2-let-planes.md) | From c55547f839b105e268eb7284ec1a35ae0217a3e7 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 21 Oct 2026 13:27:13 +0000 Subject: [PATCH 08/14] Add hypothesis-atomization guardrails against bundled verdicts A compound hypothesis is split into atoms (one subject, one predicate, one condition), each anchored to the proposer's words with its qualifiers, and given its own verdict. A bundle verdict is blocked. Recorded incident: the 2026-10-21 ClassView/rails/QR verdict (EXO-50 /026, corrected in /027). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01G22yTnyw --- .claude/agents/hypothesis-atomizer.md | 70 +++++++++ .claude/agents/premise-auditor.md | 2 + .../hypothesis-atomization-guardrails.md | 135 ++++++++++++++++++ 3 files changed, 207 insertions(+) create mode 100644 .claude/agents/hypothesis-atomizer.md create mode 100644 .claude/knowledge/hypothesis-atomization-guardrails.md diff --git a/.claude/agents/hypothesis-atomizer.md b/.claude/agents/hypothesis-atomizer.md new file mode 100644 index 000000000..0575c3abc --- /dev/null +++ b/.claude/agents/hypothesis-atomizer.md @@ -0,0 +1,70 @@ +--- +name: hypothesis-atomizer +description: > + Gate before ANY verdict on someone's hypothesis, plan, proposal, PR or + board post. Splits a compound claim into atomic hypotheses (one subject, + one predicate, one condition), anchors each to the proposer's own words + with its qualifiers, and forbids a bundle verdict. Blocks attribute + substitution (answering an easier question), qualifier dropping, straw + men, conjunction errors and own-work anchoring. Sibling of + premise-auditor (which audits the QUESTION; this audits the VERDICT). + Verdicts: ATOMIZED-CLEAN / BUNDLED (block: re-answer per atom) / + SUBSTITUTED (block: evidence answers a different question) / + QUALIFIER-DROPPED (block: refutation ignores a stated scope) / + STRAWMAN (block: atom not in the source). +model: opus +tools: Read, Glob, Grep +--- + +# hypothesis-atomizer — one claim, one verdict + +**Load first:** `.claude/knowledge/hypothesis-atomization-guardrails.md` +(G1-G10, the 2026-10-21 incident, the output contract). + +## When this card fires + +- Before posting a verdict, review, finding or decision on another + session's or the operator's idea: EXO board posts, PR reviews, plan reviews, + "is this hypothesis right?". +- Whenever the question contains two or more mechanisms joined by *and / + with / through / without / via / so that*. +- When the draft verdict contains *partly, mostly, yes for … no for …, in + principle, it depends*. Those words mean a bundle verdict is being written. + +## Procedure + +1. Read the proposer's source in full (`FIRST-HAND-SOURCE-LAW`). Search + finds candidates; reading decides. +2. Build the atom table (G1, G2, G3). Every atom gets a verbatim quote, its + location, and its qualifiers copied word for word. +3. For each atom, write the steelman (G4), then the evidence, each with the + one-line question it actually answers (G5). +4. Assign a verdict from `SUPPORTED-IN-CODE / REFUTED / OPEN / + NOT-PROPOSED` (G6). Refutations need cited evidence against the + qualified atom (G8). +5. Declare dependencies and propagate only along them (G7). +6. Run the own-work check (G9) and the reread pass (G10). + +## Blocking checks on a draft verdict + +| check | blocks with | +|---|---| +| any verdict sentence covering more than one atom | `BUNDLED` | +| evidence whose stated question is not the atom's | `SUBSTITUTED` | +| a refutation that fails under the atom's verbatim qualifier | `QUALIFIER-DROPPED` | +| an atom with no quote from the proposer, used to judge the proposer | `STRAWMAN` | +| the verdict-writer's own artifact used as the yardstick, with no unanchored verdict alongside | `OWN-WORK` (warning, must be resolved) | + +## Output contract + +Atom table → per-atom steelman, evidence and verdict → `MINE` block → the +verdict tuple → the deciding probe for every `OPEN`. No prose summary over +the bundle. + +## Discipline + +- The gate must be able to stay silent: a one-predicate question passes in + one line. +- It never decides whether an atom is *worth doing*; that is a separate atom + the operator may raise. +- Runs once per verdict, not on its own output. diff --git a/.claude/agents/premise-auditor.md b/.claude/agents/premise-auditor.md index 631348297..3fdc38aa7 100644 --- a/.claude/agents/premise-auditor.md +++ b/.claude/agents/premise-auditor.md @@ -20,6 +20,8 @@ tools: Read, Glob, Grep **READ BY:** the main thread before writing a 5+3 spec, a coresearch brief, or an escalation with options; both council harnesses call this card as a gate. Loads `.claude/knowledge/reference-frame-vs-motion.md` first. +Sibling: `hypothesis-atomizer` audits the VERDICT the same way this card +audits the QUESTION (one claim, one verdict, qualifiers kept). ## Why this card exists diff --git a/.claude/knowledge/hypothesis-atomization-guardrails.md b/.claude/knowledge/hypothesis-atomization-guardrails.md new file mode 100644 index 000000000..31392e389 --- /dev/null +++ b/.claude/knowledge/hypothesis-atomization-guardrails.md @@ -0,0 +1,135 @@ +# Hypothesis atomization — Kahneman/Tversky guardrails for verdicts + +**READ BY:** the main thread and any agent BEFORE giving a verdict on a +hypothesis, plan, proposal or PR; `hypothesis-atomizer` (the gate); +`premise-auditor` (its sibling for questions); both council harnesses +(`/5plus3` Phase 0, `/coresearch` Phase 0). + +**DECISION (operator, 2026-10-21):** "They are different hypotheticals in +one. I absolutely despise if you use Kahneman and Tversky blindness turning +separate ideas into one rejectable item." SCOPE: every verdict, review, +finding or board post that judges someone's idea. REVISIT WHEN: never on cost +grounds. + +## The failure, measured (2026-10-21, EXO-50 `session_01G22yTnyw/026`) + +The question was: *"Can ClassView and the existing ISA/PartOf coordinate +rails make relationships executable through QR folds, without creating +another domain-specific edge implementation?"* It holds at least four +independent claims. The answer: + +1. **Bundled them.** It gave one mixed verdict ("yes for hierarchies, no for + everything else") instead of one verdict per claim. +2. **Substituted an easier question.** It refuted "computing the ISA + transitive closure with QR mxm is pointless", which no plan proposes. The + plan proposes the rails as *coordinate addresses* for a bounded QR focus. +3. **Dropped a scope qualifier.** It rejected the HHTL16 zippers because + "32 B does not fit the 16 B facet". The plan says, in so many words, that + they are views and "not a claim that two 16-byte zipper arrays are already + packed into one 16-byte `FacetCascade`". +4. **Ignored a code fact it had just read.** It said the rails "do not carry" + membership. `facet.rs:30-35` lists `(group:member)` as one sanctioned + reading of an 8:8 tile. +5. **Anchored on its own work.** "EXO-119 should build on #1659" rated the + proposal against the answerer's own PR. +6. **Swapped in its own hypothesis.** "ISA ancestry is a prefix mask" is the + answerer's idea. It was presented as a verdict on the proposer's idea. + +Every one of these is System-1 behaviour. The bundle *felt* answerable in one +breath, so it was answered in one breath. + +## The mechanisms these guardrails target (stated once, used as checks) + +| bias | how it shows up in a verdict | +|---|---| +| **attribute substitution** | a hard question ("does X work as stated?") is answered with an easier one ("is X needed for the obvious case?") | +| **conjunction / disjunction errors** | a bundle A∧B∧C is judged as a unit; one weak conjunct sinks all of them, or one strong conjunct carries all of them | +| **representativeness** | a carrier that *looks like* a hierarchy is judged as only a hierarchy, ignoring readings the code also sanctions | +| **WYSIATI** | the qualifiers and alternatives absent from the verdict-writer's head (view-only, optional, candidate, lab) are treated as absent from the proposal | +| **anchoring** | the verdict-writer's own artifact becomes the yardstick the proposal is measured against | +| **framing** | the proposer's wording ("without creating…") is turned into a pass/fail test of the whole bundle | +| **confirmation** | evidence is gathered for the first verdict formed, not for each claim on its own | + +These are *checks to run forward*, not labels to stick on anyone afterwards +(the `CLAUDE.md` MIRROR rule still holds: ask, never diagnose). + +## The guardrails (G1-G10), in the order they run + +**G1 — Atomize first, verdict later.** Before any verdict, write the atom +table: `H1..Hn`, each **one subject, one predicate, one condition**. Split +on every "and", "with", "through", "without", "via", "so that", and on every +noun that names a separate mechanism. A compound question with N mechanisms +has at least N atoms. No verdict text may come before the table. + +**G2 — Quote-anchor every atom.** Each atom carries the proposer's own words +with a location (`file:line`, plan §, comment id). An atom with no quote is +the verdict-writer's own idea. It goes in a separate block headed +`MINE (not the proposer's)` and is never used to judge a proposer's atom. + +**G3 — Carry every qualifier.** Copy each atom's scope words verbatim: +*view, candidate, optional, lab, bounded, typed, not persisted, illustrative, +if, only when*. A refutation that does not hold under the qualified reading +is **void**. Test: would the refutation still stand if the proposer wrote the +qualifier in capitals? + +**G4 — Steelman before falsifying.** For each atom, state its strongest +reading in the proposer's terms, plus what would have to be true for it to +work. Then attack that reading, never a weaker neighbour. + +**G5 — Name the question your evidence answers.** For every piece of +evidence, write in one line the question it actually answers. If that is not +the atom, the evidence does not bear on the atom and the atom stays `OPEN`. +This is the guard against attribute substitution. + +**G6 — One verdict per atom, from a fixed vocabulary.** + +| verdict | needs | +|---|---| +| `SUPPORTED-IN-CODE` | a cited code fact that makes the atom true or feasible as stated | +| `REFUTED` | a cited code fact, a passed falsifier or a contradiction with a ruling, against the **qualified, steelmanned** atom | +| `OPEN` | neither yet; name the probe that would decide it | +| `NOT-PROPOSED` | the proposer never claimed it (moves to `MINE` or is dropped) | + +**There is no bundle verdict.** "Yes for X, no for Y", "partly", "mostly", +"in principle" and "it depends" about the whole are banned. If a summary is +wanted, it is the tuple of atom verdicts, `H1 S · H2 O · H3 O · H4 O`. + +**G7 — Declare dependencies; propagate only along them.** Draw the edges +(`H3 needs H2`). A refutation of `Hk` reaches another atom only through a +declared edge, and the edge must say *why*. Siblings are independent by +default. A weak conjunct never sinks the bundle; a strong one never carries +it. + +**G8 — Asymmetric burden for REFUTED.** `REFUTED` needs evidence. Absence of +evidence, a missing probe, unfamiliarity or "seems unnecessary" give `OPEN`. +Necessity ("we don't need it") is a separate question from truth ("it does +not work") and gets its own atom if raised. + +**G9 — Own-work anchor check.** If the verdict-writer authored an artifact +the verdict cites (a PR, a crate, a plan), mark the line `OWN-WORK` and give +the verdict without it as well. If the two differ, report both. + +**G10 — Reread the source after drafting.** Before publishing, reread the +proposer's text against the atom table. Every qualifier must appear in an +atom, and no atom may contradict a sentence in the source. A single miss +blocks publication. This is a mechanical check, not judgement. + +## Output contract (every verdict on someone's idea) + +1. The atom table: `id | atom (one predicate) | verbatim quote + location | + qualifiers | depends on`. +2. Per atom: the steelman (G4), the evidence with the question it answers + (G5), and the verdict (G6). +3. A `MINE` block for the verdict-writer's own hypotheses, labelled as such. +4. The verdict tuple. No prose summary over the bundle. +5. For every `OPEN`, the probe that decides it. + +## Can it fire / can it stay silent + +- **Fires:** the 2026-10-21 answer above fails G1 (one mixed verdict), G3 + (the "view" qualifier was dropped for HHTL16), G5 (the closure argument + answered a question nobody asked) and G2 (a `MINE` hypothesis was used as + the verdict). +- **Stays silent:** a single-predicate question ("does `WideFieldMask::with` + take a `u8`?") has one atom, one quote and one cited verdict, so every + guard passes without extra ceremony. From 5f4b9775d76dcfb5d5129596722b22d5ea6623fc Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 21 Oct 2026 13:28:11 +0000 Subject: [PATCH 09/14] Guardrail G11: stratify before pooling a verdict (Simpson's paradox) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01G22yTnyw --- .claude/agents/hypothesis-atomizer.md | 4 ++- .../hypothesis-atomization-guardrails.md | 28 +++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/.claude/agents/hypothesis-atomizer.md b/.claude/agents/hypothesis-atomizer.md index 0575c3abc..1b3c47e2e 100644 --- a/.claude/agents/hypothesis-atomizer.md +++ b/.claude/agents/hypothesis-atomizer.md @@ -11,7 +11,8 @@ description: > Verdicts: ATOMIZED-CLEAN / BUNDLED (block: re-answer per atom) / SUBSTITUTED (block: evidence answers a different question) / QUALIFIER-DROPPED (block: refutation ignores a stated scope) / - STRAWMAN (block: atom not in the source). + STRAWMAN (block: atom not in the source) / POOLED (block: a verdict + that reverses in some stratum — Simpson's paradox, G11). model: opus tools: Read, Glob, Grep --- @@ -53,6 +54,7 @@ tools: Read, Glob, Grep | evidence whose stated question is not the atom's | `SUBSTITUTED` | | a refutation that fails under the atom's verbatim qualifier | `QUALIFIER-DROPPED` | | an atom with no quote from the proposer, used to judge the proposer | `STRAWMAN` | +| a verdict pooled over strata that behave differently (carrier readings, relation law, axis, focus size, consumer, measurement arm); any `S/O`, "mostly" or "depends" cell | `POOLED` (block: one verdict per stratum, G11) | | the verdict-writer's own artifact used as the yardstick, with no unanchored verdict alongside | `OWN-WORK` (warning, must be resolved) | ## Output contract diff --git a/.claude/knowledge/hypothesis-atomization-guardrails.md b/.claude/knowledge/hypothesis-atomization-guardrails.md index 31392e389..be56fe6d1 100644 --- a/.claude/knowledge/hypothesis-atomization-guardrails.md +++ b/.claude/knowledge/hypothesis-atomization-guardrails.md @@ -114,6 +114,34 @@ proposer's text against the atom table. Every qualifier must appear in an atom, and no atom may contradict a sentence in the source. A single miss blocks publication. This is a mechanical check, not judgement. +**G11 — Stratify before you pool (Simpson's paradox).** An atom is still +a pooled population whenever its subject has strata that behave differently. +Before giving an atom a verdict, name the variable it could be confounded +by, and give a verdict per stratum. The usual lurking variables here: + +| atom subject | stratify by | +|---|---| +| a content-blind carrier (`FacetCascade`, an 8:8 tile, a mask) | each sanctioned reading (`part_of:is_a`, `group:member`, `column:row`, palette) | +| a relation | its law: tree/prefix-closed vs general graph; set vs bag | +| a register / QR tile | axis (`Source×Target` vs `Method×Slot`); focus ≤ 256 vs escape | +| an outcome across consumers | each consumer (SAP, IAM, HubSPO, Stockfish) | +| a timing or a gate | each arm / path (pruned vs unpruned, ordered vs unordered lanes, warm vs cold) | + +Rules: +- A pooled verdict that **reverses** in any stratum is void. Report the + strata, not the pool. +- A verdict cell that reads `S/O`, "mostly" or "depends" is a pooled cell. + Split it into per-stratum rows. +- A gate that passes on one path and fails on another is two results + ("prepare gate passes pruned, fails unpruned"), never one. +- The strata come from the code and the plan (sanctioned readings, + declared axes, named arms), never from the verdict-writer's taste. + +Measured instance (2026-10-21): `/026` said "the rails do not carry +relationships", pooled over tile readings; in the `(group:member)` stratum +(`facet.rs:30-35`) the verdict reverses. `/027` still pooled H2 into `S/O` +and H3 over axis and focus size. + ## Output contract (every verdict on someone's idea) 1. The atom table: `id | atom (one predicate) | verbatim quote + location | From 7cfc32cf94dec26ba631d37178a8ad25df177633 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 21 Oct 2026 13:28:18 +0000 Subject: [PATCH 10/14] hypothesis-atomizer: card header names G1-G11 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01G22yTnyw --- .claude/agents/hypothesis-atomizer.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.claude/agents/hypothesis-atomizer.md b/.claude/agents/hypothesis-atomizer.md index 1b3c47e2e..fcd26a292 100644 --- a/.claude/agents/hypothesis-atomizer.md +++ b/.claude/agents/hypothesis-atomizer.md @@ -20,7 +20,7 @@ tools: Read, Glob, Grep # hypothesis-atomizer — one claim, one verdict **Load first:** `.claude/knowledge/hypothesis-atomization-guardrails.md` -(G1-G10, the 2026-10-21 incident, the output contract). +(G1-G11, the 2026-10-21 incident, the output contract). ## When this card fires From aa23d7f53344901b7b8ebbdab69385b910b18721 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 21 Oct 2026 15:03:46 +0000 Subject: [PATCH 11/14] =?UTF-8?q?dir-rmap-probe:=20ClassView=20x=20DR=20de?= =?UTF-8?q?mo=20=E2=80=94=20effective=20roles=20compiled=20from=20the=20cl?= =?UTF-8?q?ass's=20compute=5Fdag?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The user class declares through the contract's existing ClassView surface that effective_roles is computed from member_of. A direct-routing table maps that target position to the existing compiled closure fold (Compiled::roles_of); execute_compute_dag runs it. No new relationship, association or planner type. Checked on every user against dir-sim and the crepe oracle, before and after removing one of two support paths; a class that does not declare the behavior computes nothing. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01G22yTnyw --- crates/dir-rmap-probe/Cargo.toml | 6 + crates/dir-rmap-probe/tests/classview_dr.rs | 194 ++++++++++++++++++++ 2 files changed, 200 insertions(+) create mode 100644 crates/dir-rmap-probe/tests/classview_dr.rs diff --git a/crates/dir-rmap-probe/Cargo.toml b/crates/dir-rmap-probe/Cargo.toml index 2bbf11043..02cd55285 100644 --- a/crates/dir-rmap-probe/Cargo.toml +++ b/crates/dir-rmap-probe/Cargo.toml @@ -34,6 +34,12 @@ crepe = { git = "https://github.com/AdaWorldAPI/crepe", branch = "ccr-d3ce1263-i # Same patches as lance-graph-dir-sim: one ndarray identity, and the OGAR # sibling checkout so a paired OGAR change is tested before it merges. +# The ClassView x DR demo (tests/classview_dr.rs): the class declares the +# behavior through the contract's ClassView / compute_dag; nothing else from +# the contract is used. +[dev-dependencies] +lance-graph-contract = { path = "../lance-graph-contract" } + [patch."https://github.com/AdaWorldAPI/ndarray"] ndarray = { path = "../../../ndarray" } diff --git a/crates/dir-rmap-probe/tests/classview_dr.rs b/crates/dir-rmap-probe/tests/classview_dr.rs new file mode 100644 index 000000000..6fce547ae --- /dev/null +++ b/crates/dir-rmap-probe/tests/classview_dr.rs @@ -0,0 +1,194 @@ +//! ClassView × DR, IAM case: a behavior compiled from what the class knows. +//! +//! The user class declares, through the contract's existing `ClassView` +//! surface, that `effective_roles` is computed from `member_of` +//! (`compute_dag`). Direct routing maps that target position to an existing +//! executable: the compiled membership-closure fold `Compiled::roles_of` +//! (arm B+). The contract's `execute_compute_dag` runs it. No relationship +//! type, association model or planner is introduced: the class names the +//! behavior, the routing table names the fold, the bindings are the probe's. +//! +//! Checked on every user against dir-sim's own walks (arm A) and the crepe +//! oracle (arm D), before and after an overlay that removes one of two +//! support paths. + +use dir_rmap_probe::fixture::{Fixture, Spec}; +use dir_rmap_probe::prep::{Chain, Compiled, Ordinals}; +use dir_rmap_probe::{Edit, World, a_groups, oracle, templates}; +use lance_graph_contract::class_view::{ClassId, ClassView, ComputeEdge, execute_compute_dag}; +use lance_graph_contract::ontology::{DisplayTemplate, FieldRef}; +use ogar_dir_core::Guid128; +use std::collections::HashMap; + +const USER: ClassId = 1; +/// Field positions of the user class: its field basis. +const MEMBER_OF: u8 = 0; +const EFFECTIVE_ROLES: u8 = 1; + +/// What the class knows: two fields, and that one is computed from the other. +struct IamUser { + fields: Vec, + dag: &'static [ComputeEdge], +} + +static ROLES_FROM_MEMBERSHIP: [ComputeEdge; 1] = [ComputeEdge { + target: EFFECTIVE_ROLES, + inputs: &[MEMBER_OF], +}]; + +impl IamUser { + fn new(dag: &'static [ComputeEdge]) -> Self { + IamUser { + fields: vec![ + FieldRef::new("urn:iam:user:memberOf", "member_of"), + FieldRef::new("urn:iam:user:effectiveRoles", "effective_roles"), + ], + dag, + } + } +} + +impl ClassView for IamUser { + fn fields(&self, class: ClassId) -> &[FieldRef] { + if class == USER { &self.fields } else { &[] } + } + fn template(&self, _: ClassId) -> DisplayTemplate { + DisplayTemplate::Detail + } + fn dolce_category_id(&self, _: ClassId) -> u8 { + 0 + } + fn compute_dag(&self, class: ClassId) -> &[ComputeEdge] { + if class == USER { self.dag } else { &[] } + } +} + +/// Direct routing: field position → the executable bound to it. `None` for +/// a position that is read from the bindings, not computed. +type Route = fn(&Compiled, u16) -> u64; +const DR: [Option; 2] = [None, Some(Compiled::roles_of)]; + +/// The per-user store the compiled behavior writes into. +struct Cell<'a> { + compiled: &'a Compiled, + user: u16, + roles: Option, +} + +/// Run what the class declares for one user. `None` when the class does not +/// declare the behavior: nothing is computed that the class does not know. +fn run(view: &dyn ClassView, compiled: &Compiled, user: u16) -> Option { + let mut cell = Cell { + compiled, + user, + roles: None, + }; + execute_compute_dag(view.compute_dag(USER), &mut cell, |c, target| { + let route = DR.get(target as usize).copied().flatten().ok_or(target)?; + c.roles = Some(route(c.compiled, c.user)); + Ok::<(), u8>(()) + }) + .expect("every declared target has a route"); + cell.roles +} + +/// Roles as dir-sim answers them (arm A): the grants of the SIDs it reports. +fn a_roles(w: &World, u: &Guid128) -> u64 { + let by_guid: HashMap = + w.f.grants.iter().fold(HashMap::new(), |mut m, &(g, r)| { + *m.entry(w.f.groups[g as usize]).or_default() |= 1 << r; + m + }); + a_groups(&w.view(), u, Chain::Security) + .iter() + .fold(0, |a, g| a | by_guid.get(g).copied().unwrap_or(0)) +} + +fn check(w: &World) -> usize { + let view = IamUser::new(&ROLES_FROM_MEMBERSHIP); + let compiled = Compiled::compile(w.prepare()); + let ord = Ordinals::of(&w.f, &w.view()); + let d = oracle::run(&w.f, &w.direct, &w.nested, &templates(&w.f)); + let mut d_roles = vec![0u64; w.f.users.len()]; + for &(u, role) in &d.roles { + d_roles[u as usize] |= 1 << role; + } + let mut holders = 0; + for (i, ug) in w.f.users.iter().enumerate() { + let got = run(&view, &compiled, ord.user[i]).expect("declared"); + assert_eq!(got, a_roles(w, ug), "dir-sim roles, user {i}"); + assert_eq!(got, d_roles[i], "oracle roles, user {i}"); + holders += usize::from(got != 0); + } + holders +} + +#[test] +fn the_declared_behavior_equals_dir_sim_and_the_oracle_for_every_user() { + let w = World::new(Fixture::generate(Spec::small())); + let holders = check(&w); + // Anti-vacuity: the comparison is meaningful only if roles are held by + // some users and not by all. + assert!( + holders > 0 && holders < w.f.users.len(), + "holders {holders}" + ); +} + +#[test] +fn removing_one_support_path_keeps_the_role_the_other_path_carries() { + let mut w = World::new(Fixture::generate(Spec::small())); + check(&w); + // Make a role doubly supported, then remove one support. Roles come only + // through security groups, so both supports must be security groups that + // grant the same role: a user directly in g1, and a second group g2 + // granting a shared role that the user is not yet in. + let grants: HashMap = w.f.grants.iter().fold(HashMap::new(), |mut m, &(g, r)| { + *m.entry(g).or_default() |= 1 << r; + m + }); + let sec = |g: u32| w.f.security[g as usize] == Some(true); + let g_of = |g: u32| grants.get(&g).copied().unwrap_or(0); + let (u, g1, g2) = w + .direct + .iter() + .filter(|&&(_, g1)| sec(g1) && g_of(g1) != 0) + .find_map(|&(u, g1)| { + (0..w.f.groups.len() as u32) + .find(|&g2| { + g2 != g1 && sec(g2) && g_of(g1) & g_of(g2) != 0 && !w.direct.contains(&(u, g2)) + }) + .map(|g2| (u, g1, g2)) + }) + .expect("fixture has two security groups granting one role"); + w.apply(&[Edit::AddDirect(u, g2)]); + check(&w); + let shared = |w: &World| { + let compiled = Compiled::compile(w.prepare()); + let ord = Ordinals::of(&w.f, &w.view()); + run( + &IamUser::new(&ROLES_FROM_MEMBERSHIP), + &compiled, + ord.user[u as usize], + ) + .unwrap() + }; + let before = shared(&w); + w.apply(&[Edit::RemoveDirect(u, g1)]); + check(&w); + let after = shared(&w); + let both = g_of(g1) & g_of(g2); + assert_eq!(before & both, both, "precondition: both supports grant it"); + assert_eq!(after & both, both, "the role survives losing one support"); +} + +#[test] +fn a_class_that_does_not_declare_the_behavior_computes_nothing() { + let w = World::new(Fixture::generate(Spec::small())); + let compiled = Compiled::compile(w.prepare()); + let ord = Ordinals::of(&w.f, &w.view()); + let silent = IamUser::new(&[]); + for i in 0..w.f.users.len() { + assert_eq!(run(&silent, &compiled, ord.user[i]), None); + } +} From d76c161cdfaf38239903d5aa02b0c401f046f056 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 21 Oct 2026 15:08:33 +0000 Subject: [PATCH 12/14] dir-rmap-probe: cost of the ClassView x DR IAM behavior against the direct paths Per-user ns for the existing dir-sim walk, the prepared CSR walk, the bare compiled fold, the fold reached through execute_compute_dag, and the declaration resolved once per class. Preparation is reported as a failed gate (millisecond scale), never as a measurement. Board entry 2026-10-21-classview-dr-iam. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01G22yTnyw --- .../entries/2026-10-21-classview-dr-iam.md | 56 +++++ .claude/board/entries/README.md | 1 + .../examples/classview_dr_cost.rs | 191 ++++++++++++++++++ 3 files changed, 248 insertions(+) create mode 100644 .claude/board/entries/2026-10-21-classview-dr-iam.md create mode 100644 crates/dir-rmap-probe/examples/classview_dr_cost.rs diff --git a/.claude/board/entries/2026-10-21-classview-dr-iam.md b/.claude/board/entries/2026-10-21-classview-dr-iam.md new file mode 100644 index 000000000..92f880abd --- /dev/null +++ b/.claude/board/entries/2026-10-21-classview-dr-iam.md @@ -0,0 +1,56 @@ +# 2026-10-21 — ClassView × DR, IAM: a behavior compiled from what the class knows + +**Status:** MEASURED (lab, `crates/dir-rmap-probe`). **Basis:** operator +kickoff "Quack Binding | ClassView | DR" and its add-on "compile behavior +through relationship mapping" (2026-10-21). No new relationship, +association, method-descriptor or planner type. + +## What runs + +- The user class declares, through the existing `ClassView::compute_dag`, + that `effective_roles` (position 1) is computed from `member_of` + (position 0). +- A direct-routing table maps position 1 to the existing compiled closure + fold `Compiled::roles_of`. +- `tests/classview_dr.rs`: the routed behavior equals dir-sim's own walk and + the crepe oracle for every user; a doubly supported role survives losing + one support; a class that does not declare the behavior computes nothing. + Disable runs: a wrong route fails two tests, ignoring the declaration + fails the silence test. + +## Measured (`examples/classview_dr_cost.rs`, 8,000 users, 1,495 groups) + +Per user, every answer checked equal to the direct path first: + +| route | ns | +|---|---| +| direct dir-sim walk (existing path, negative control) | ~450,000 | +| direct prepared CSR walk | ~190 | +| compiled fold, called directly | ~13–19 | +| ClassView × DR via `execute_compute_dag` per row | ~100 | +| ClassView × DR, declaration resolved once per class | ~13–19 (±0.4 over the fold) | + +Preparation (`prepare` + `compile`, once per version): FAILED GATE +(millisecond scale), the same preparation D-RMAP-DIR-1 already carries. + +## Findings, per stratum + +- **Correctness:** equal to dir-sim and the oracle (test-pinned). +- **Dispatch:** `execute_compute_dag` recomputes the topological order and + allocates on every call (~85 ns per row). Resolving the declaration once + per class removes it. ClassView decides once; only the routed fold runs + per row. +- **Binding legality:** group slots are dir-sim ordinals of one snapshot + version; the probe's `Witness` is lab-local. The contract-level + frozen-codebook witness is still missing (shared with D-CMR-5, D-RU-6). +- **SAP (bypass):** `BAPI_ORDINALS` is already a position map over + `CatsSchema`'s ClassView basis; existing tests (`edges.rs` + `bapi_order_is_a_permutation_over_canonical_field_identity`, byte pins) + pass. KEEP_DIRECT: no DR table, no new code. + +## Open + +- Whether to add a once-per-class resolution helper next to + `execute_compute_dag` in the contract (the measured dispatch cost). +- The Odoo method case and relalg as a routed method are not run yet. +- Preparation fails the gate. diff --git a/.claude/board/entries/README.md b/.claude/board/entries/README.md index bffdc03cb..ff5da6548 100644 --- a/.claude/board/entries/README.md +++ b/.claude/board/entries/README.md @@ -43,6 +43,7 @@ exactly one of them, never both. | 2026-10-21 | `D-CMR-2b` | | [2026-10-21-d-cmr-2b-scratch-probe.md](2026-10-21-d-cmr-2b-scratch-probe.md) | | 2026-10-21 | `D-CMR-2` | | [2026-10-21-d-cmr-2-lowering.md](2026-10-21-d-cmr-2-lowering.md) | | 2026-10-21 | `D-RU-6` | | [2026-10-21-coresearch-ms-vs-ns-folding.md](2026-10-21-coresearch-ms-vs-ns-folding.md) | +| 2026-10-21 | `classview-dr-iam` | | [2026-10-21-classview-dr-iam.md](2026-10-21-classview-dr-iam.md) | | 2026-10-20 | `D-RU-1` | | [2026-10-20-u1-jc-batch.md](2026-10-20-u1-jc-batch.md) | | 2026-10-20 | `thought-git-replay-api-write-rubicon` | | [2026-10-20-thought-git-replay-api-write-rubicon.md](2026-10-20-thought-git-replay-api-write-rubicon.md) | | 2026-10-20 | `ADR-2026-10-20-SIX` | | [2026-10-20-six-decisions.md](2026-10-20-six-decisions.md) | diff --git a/crates/dir-rmap-probe/examples/classview_dr_cost.rs b/crates/dir-rmap-probe/examples/classview_dr_cost.rs new file mode 100644 index 000000000..00e345770 --- /dev/null +++ b/crates/dir-rmap-probe/examples/classview_dr_cost.rs @@ -0,0 +1,191 @@ +//! Cost of the ClassView × DR IAM behavior against the direct paths. +//! +//! ```text +//! cargo run --release --example classview_dr_cost # 8,000 users +//! cargo run --release --example classview_dr_cost -- 65000 +//! ``` +//! +//! Four routes to "the roles user u holds", each checked equal to the direct +//! dir-sim answer before it is timed: +//! +//! * `direct-view`: dir-sim's own SID walk plus a grant map (the shape +//! `ViewActors::for_principal` reads) — the mandatory negative control; +//! * `direct-csr`: the prepared CSR walk (`Prepared::roles_of`); +//! * `fold`: the compiled closure fold called directly (`Compiled::roles_of`); +//! * `classview-dr`: the same fold reached through the class's declaration +//! (`compute_dag`) and the routing table, via `execute_compute_dag`. +//! +//! Preparation (`prepare` + `compile`) is reported separately, never folded +//! into a per-user figure. + +use dir_rmap_probe::fixture::{Fixture, Spec}; +use dir_rmap_probe::prep::{Chain, Compiled, Ordinals, words}; +use dir_rmap_probe::{World, a_groups}; +use lance_graph_contract::class_view::{ + ClassId, ClassView, ComputeEdge, compute_dag_topo_order, execute_compute_dag, +}; +use lance_graph_contract::ontology::{DisplayTemplate, FieldRef}; +use ogar_dir_core::Guid128; +use std::collections::HashMap; +use std::hint::black_box; +use std::time::{Duration, Instant}; + +const USER: ClassId = 1; +static DAG: [ComputeEdge; 1] = [ComputeEdge { + target: 1, + inputs: &[0], +}]; +type Route = fn(&Compiled, u16) -> u64; +const DR: [Option; 2] = [None, Some(Compiled::roles_of)]; + +struct IamUser(Vec); +impl ClassView for IamUser { + fn fields(&self, _: ClassId) -> &[FieldRef] { + &self.0 + } + fn template(&self, _: ClassId) -> DisplayTemplate { + DisplayTemplate::Detail + } + fn dolce_category_id(&self, _: ClassId) -> u8 { + 0 + } + fn compute_dag(&self, _: ClassId) -> &[ComputeEdge] { + &DAG + } +} + +fn via_classview(view: &dyn ClassView, c: &Compiled, u: u16) -> u64 { + let mut out = (c, u, 0u64); + execute_compute_dag(view.compute_dag(USER), &mut out, |s, t| { + let route = DR[t as usize].ok_or(t)?; + s.2 = route(s.0, s.1); + Ok::<(), u8>(()) + }) + .expect("routed"); + out.2 +} + +/// The behavior compiled once per class: the declaration is resolved to the +/// routed executables in dependency order; per row only the routes run. +fn compile_class(view: &dyn ClassView) -> Vec { + compute_dag_topo_order(view.compute_dag(USER)) + .expect("acyclic") + .into_iter() + .map(|t| DR[t as usize].expect("routed")) + .collect() +} + +fn median(mut v: Vec) -> Duration { + v.sort(); + v[v.len() / 2] +} + +fn per_user(n: usize, reps: usize, mut f: impl FnMut()) -> f64 { + let mut t = Vec::with_capacity(reps); + for _ in 0..reps { + let s = Instant::now(); + f(); + t.push(s.elapsed()); + } + median(t).as_nanos() as f64 / n as f64 +} + +fn main() { + let users: usize = std::env::args() + .nth(1) + .map_or(8_000, |s| s.parse().expect("user count")); + let w = World::new(Fixture::generate(Spec::iam(users))); + let v = w.view(); + let ord = Ordinals::of(&w.f, &v); + let by_guid: HashMap = + w.f.grants.iter().fold(HashMap::new(), |mut m, &(g, r)| { + *m.entry(w.f.groups[g as usize]).or_default() |= 1 << r; + m + }); + let direct = |u: &Guid128| { + a_groups(&v, u, Chain::Security) + .iter() + .fold(0u64, |a, g| a | by_guid.get(g).copied().unwrap_or(0)) + }; + + let s = Instant::now(); + let b = w.prepare(); + let t_prepare = s.elapsed(); + let s = Instant::now(); + let c = Compiled::compile(w.prepare()); + let t_compile = s.elapsed().saturating_sub(t_prepare); + let view = IamUser(vec![ + FieldRef::new("urn:iam:user:memberOf", "member_of"), + FieldRef::new("urn:iam:user:effectiveRoles", "effective_roles"), + ]); + + let (mut scratch, mut queue) = (vec![0u64; words(v.groups_len())], Vec::new()); + for (i, ug) in w.f.users.iter().enumerate() { + let want = direct(ug); + let uo = ord.user[i]; + assert_eq!(b.roles_of(uo, &mut scratch, &mut queue), want, "csr {i}"); + assert_eq!(c.roles_of(uo), want, "fold {i}"); + assert_eq!(via_classview(&view, &c, uo), want, "classview-dr {i}"); + } + + let n = w.f.users.len(); + let reps = 15; + let t_view = per_user(n, 5, || { + for ug in &w.f.users { + black_box(direct(ug)); + } + }); + let t_csr = per_user(n, reps, || { + for &uo in &ord.user { + black_box(b.roles_of(uo, &mut scratch, &mut queue)); + } + }); + let t_fold = per_user(n, reps, || { + for &uo in &ord.user { + black_box(c.roles_of(black_box(uo))); + } + }); + let routes = compile_class(&view); + for &uo in &ord.user { + assert_eq!(routes[0](&c, uo), c.roles_of(uo), "compiled"); + } + let t_compiled = per_user(n, reps, || { + for &uo in &ord.user { + for r in &routes { + black_box(r(&c, black_box(uo))); + } + } + }); + let t_dr = per_user(n, reps, || { + for &uo in &ord.user { + black_box(via_classview(&view, &c, black_box(uo))); + } + }); + + println!( + "users {n}, groups {}, all {n} answers checked equal to direct-view", + v.groups_len() + ); + println!("per user (median of runs):"); + println!(" direct-view {t_view:>10.1} ns (existing dir-sim path)"); + println!(" direct-csr {t_csr:>10.1} ns"); + println!(" fold {t_fold:>10.1} ns"); + println!( + " classview-dr {t_dr:>10.1} ns (dispatch over fold: {:+.1} ns)", + t_dr - t_fold + ); + println!( + " classview-dr compiled once per class {t_compiled:>6.1} ns (over fold: {:+.1} ns)", + t_compiled - t_fold + ); + // P0: a millisecond result is a failed gate, never a measurement. + let prep = t_prepare + t_compile; + if prep >= Duration::from_millis(1) { + println!("preparation, once per version: FAILED GATE (millisecond scale)"); + } else { + println!( + "preparation, once per version: {:.1} us", + prep.as_secs_f64() * 1e6 + ); + } +} From a9f05b2083f7df4ca960dbd9a43487495c4b61ae Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 21 Oct 2026 15:25:35 +0000 Subject: [PATCH 13/14] crepe-plan-probe: coaddressed reads sides in place after rebase onto main #1658's co-addressed join used Side::values(), a slot-order copy of the non-key column. #1659 removed that copy. The restriction now runs over the relation's lane in row order, and each side row reads its bit through Side::positions(); values_at reads lane()[positions()[j]]. Store is lane based (Store::from_rows) in the module, its tests and its example. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01G22yT6htkcdyXsihxxXdrg --- .../crepe-plan-probe/examples/coaddressed.rs | 6 ++-- crates/crepe-plan-probe/src/coaddressed.rs | 35 ++++++++++--------- 2 files changed, 21 insertions(+), 20 deletions(-) diff --git a/crates/crepe-plan-probe/examples/coaddressed.rs b/crates/crepe-plan-probe/examples/coaddressed.rs index 81fb9bce6..d7a29e125 100644 --- a/crates/crepe-plan-probe/examples/coaddressed.rs +++ b/crates/crepe-plan-probe/examples/coaddressed.rs @@ -94,10 +94,8 @@ impl Data { t: rows(0x2545_F491, n, centers, queries), } } - fn store(&self) -> Store<'_> { - Store { - relations: vec![(R, &self.r, 1), (S, &self.s, 2), (T, &self.t, 3)], - } + fn store(&self) -> Store { + Store::from_rows([(R, &self.r, 1), (S, &self.s, 2), (T, &self.t, 3)]) } } diff --git a/crates/crepe-plan-probe/src/coaddressed.rs b/crates/crepe-plan-probe/src/coaddressed.rs index 20671ae4e..9530183c8 100644 --- a/crates/crepe-plan-probe/src/coaddressed.rs +++ b/crates/crepe-plan-probe/src/coaddressed.rs @@ -67,7 +67,7 @@ pub struct CoSide { impl CoSide { pub fn new(side: Side) -> CoSide { let off = side.offsets(); - let mut slot = Vec::with_capacity(side.values().len()); + let mut slot = Vec::with_capacity(side.positions().len()); for s in 0..off.len() - 1 { for _ in off[s]..off[s + 1] { slot.push(s as u32); @@ -77,7 +77,7 @@ impl CoSide { } pub fn n_rows(&self) -> usize { - self.side.values().len() + self.side.positions().len() } /// Rows of this side at slot `s`, as a row range. @@ -112,14 +112,11 @@ pub fn layout_key(key: SideKey) -> SideKey { impl CoTable { /// Lays out every side of `plan` from all its rows, ignoring restrictions. - pub fn prepare(plan: &StarPlan, store: &Store<'_>) -> CoTable { - let inputs: Vec<(RelationView<'_>, usize)> = plan + pub fn prepare(plan: &StarPlan, store: &Store) -> CoTable { + let inputs: Vec<(&RelationView, usize)> = plan .sides .iter() - .map(|s| { - let (rows, generation) = store.view(s.key.rel); - (RelationView { rows, generation }, s.key.key_col as usize) - }) + .map(|s| (store.view(s.key.rel), s.key.key_col as usize)) .collect(); let table = JoinTable::prepare_star(&inputs); CoTable { @@ -331,9 +328,13 @@ fn select( let mut counts = vec![0i64; n]; let mut plane = vec![0u64; words_for(n)]; if rel != Rel::Empty && rows_n > 0 { - let lanes = [LaneRef::U32(s.side.values())]; + // The restriction runs over the relation's lane in place, in row + // order; side row `j` then reads its bit at `positions()[j]`. + let lane = s.side.lane(); + let mut lane_rows = vec![0u64; words_for(lane.len())]; + let lanes = [LaneRef::U32(lane)]; let view = Planes { - n_rows: rows_n, + n_rows: lane.len(), masks: &[], lanes: &lanes, }; @@ -343,10 +344,14 @@ fn select( measure: Measure::Keep, }, &view, - Some(&mut rows), + Some(&mut lane_rows), ) .map_err(CoRefusal::Relalg)?; add_run(cost, run, program, programs); + for (j, &p) in s.side.positions().iter().enumerate() { + let p = p as usize; + rows[j >> 6] |= (lane_rows[p >> 6] >> (p & 63) & 1) << (j & 63); + } project(s, &rows, &mut counts, cost, programs)?; cost.sides_filtered += 1; cost.side_rows_read += rows_n; @@ -425,7 +430,7 @@ impl CoEval { }; side.run(s).filter_map(move |j| { let keep = rows.is_none_or(|m| m[j >> 6] >> (j & 63) & 1 == 1); - keep.then(|| side.side.values()[j]) + keep.then(|| side.side.lane()[side.side.positions()[j] as usize]) }) } @@ -563,10 +568,8 @@ mod tests { t: rows(0x2545_F491, 400, 40), } } - fn store(&self) -> Store<'_> { - Store { - relations: vec![(R, &self.r, 1), (S, &self.s, 2), (T, &self.t, 3)], - } + fn store(&self) -> Store { + Store::from_rows([(R, &self.r, 1), (S, &self.s, 2), (T, &self.t, 3)]) } } From 79e31661b7299f49899ef0a454a6e8d21e5248a9 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 21 Oct 2026 15:27:12 +0000 Subject: [PATCH 14/14] =?UTF-8?q?board:=20coresearch=20exploration=20map?= =?UTF-8?q?=20=E2=80=94=20ClassView=20capabilities=20and=20an=20algebra=20?= =?UTF-8?q?polyfill?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five scouts, three co-architects, 33-row crosswalk. Adopt-now: X9 (normalize vs Kleene oracle) and X10 (realization conformance helper). Eight pre-registered probes; one contract bridge left open for the operator (defaulted ClassView::semantics). Names a soundness gap: production normalize is two-valued on NULL-capable lanes. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01G22yT6htkcdyXsihxxXdrg --- ...1-coresearch-classview-algebra-polyfill.md | 147 ++++++++++++++++++ .claude/board/entries/README.md | 3 +- 2 files changed, 149 insertions(+), 1 deletion(-) create mode 100644 .claude/board/entries/2026-10-21-coresearch-classview-algebra-polyfill.md diff --git a/.claude/board/entries/2026-10-21-coresearch-classview-algebra-polyfill.md b/.claude/board/entries/2026-10-21-coresearch-classview-algebra-polyfill.md new file mode 100644 index 000000000..c4ffe7308 --- /dev/null +++ b/.claude/board/entries/2026-10-21-coresearch-classview-algebra-polyfill.md @@ -0,0 +1,147 @@ +# Co-research: what classes can do, and an algebra polyfill on ClassView — exploration map (2026-10-21) + +**Status:** OPEN — exploration map; ratifies nothing. Question (operator, 2026-10-21): +"Was können Klassen und wo sind Potentiale und Synergien — besonders für Polyfill auf +Algebra." Harness: `.claude/agents/coresearch-council.md`. + +**Premise gate (PREMISE-SPLIT).** "Realization selected per class" mixed two lifetimes: +- (a) what a class DECLARES: its law and the properties that law satisfies (per class); +- (b) which realization is CHOSEN for a binding instance (per class × binding state: focus + size, version, witness availability, answer type). + +**Anchors held:** +- A1: ClassView decides, DR maps; no new Association, RelationshipShape, Through, + MethodDescriptor or planner type without a proven gap. +- A2: D-RA-2 — the contract owns the laws, consumers own execution. +- A3: ADMISSION B. +- A4: no ms; mask-risc is the reference fold. +- A5: SIMD via ndarray. +- A6: G1–G11, one verdict per atom and stratum. + +## Council + +**Scouts:** +- code cartographer; +- internal prior art; +- literature: Green–Karvounarakis–Tannen 2007, FAQ, GraphBLAS math, TACO format + abstraction, Halide, tagless-final, egg (all SECTION-READ); +- systems: PostgreSQL operator classes and families, SuiteSparse:GraphBLAS, MLIR linalg, + Soufflé, DuckDB, Odoo ORM; +- concepts: typeclass coherence, institutions, provenance semirings, Galois connections, + Kleene K3, OWL 2, FCA, binding-time analysis. + +**Co-architects:** bridge, firewall/fit, falsifier. The crosswalk has 33 rows (X1–X22 from +outside, U1–U11 from the operator's reflection); raw output is banked in the session +scratchpad. + +## What the outside world agrees on + +1. **The law is bound at declaration or bind time, never per row.** Every system and paper + does this (PG opclass lookup, GraphBLAS factory/JIT kernels, Halide schedules, TACO + codegen, tagless-final). This matches the IAM measurement: resolving the class once + costs ±0.4 ns over the fold, while `execute_compute_dag` per row costs ~85 ns + (entry 2026-10-21-classview-dr-iam). +2. **A cheaper route is licensed by a law PROPERTY:** + - annihilator 0 → absent elements may be skipped; + - associativity → reordering or parallelism; + - semiring homomorphism → a result may be translated (positive RA only); + - commuting aggregates → variables may be reordered. +3. **Nobody checks that a realization satisfies its law.** PostgreSQL says so explicitly + for families, and DuckDB's lossy casts change the law silently. The common gap is a + conformance gate. + +## Corrections the co-architects made to the crosswalk + +- **M1.** `Activation::register_law_for` returns `register128::RegisterLaw` + (RelativeOffset / AxisPosition / Support), which is a register byte law, not an algebra + law. The per-operation algebra carrier is `set_rules::Semantics`. X2 stays PARTIAL on + that basis alone. +- **M2.** `execute_compute_dag` owns sequencing only (topo order plus abort). Each target's + value is a consumer closure called per call, so "dependency → recompute" is PARTIAL, not + ALREADY-HAVE. +- **M3 (soundness gap).** The production `normalize`/`simplify` (`set_algebra.rs:164-215`) + is two-valued and applies complement rules unconditionally. The Kleene oracle exists only + under `#[cfg(test)]` (`set_rules.rs:609+`). Any lane that can carry NULL is normalized + under two-valued laws today. +- **M4.** "compute_dag → executed fold" holds in the IAM probe path only. The contract + executor is the per-target closure route, so the contract-level grade is PARTIAL. +- **M5.** "why-provenance = union of input masks" holds for monotone queries only; under + minus/NOT it over-reports. + +## The operator's reflection, per atom (no bundle verdict) + +| atom | verdict | evidence | +|---|---|---| +| class = signature (T,F,R,M,G) | PARTIAL | T, F, R reachable on ClassView. M (MethodSig / MethodTable) and G (ActionDef guard) sit on other surfaces, which OGAR doctrine wants. | +| A relations → relalg | single table: SUPPORTED · join / composition / closure: OPEN | relalg lacks D-RA-3/4 | +| B computed properties → fold | IAM stratum (Boolean, one relation): SUPPORTED-IN-CODE · other strata: OPEN | `dir-rmap-probe/tests/classview_dr.rs` | +| C polymorphism = masked dispatch by ClassId | OPEN | `ternlog_dispatch` precedent; probe P1 | +| D guards as Boolean masks | PARTIAL | computing a mask never authorizes (`ActionInvocation::commit`) | +| E NARS reasoning | as finisher: SUPPORTED (D-CMH-1 `NarsRevision`) · as semiring: OPEN | two scouts independently say not shown; probe P6 | +| CSE | SUPPORTED-IN-CODE | `SetProgram` lets / share | +| fusion | OPEN | D-CMH-1 fused multi-sink folds | +| incremental | OPEN | `execute_compute_dag` recomputes every target | +| law table: Boolean / bag / topological | SUPPORTED-IN-CODE | `Semantics.truth`, `Semantics.population`, `compute_dag_topo_order` | +| law table: sum | PARTIAL | `Semantics.arith` modes | +| law table: min-plus | OPEN | probe P2 tests whether it lowers onto Boolean frontier folds | +| law table: NARS | finisher only | | +| biggest lever: compute_dag → RISC fold | IAM: SUPPORTED · elsewhere: OPEN | | + +## Exploration map + +**ADOPT-NOW** (test-only or a defaulted method; nothing changes behaviour): +- **X9 / M3:** extend the `set_rules` rewrite-equality loop to `Truth::ThreeValued` against + the Kleene oracle (~20 lines). Either `NotTwoValued` refuses every complement rule, or the + loop prints a counterexample against `normalize`. +- **X10:** a conformance helper generalized from `classview_dr.rs::check()` (~30 lines). + Every declared realization must match the mask-risc reference fold, per stratum. + +**PROBE** (each pre-registered by the falsifier with assertion, disable and silent twin): +- **P1 (X17+U4):** polymorphism as a classid-mask partition plus already compiled folds. + Two classes, two `compute_dag`s, resolved once. Equals per-row dispatch on every row, + with timing in ns/row. Green means virtual dispatch is a fold primitive. +- **P2 (X13+U10):** min-plus hop distance from iterated Boolean frontier masks, against + BFS on the dir-sim nest. Green means `ComputeEdge` needs no aggregate field. +- **P3 (X4):** law-property flags derived from the `Refusal` catalogue + (`fn licensed(&Semantics)`, ~15 lines). Predicted admit/refuse must equal actual for every + Rule × Semantics pair. +- **P4 (X6+X8):** `Count > 0 ⇔ Any` on positive programs; at least one counterexample + under minus/NOT. +- **P5 (X15/U9):** incremental recompute via forward reachability from a dirty + `WideFieldMask` (`execute_compute_dag_dirty`, ~25 lines). Equals the full recompute. +- **P6 (U6):** NARS revision associativity and distributivity on a quantized grid. The + semiring claim needs 0 violations. +- **P7 (X18/X19/X20):** a ClassView law suite plus a cross-provider coherence diff. Assert + that the providers' classid overlap is non-empty first. +- **P8 (X11):** frozen codebook: Eq always holds; Lt only when order-preserving. + +**PARK:** +- X3: candidate realizations stay consumer-side (the IAM `DR` array) per A2, until a + second stratum needs them in the contract. +- X5: blocked on the frozen-codebook witness. +- X12: Soufflé `eqrel` closure (D-RU-6 Lab). +- X13: no aggregate field until P2 decides. +- X22: FCA. +- U8: fused folds, owned by D-CMH-1. + +**SKIP:** +- X14: egg on the bind path breaks A1 (cost extraction is a planner). Offline normal-form + generation only, never per binding. +- Any per-row realization choice. + +**Smallest contract bridge, OPEN for the operator:** a defaulted +`ClassView::semantics(&self, ClassId) -> Option` returning `None` +(~10 lines), fail-closed like `epistemic_reading`. The class never sets `universe`, which is +bind-time. This makes the law reachable per class from an existing type, with no new type. + +## Not searched + +- worst-case-optimal joins; FAQ §5–8 (representation and tractability); +- algebraic specification / initial algebras; +- the PG GiST internals beyond the opclass chapter; +- the NarsTruth source; +- whether mask-risc `differential.rs` already covers P4. + +## Closeout + +`STATUS: open | OUTCOME: exploration map, 2 adopt-now test probes, 8 probes, 1 contract bridge proposal; one soundness gap named (M3) | OPEN: operator picks which to take forward` diff --git a/.claude/board/entries/README.md b/.claude/board/entries/README.md index ff5da6548..53acbe1a2 100644 --- a/.claude/board/entries/README.md +++ b/.claude/board/entries/README.md @@ -25,7 +25,7 @@ index row, (3) no duplicate entry id. Checks 1 and 2 are deliberately opposite directions; the stranding this convention prevents shows up in exactly one of them, never both. -411 entries, 2026-08-06 .. 2026-10-21. +413 entries, 2026-08-06 .. 2026-10-21. | date | entry id | finding | file | |---|---|---|---| @@ -43,6 +43,7 @@ exactly one of them, never both. | 2026-10-21 | `D-CMR-2b` | | [2026-10-21-d-cmr-2b-scratch-probe.md](2026-10-21-d-cmr-2b-scratch-probe.md) | | 2026-10-21 | `D-CMR-2` | | [2026-10-21-d-cmr-2-lowering.md](2026-10-21-d-cmr-2-lowering.md) | | 2026-10-21 | `D-RU-6` | | [2026-10-21-coresearch-ms-vs-ns-folding.md](2026-10-21-coresearch-ms-vs-ns-folding.md) | +| 2026-10-21 | `coresearch-classview-algebra-polyfill` | | [2026-10-21-coresearch-classview-algebra-polyfill.md](2026-10-21-coresearch-classview-algebra-polyfill.md) | | 2026-10-21 | `classview-dr-iam` | | [2026-10-21-classview-dr-iam.md](2026-10-21-classview-dr-iam.md) | | 2026-10-20 | `D-RU-1` | | [2026-10-20-u1-jc-batch.md](2026-10-20-u1-jc-batch.md) | | 2026-10-20 | `thought-git-replay-api-write-rubicon` | | [2026-10-20-thought-git-replay-api-write-rubicon.md](2026-10-20-thought-git-replay-api-write-rubicon.md) |