From d6eb436c9edaf0131ffb1d1888cd196bd2cb5f1a Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 21 Oct 2026 19:48:51 +0000 Subject: [PATCH 1/5] k3_pg_parity: Kleene oracle evaluates TRUE/FALSE planes and counts by popcount The reference evaluator held the table as Vec and evaluated each row separately. It now holds column lanes with validity planes and evaluates whole (T, F) bit planes; the count is a popcount over the T plane. Rendering the INSERT still reads lanes side by side, because that is the input given to PostgreSQL, not a comparison. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01G22yT6htkcdyXsihxxXdrg --- .../examples/k3_pg_parity.rs | 191 ++++++++++++------ 1 file changed, 128 insertions(+), 63 deletions(-) diff --git a/crates/lance-graph-contract/examples/k3_pg_parity.rs b/crates/lance-graph-contract/examples/k3_pg_parity.rs index 5ae63d409..369419f5b 100644 --- a/crates/lance-graph-contract/examples/k3_pg_parity.rs +++ b/crates/lance-graph-contract/examples/k3_pg_parity.rs @@ -5,7 +5,8 @@ //! PostgreSQL counts the rows each `WHERE` keeps for the expression as //! written, after [`normalize_kleene`], and after the two-valued //! [`normalize`]. A Kleene evaluator in this file counts the same rows -//! without PostgreSQL. Expected: as written = Kleene = PostgreSQL for every +//! without PostgreSQL: it evaluates whole TRUE/FALSE bit planes over the +//! column lanes and counts by popcount, never per row. Expected: as written = Kleene = PostgreSQL for every //! expression; the two-valued form differs on some. //! //! Needs `psql` and a server; connection from the usual `PG*` variables. @@ -36,24 +37,53 @@ impl Rng { } } -/// Columns: `a0`, `a1` nullable int; `u0` nullable int code; `s0`, `s1` -/// boolean, never NULL (a `Sym` is two-valued). -struct Row { - a: [Option; 2], - u: Option, - s: [bool; 2], +const WORDS: usize = ROWS.div_ceil(64); + +/// A column as lanes, never as rows: a value lane plus a validity plane per +/// nullable column (`a0`, `a1` int, `u0` int code) and a bit plane per +/// two-valued boolean column (`s0`, `s1`; a `Sym` is never NULL). +struct Table { + a: [Vec; 2], + a_valid: [[u64; WORDS]; 2], + u: Vec, + u_valid: [u64; WORDS], + s: [[u64; WORDS]; 2], } -fn rows(rng: &mut Rng) -> Vec { - let mut v = || (rng.below(4) != 0).then(|| rng.below(9) as i32 - 4); - let a: Vec<_> = (0..ROWS).map(|_| [v(), v()]).collect(); - a.into_iter() - .map(|a| Row { - a, - u: (rng.below(4) != 0).then(|| rng.below(3) as u32), - s: [rng.below(2) == 0, rng.below(2) == 0], - }) - .collect() +fn set(plane: &mut [u64; WORDS], i: usize) { + plane[i / 64] |= 1 << (i % 64); +} + +fn has(plane: &[u64; WORDS], i: usize) -> bool { + plane[i / 64] >> (i % 64) & 1 == 1 +} + +fn table(rng: &mut Rng) -> Table { + let mut t = Table { + a: [vec![0; ROWS], vec![0; ROWS]], + a_valid: [[0; WORDS]; 2], + u: vec![0; ROWS], + u_valid: [0; WORDS], + s: [[0; WORDS]; 2], + }; + for i in 0..ROWS { + for l in 0..2 { + if rng.below(4) != 0 { + t.a[l][i] = rng.below(9) as i32 - 4; + set(&mut t.a_valid[l], i); + } + } + if rng.below(4) != 0 { + t.u[i] = rng.below(3) as u32; + set(&mut t.u_valid, i); + } + for l in 0..2 { + if rng.below(2) == 0 { + set(&mut t.s[l], i); + } + } + } + t } fn leaf(rng: &mut Rng) -> SetExpr { @@ -130,69 +160,100 @@ fn sql(e: &SetExpr) -> String { } } -/// Kleene evaluation, by definition (`None` = UNKNOWN). -fn kleene(r: &Row, e: &SetExpr) -> Option { +/// Kleene evaluation as two planes: `(T, F)`, where a bit set in neither +/// plane is UNKNOWN. A leaf is one compare per lane slot into a mask; NOT +/// swaps the planes; AND / OR combine whole words. Nothing is decoded into +/// rows, and the oracle's answer is a popcount of the TRUE plane. +fn kleene(t: &Table, e: &SetExpr) -> ([u64; WORDS], [u64; WORDS]) { + let all = { + let mut p = [0u64; WORDS]; + (0..ROWS).for_each(|i| set(&mut p, i)); + p + }; match e { - SetExpr::All => Some(true), - SetExpr::Empty => Some(false), - SetExpr::Sym(i) => Some(r.s[*i as usize]), - SetExpr::Res(Res::I32 { lane, op, t }) => r.a[*lane as usize].map(|x| match op { - Cmp::Lt => x < *t, - Cmp::Le => x <= *t, - Cmp::Gt => x > *t, - Cmp::Ge => x >= *t, - Cmp::Eq => x == *t, - Cmp::Ne => x != *t, - }), - SetExpr::Res(Res::U32 { eq, v, .. }) => r.u.map(|x| (x == *v) == *eq), - SetExpr::Not(c) => kleene(r, c).map(|b| !b), - SetExpr::And(cs) => { - let mut unknown = false; - for c in cs { - match kleene(r, c) { - Some(false) => return Some(false), - None => unknown = true, - Some(_) => {} - } - } - (!unknown).then_some(true) + SetExpr::All => (all, [0; WORDS]), + SetExpr::Empty => ([0; WORDS], all), + SetExpr::Sym(i) => { + let s = t.s[*i as usize]; + (s, std::array::from_fn(|w| all[w] & !s[w])) } - SetExpr::Or(cs) => { - let mut unknown = false; - for c in cs { - match kleene(r, c) { - Some(true) => return Some(true), - None => unknown = true, - Some(_) => {} - } + SetExpr::Res(r) => { + let (mut tp, mut fp) = ([0u64; WORDS], [0u64; WORDS]); + for i in 0..ROWS { + let hit = match r { + Res::I32 { lane, op, t: c } => { + if !has(&t.a_valid[*lane as usize], i) { + continue; + } + let x = t.a[*lane as usize][i]; + match op { + Cmp::Lt => x < *c, + Cmp::Le => x <= *c, + Cmp::Gt => x > *c, + Cmp::Ge => x >= *c, + Cmp::Eq => x == *c, + Cmp::Ne => x != *c, + } + } + Res::U32 { eq, v, .. } => { + if !has(&t.u_valid, i) { + continue; + } + (t.u[i] == *v) == *eq + } + }; + set(if hit { &mut tp } else { &mut fp }, i); } - (!unknown).then_some(false) + (tp, fp) + } + SetExpr::Not(c) => { + let (tp, fp) = kleene(t, c); + (fp, tp) } + SetExpr::And(cs) => cs.iter().fold((all, [0; WORDS]), |(at, af), c| { + let (ct, cf) = kleene(t, c); + ( + std::array::from_fn(|w| at[w] & ct[w]), + std::array::from_fn(|w| af[w] | cf[w]), + ) + }), + SetExpr::Or(cs) => cs.iter().fold(([0; WORDS], all), |(at, af), c| { + let (ct, cf) = kleene(t, c); + ( + std::array::from_fn(|w| at[w] | ct[w]), + std::array::from_fn(|w| af[w] & cf[w]), + ) + }), } } -fn lit(v: Option) -> String { - v.map_or("NULL".into(), |x| x.to_string()) +fn lit(valid: bool, v: T) -> String { + if valid { + v.to_string() + } else { + "NULL".into() + } } fn main() { let mut rng = Rng(0x00D1_5EED_0003_0001); - let table = rows(&mut rng); + let table = table(&mut rng); let exprs: Vec = (0..EXPRS).map(|_| expr(&mut rng, 4)).collect(); let mut script = String::from( "CREATE TEMP TABLE t (a0 int, a1 int, u0 int, s0 bool, s1 bool);\nINSERT INTO t VALUES\n", ); - let values: Vec = table - .iter() - .map(|r| { + // Rendering the INSERT is the one place lanes are read side by side: it + // is the input handed to PostgreSQL, not a comparison. + let values: Vec = (0..ROWS) + .map(|i| { format!( "({}, {}, {}, {}, {})", - lit(r.a[0]), - lit(r.a[1]), - lit(r.u), - r.s[0], - r.s[1] + lit(has(&table.a_valid[0], i), table.a[0][i]), + lit(has(&table.a_valid[1], i), table.a[1][i]), + lit(has(&table.u_valid, i), table.u[i]), + has(&table.s[0], i), + has(&table.s[1], i) ) }) .collect(); @@ -252,7 +313,11 @@ fn main() { let (mut kleene_ok, mut two_valued_wrong, mut rewritten) = (0, 0, 0); for (e, line) in exprs.iter().zip(&lines) { let n: Vec = line.split(',').map(|x| x.parse().expect("count")).collect(); - let oracle = table.iter().filter(|r| kleene(r, e) == Some(true)).count(); + let oracle: usize = kleene(&table, e) + .0 + .iter() + .map(|w| w.count_ones() as usize) + .sum(); assert_eq!(n[0], oracle, "PostgreSQL vs the Kleene oracle: {e:?}"); assert_eq!(n[1], n[0], "normalize_kleene changed the count: {e:?}"); kleene_ok += 1; From 55c42b0671a3bcf95076783401800b52b806e5f5 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 21 Oct 2026 19:50:22 +0000 Subject: [PATCH 2/5] set_algebra, set_rules tests: Kleene oracles evaluate (T, F) masks Both reference evaluators recursed per row (Option at one row). They now evaluate whole TRUE/FALSE masks over the column lanes; the set_rules measures fold the TRUE mask against the multiplicity and value lanes in slot order. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01G22yT6htkcdyXsihxxXdrg --- .../lance-graph-contract/src/set_algebra.rs | 68 ++++++----- crates/lance-graph-contract/src/set_rules.rs | 112 ++++++++++-------- 2 files changed, 96 insertions(+), 84 deletions(-) diff --git a/crates/lance-graph-contract/src/set_algebra.rs b/crates/lance-graph-contract/src/set_algebra.rs index 513b49222..3435389e0 100644 --- a/crates/lance-graph-contract/src/set_algebra.rs +++ b/crates/lance-graph-contract/src/set_algebra.rs @@ -850,51 +850,55 @@ mod tests { [lane(), lane(), lane(), lane()] } - /// Kleene evaluation, by definition: `None` is UNKNOWN. A residual is - /// UNKNOWN where its lane is NULL; a `Sym` is two-valued. - fn kleene(w: &World, null: &[Vec; 4], e: &SetExpr, row: usize) -> Option { + /// Kleene evaluation as two masks, `(T, F)`: a row set in neither is + /// UNKNOWN. A residual is UNKNOWN where its lane is NULL; a `Sym` is + /// two-valued. NOT swaps the masks; AND / OR combine whole masks. The + /// oracle never evaluates an expression row by row. + fn kleene(w: &World, null: &[Vec; 4], e: &SetExpr) -> (Vec, Vec) { + let zip = |a: &[bool], b: &[bool], f: fn(bool, bool) -> bool| -> Vec { + a.iter().zip(b).map(|(x, y)| f(*x, *y)).collect() + }; match e { - SetExpr::All => Some(true), - SetExpr::Empty => Some(false), - SetExpr::Sym(i) => Some(w.syms[*i as usize][row]), + SetExpr::All => (vec![true; N], vec![false; N]), + SetExpr::Empty => (vec![false; N], vec![true; N]), + SetExpr::Sym(i) => { + let s = &w.syms[*i as usize]; + (s.clone(), s.iter().map(|b| !b).collect()) + } SetExpr::Res(r) => { let slot = match *r { Res::I32 { lane, .. } => lane as usize, Res::U32 { lane, .. } => 2 + lane as usize, }; - (!null[slot][row]).then(|| res_holds(w, *r, row)) + let holds = eval(w, e); + let known: Vec = null[slot].iter().map(|n| !n).collect(); + ( + zip(&holds, &known, |h, k| h && k), + zip(&holds, &known, |h, k| !h && k), + ) } - SetExpr::Not(c) => kleene(w, null, c, row).map(|b| !b), - SetExpr::And(cs) => { - let mut unknown = false; - for c in cs { - match kleene(w, null, c, row) { - Some(false) => return Some(false), - None => unknown = true, - Some(_) => {} - } - } - (!unknown).then_some(true) - } - SetExpr::Or(cs) => { - let mut unknown = false; - for c in cs { - match kleene(w, null, c, row) { - Some(true) => return Some(true), - None => unknown = true, - Some(_) => {} - } - } - (!unknown).then_some(false) + SetExpr::Not(c) => { + let (t, f) = kleene(w, null, c); + (f, t) } + SetExpr::And(cs) => cs + .iter() + .fold((vec![true; N], vec![false; N]), |(t, f), c| { + let (ct, cf) = kleene(w, null, c); + (zip(&t, &ct, |a, b| a && b), zip(&f, &cf, |a, b| a || b)) + }), + SetExpr::Or(cs) => cs + .iter() + .fold((vec![false; N], vec![true; N]), |(t, f), c| { + let (ct, cf) = kleene(w, null, c); + (zip(&t, &ct, |a, b| a || b), zip(&f, &cf, |a, b| a && b)) + }), } } /// The rows where `e` is TRUE: what a measure counts and `WHERE` keeps. fn true_rows(w: &World, null: &[Vec; 4], e: &SetExpr) -> Vec { - (0..N) - .map(|row| kleene(w, null, e, row) == Some(true)) - .collect() + kleene(w, null, e).0 } /// FAILS IF: `normalize_kleene` changes the rows where an expression is diff --git a/crates/lance-graph-contract/src/set_rules.rs b/crates/lance-graph-contract/src/set_rules.rs index e5627a94a..cc9f77bec 100644 --- a/crates/lance-graph-contract/src/set_rules.rs +++ b/crates/lance-graph-contract/src/set_rules.rs @@ -629,48 +629,53 @@ mod tests { } } - /// Kleene evaluation: `None` is UNKNOWN. - fn set_at(t: &Table, e: &SetExpr, row: usize) -> Option { + /// Kleene evaluation as two masks, `(T, F)`: a row set in neither is + /// UNKNOWN. A leaf is one compare per lane slot; NOT swaps the masks; + /// AND / OR combine whole masks. Never evaluated row by row. + fn set_planes(t: &Table, e: &SetExpr) -> (Vec, Vec) { + let n = t.n(); + let zip = |a: &[bool], b: &[bool], f: fn(bool, bool) -> bool| -> Vec { + a.iter().zip(b).map(|(x, y)| f(*x, *y)).collect() + }; + let leaf = |lane: u16, hit: &dyn Fn(i32) -> bool| -> (Vec, Vec) { + let l = &t.lanes[lane as usize]; + ( + l.iter().map(|v| v.is_some_and(hit)).collect(), + l.iter().map(|v| v.is_some_and(|x| !hit(x))).collect(), + ) + }; match e { - SetExpr::All => Some(true), - SetExpr::Empty => Some(false), - SetExpr::Sym(k) => Some(t.syms[*k as usize][row]), - SetExpr::Res(Res::I32 { lane, op, t: c }) => { - t.lanes[*lane as usize][row].map(|v| match op { - Cmp::Lt => v < *c, - Cmp::Le => v <= *c, - Cmp::Gt => v > *c, - Cmp::Ge => v >= *c, - Cmp::Eq => v == *c, - Cmp::Ne => v != *c, - }) + SetExpr::All => (vec![true; n], vec![false; n]), + SetExpr::Empty => (vec![false; n], vec![true; n]), + SetExpr::Sym(k) => { + let s = &t.syms[*k as usize]; + (s.clone(), s.iter().map(|b| !b).collect()) } - SetExpr::Res(Res::U32 { lane, eq, v }) => { - t.lanes[*lane as usize][row].map(|x| (x as u32 == *v) == *eq) - } - SetExpr::Not(c) => set_at(t, c, row).map(|b| !b), - SetExpr::And(cs) => { - let mut acc = Some(true); - for c in cs { - acc = match (acc, set_at(t, c, row)) { - (Some(false), _) | (_, Some(false)) => Some(false), - (Some(true), Some(true)) => Some(true), - _ => None, - }; - } - acc - } - SetExpr::Or(cs) => { - let mut acc = Some(false); - for c in cs { - acc = match (acc, set_at(t, c, row)) { - (Some(true), _) | (_, Some(true)) => Some(true), - (Some(false), Some(false)) => Some(false), - _ => None, - }; - } - acc + SetExpr::Res(Res::I32 { lane, op, t: c }) => leaf(*lane, &|v| match op { + Cmp::Lt => v < *c, + Cmp::Le => v <= *c, + Cmp::Gt => v > *c, + Cmp::Ge => v >= *c, + Cmp::Eq => v == *c, + Cmp::Ne => v != *c, + }), + SetExpr::Res(Res::U32 { lane, eq, v }) => leaf(*lane, &|x| (x as u32 == *v) == *eq), + SetExpr::Not(c) => { + let (tp, fp) = set_planes(t, c); + (fp, tp) } + SetExpr::And(cs) => cs + .iter() + .fold((vec![true; n], vec![false; n]), |(a, b), c| { + let (ct, cf) = set_planes(t, c); + (zip(&a, &ct, |x, y| x && y), zip(&b, &cf, |x, y| x || y)) + }), + SetExpr::Or(cs) => cs + .iter() + .fold((vec![false; n], vec![true; n]), |(a, b), c| { + let (ct, cf) = set_planes(t, c); + (zip(&a, &ct, |x, y| x || y), zip(&b, &cf, |x, y| x && y)) + }), } } @@ -725,22 +730,25 @@ mod tests { } } Measure::Count(s) => { - let mut v = fit(acc, 0); - for r in 0..t.n() { - if set_at(t, s, r) == Some(true) { - v = binop(acc, v, fit(acc, t.mult[r].into()), false); - } - } - v + // A fold over the TRUE mask, weighted by the multiplicity lane. + let keep = set_planes(t, s).0; + keep.iter() + .zip(&t.mult) + .filter(|(k, _)| **k) + .fold(fit(acc, 0), |v, (_, m)| { + binop(acc, v, fit(acc, (*m).into()), false) + }) } Measure::Sum { lane, set } => { + // A fold over the TRUE mask and the value lane; the + // accumulator steps once per multiplicity unit so an eager + // overflow surfaces where it would. + let keep = set_planes(t, set).0; let mut v = fit(acc, 0); - for r in 0..t.n() { - if set_at(t, set, r) == Some(true) { - if let Some(x) = t.lanes[*lane as usize][r] { - for _ in 0..t.mult[r] { - v = binop(acc, v, fit(acc, x.into()), false); - } + for ((k, x), m) in keep.iter().zip(&t.lanes[*lane as usize]).zip(&t.mult) { + if let (true, Some(x)) = (*k, x) { + for _ in 0..*m { + v = binop(acc, v, fit(acc, (*x).into()), false); } } } From 13607a49fddf7c62e8d9582902bb294b2cd0970c Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 21 Oct 2026 19:51:55 +0000 Subject: [PATCH 3/5] board: record the no-tuple oracle decision; withdraw the datafrog oracle role Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01G22yT6htkcdyXsihxxXdrg --- .../2026-10-21-coresearch-plans-mapping-foresight.md | 6 +++--- .../entries/2026-10-21-eval-egglog-datafrog-ascent.md | 7 +++++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/.claude/board/entries/2026-10-21-coresearch-plans-mapping-foresight.md b/.claude/board/entries/2026-10-21-coresearch-plans-mapping-foresight.md index 465351d51..651310f43 100644 --- a/.claude/board/entries/2026-10-21-coresearch-plans-mapping-foresight.md +++ b/.claude/board/entries/2026-10-21-coresearch-plans-mapping-foresight.md @@ -96,8 +96,8 @@ Bridge findings worth a probe: - **Quack `Filter::Semijoin` is a value-directed foreign gather** (`lance-graph-quack/src/lib.rs:152, 366-391`, lowering to `MaskOp::Gather`). ADMISSION B lists it as pending call-site classification. No `NOT IN`, `EXCEPT` or anti-join lowering may build on it until that classification lands. - **Oracle scope.** `CLAUDE.md` already rules that answers are compared by folds, "for tests and oracles too". So every oracle inside the workspace is bound. Some existing oracles walk decoded rows: - - `k3_pg_parity`'s Kleene reference evaluator; - - the K3 unit tests in `set_algebra` / `set_rules`; + - `k3_pg_parity`'s Kleene reference evaluator (converted to (T, F) masks 2026-10-21); + - the K3 unit tests in `set_algebra` / `set_rules` (converted 2026-10-21); - `join_probe.rs`; - `walk_mask.rs:361-372`. @@ -106,7 +106,7 @@ Bridge findings worth a probe: ## Open (operator) -1. Does "oracles compare by folds" admit a reference engine that holds tuples internally, if the comparison itself is fold against fold? This decides datafrog's role and the reference evaluators listed above. +1. ~~Does "oracles compare by folds" admit a reference engine that holds tuples internally?~~ **Answered 2026-10-21:** no (*"It's not smart to compare tuples they are a million times more expensive"*). The datafrog oracle role is withdrawn (see the eval entry's update). The three Kleene oracles named above now evaluate (T, F) masks. Still row-shaped and unconverted: `join_probe.rs` and `walk_mask.rs:361-372`. 2. Adopt the reconciled D-RFT table (A1) before anyone claims D-RFT-2/4/5 on EXO-50? 3. Who classifies `Semijoin` under ADMISSION B? diff --git a/.claude/board/entries/2026-10-21-eval-egglog-datafrog-ascent.md b/.claude/board/entries/2026-10-21-eval-egglog-datafrog-ascent.md index 1169ee8cd..b9c8165a4 100644 --- a/.claude/board/entries/2026-10-21-eval-egglog-datafrog-ascent.md +++ b/.claude/board/entries/2026-10-21-eval-egglog-datafrog-ascent.md @@ -2,6 +2,13 @@ **Status:** OPEN. This is an exploration map, not a ratification. Nothing in it is measured, and no code was written. The scope is a complement to `lance-graph-relalg`; replacing it is not on the table. +**Update 2026-10-21 (operator decision on oracle scope).** DECISION: *"It's not smart to compare tuples they are a million times more expensive."* SCOPE: every oracle in this workspace, including reference engines used only in tests. BASIS: the 2026-10-21 P0 rule, which already said "answers are compared by folds", now applied to the reference side too. Consequences for this entry: + +- **Role (b) is withdrawn for datafrog and Ascent.** Both hold their relations as owned tuples (§2), so neither can serve as an oracle. P-D1, P-A1 and P-A2 are SUPERSEDED. +- **The replacement oracle is an independent fold.** For D-RFT-3 / D-TGB-10 recursion: a second closure algorithm over the same mask planes (for example Warshall over a ≤256 tile, against iterated frontier masks), compared terminal against terminal. Independence comes from the algorithm, not from a second data model. This is a working model, not yet a probe. +- **P-E1 keeps its egglog half** (offline, counts only), but its Kleene check now runs against the mask oracle: the per-row `set_at` it names was replaced by `set_planes`, which evaluates (T, F) masks. +- Done in the same change: the three Kleene oracles (`k3_pg_parity`, `set_algebra` tests, `set_rules` tests) now evaluate whole TRUE/FALSE masks and count by popcount or fold. Each was disable-checked: a two-valued NOT turns its tests red. + **Evidence grades.** Claims about our own tree are marked `VERIFIED-IN-CODE`, with `file:line`. Claims from outside use these grades: - **READ-IN-FULL:** read completely. From b9e2ab9d309811e50c95ae0247a7cc0880588b7c Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 21 Oct 2026 20:07:27 +0000 Subject: [PATCH 4/5] quack: k3_where_pg_parity runs SQL WHERE through sql_where + mask-risc against PostgreSQL Restores the contract-side oracles and k3_pg_parity to main (no existing code replaced). Adds a Quack example that executes each expression through the engine that already exists: Filter::sql_where, lower and lower_fused, mask-risc execute_into with the Keep terminal. PostgreSQL returns the TRUE and the FALSE rows per expression; equality is mask-risc Xor followed by the Any terminal, on both masks. normalize_kleene is held to its WHERE-terminal contract: TRUE rows exact, FALSE-row movement reported. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01G22yT6htkcdyXsihxxXdrg --- .../examples/k3_pg_parity.rs | 191 +++----- .../lance-graph-contract/src/set_algebra.rs | 68 ++- crates/lance-graph-contract/src/set_rules.rs | 112 +++-- .../examples/k3_where_pg_parity.rs | 427 ++++++++++++++++++ 4 files changed, 574 insertions(+), 224 deletions(-) create mode 100644 crates/lance-graph-quack/examples/k3_where_pg_parity.rs diff --git a/crates/lance-graph-contract/examples/k3_pg_parity.rs b/crates/lance-graph-contract/examples/k3_pg_parity.rs index 369419f5b..5ae63d409 100644 --- a/crates/lance-graph-contract/examples/k3_pg_parity.rs +++ b/crates/lance-graph-contract/examples/k3_pg_parity.rs @@ -5,8 +5,7 @@ //! PostgreSQL counts the rows each `WHERE` keeps for the expression as //! written, after [`normalize_kleene`], and after the two-valued //! [`normalize`]. A Kleene evaluator in this file counts the same rows -//! without PostgreSQL: it evaluates whole TRUE/FALSE bit planes over the -//! column lanes and counts by popcount, never per row. Expected: as written = Kleene = PostgreSQL for every +//! without PostgreSQL. Expected: as written = Kleene = PostgreSQL for every //! expression; the two-valued form differs on some. //! //! Needs `psql` and a server; connection from the usual `PG*` variables. @@ -37,53 +36,24 @@ impl Rng { } } -const WORDS: usize = ROWS.div_ceil(64); - -/// A column as lanes, never as rows: a value lane plus a validity plane per -/// nullable column (`a0`, `a1` int, `u0` int code) and a bit plane per -/// two-valued boolean column (`s0`, `s1`; a `Sym` is never NULL). -struct Table { - a: [Vec; 2], - a_valid: [[u64; WORDS]; 2], - u: Vec, - u_valid: [u64; WORDS], - s: [[u64; WORDS]; 2], -} - -fn set(plane: &mut [u64; WORDS], i: usize) { - plane[i / 64] |= 1 << (i % 64); +/// Columns: `a0`, `a1` nullable int; `u0` nullable int code; `s0`, `s1` +/// boolean, never NULL (a `Sym` is two-valued). +struct Row { + a: [Option; 2], + u: Option, + s: [bool; 2], } -fn has(plane: &[u64; WORDS], i: usize) -> bool { - plane[i / 64] >> (i % 64) & 1 == 1 -} - -fn table(rng: &mut Rng) -> Table { - let mut t = Table { - a: [vec![0; ROWS], vec![0; ROWS]], - a_valid: [[0; WORDS]; 2], - u: vec![0; ROWS], - u_valid: [0; WORDS], - s: [[0; WORDS]; 2], - }; - for i in 0..ROWS { - for l in 0..2 { - if rng.below(4) != 0 { - t.a[l][i] = rng.below(9) as i32 - 4; - set(&mut t.a_valid[l], i); - } - } - if rng.below(4) != 0 { - t.u[i] = rng.below(3) as u32; - set(&mut t.u_valid, i); - } - for l in 0..2 { - if rng.below(2) == 0 { - set(&mut t.s[l], i); - } - } - } - t +fn rows(rng: &mut Rng) -> Vec { + let mut v = || (rng.below(4) != 0).then(|| rng.below(9) as i32 - 4); + let a: Vec<_> = (0..ROWS).map(|_| [v(), v()]).collect(); + a.into_iter() + .map(|a| Row { + a, + u: (rng.below(4) != 0).then(|| rng.below(3) as u32), + s: [rng.below(2) == 0, rng.below(2) == 0], + }) + .collect() } fn leaf(rng: &mut Rng) -> SetExpr { @@ -160,100 +130,69 @@ fn sql(e: &SetExpr) -> String { } } -/// Kleene evaluation as two planes: `(T, F)`, where a bit set in neither -/// plane is UNKNOWN. A leaf is one compare per lane slot into a mask; NOT -/// swaps the planes; AND / OR combine whole words. Nothing is decoded into -/// rows, and the oracle's answer is a popcount of the TRUE plane. -fn kleene(t: &Table, e: &SetExpr) -> ([u64; WORDS], [u64; WORDS]) { - let all = { - let mut p = [0u64; WORDS]; - (0..ROWS).for_each(|i| set(&mut p, i)); - p - }; +/// Kleene evaluation, by definition (`None` = UNKNOWN). +fn kleene(r: &Row, e: &SetExpr) -> Option { match e { - SetExpr::All => (all, [0; WORDS]), - SetExpr::Empty => ([0; WORDS], all), - SetExpr::Sym(i) => { - let s = t.s[*i as usize]; - (s, std::array::from_fn(|w| all[w] & !s[w])) - } - SetExpr::Res(r) => { - let (mut tp, mut fp) = ([0u64; WORDS], [0u64; WORDS]); - for i in 0..ROWS { - let hit = match r { - Res::I32 { lane, op, t: c } => { - if !has(&t.a_valid[*lane as usize], i) { - continue; - } - let x = t.a[*lane as usize][i]; - match op { - Cmp::Lt => x < *c, - Cmp::Le => x <= *c, - Cmp::Gt => x > *c, - Cmp::Ge => x >= *c, - Cmp::Eq => x == *c, - Cmp::Ne => x != *c, - } - } - Res::U32 { eq, v, .. } => { - if !has(&t.u_valid, i) { - continue; - } - (t.u[i] == *v) == *eq - } - }; - set(if hit { &mut tp } else { &mut fp }, i); + SetExpr::All => Some(true), + SetExpr::Empty => Some(false), + SetExpr::Sym(i) => Some(r.s[*i as usize]), + SetExpr::Res(Res::I32 { lane, op, t }) => r.a[*lane as usize].map(|x| match op { + Cmp::Lt => x < *t, + Cmp::Le => x <= *t, + Cmp::Gt => x > *t, + Cmp::Ge => x >= *t, + Cmp::Eq => x == *t, + Cmp::Ne => x != *t, + }), + SetExpr::Res(Res::U32 { eq, v, .. }) => r.u.map(|x| (x == *v) == *eq), + SetExpr::Not(c) => kleene(r, c).map(|b| !b), + SetExpr::And(cs) => { + let mut unknown = false; + for c in cs { + match kleene(r, c) { + Some(false) => return Some(false), + None => unknown = true, + Some(_) => {} + } } - (tp, fp) + (!unknown).then_some(true) } - SetExpr::Not(c) => { - let (tp, fp) = kleene(t, c); - (fp, tp) + SetExpr::Or(cs) => { + let mut unknown = false; + for c in cs { + match kleene(r, c) { + Some(true) => return Some(true), + None => unknown = true, + Some(_) => {} + } + } + (!unknown).then_some(false) } - SetExpr::And(cs) => cs.iter().fold((all, [0; WORDS]), |(at, af), c| { - let (ct, cf) = kleene(t, c); - ( - std::array::from_fn(|w| at[w] & ct[w]), - std::array::from_fn(|w| af[w] | cf[w]), - ) - }), - SetExpr::Or(cs) => cs.iter().fold(([0; WORDS], all), |(at, af), c| { - let (ct, cf) = kleene(t, c); - ( - std::array::from_fn(|w| at[w] | ct[w]), - std::array::from_fn(|w| af[w] & cf[w]), - ) - }), } } -fn lit(valid: bool, v: T) -> String { - if valid { - v.to_string() - } else { - "NULL".into() - } +fn lit(v: Option) -> String { + v.map_or("NULL".into(), |x| x.to_string()) } fn main() { let mut rng = Rng(0x00D1_5EED_0003_0001); - let table = table(&mut rng); + let table = rows(&mut rng); let exprs: Vec = (0..EXPRS).map(|_| expr(&mut rng, 4)).collect(); let mut script = String::from( "CREATE TEMP TABLE t (a0 int, a1 int, u0 int, s0 bool, s1 bool);\nINSERT INTO t VALUES\n", ); - // Rendering the INSERT is the one place lanes are read side by side: it - // is the input handed to PostgreSQL, not a comparison. - let values: Vec = (0..ROWS) - .map(|i| { + let values: Vec = table + .iter() + .map(|r| { format!( "({}, {}, {}, {}, {})", - lit(has(&table.a_valid[0], i), table.a[0][i]), - lit(has(&table.a_valid[1], i), table.a[1][i]), - lit(has(&table.u_valid, i), table.u[i]), - has(&table.s[0], i), - has(&table.s[1], i) + lit(r.a[0]), + lit(r.a[1]), + lit(r.u), + r.s[0], + r.s[1] ) }) .collect(); @@ -313,11 +252,7 @@ fn main() { let (mut kleene_ok, mut two_valued_wrong, mut rewritten) = (0, 0, 0); for (e, line) in exprs.iter().zip(&lines) { let n: Vec = line.split(',').map(|x| x.parse().expect("count")).collect(); - let oracle: usize = kleene(&table, e) - .0 - .iter() - .map(|w| w.count_ones() as usize) - .sum(); + let oracle = table.iter().filter(|r| kleene(r, e) == Some(true)).count(); assert_eq!(n[0], oracle, "PostgreSQL vs the Kleene oracle: {e:?}"); assert_eq!(n[1], n[0], "normalize_kleene changed the count: {e:?}"); kleene_ok += 1; diff --git a/crates/lance-graph-contract/src/set_algebra.rs b/crates/lance-graph-contract/src/set_algebra.rs index 3435389e0..513b49222 100644 --- a/crates/lance-graph-contract/src/set_algebra.rs +++ b/crates/lance-graph-contract/src/set_algebra.rs @@ -850,55 +850,51 @@ mod tests { [lane(), lane(), lane(), lane()] } - /// Kleene evaluation as two masks, `(T, F)`: a row set in neither is - /// UNKNOWN. A residual is UNKNOWN where its lane is NULL; a `Sym` is - /// two-valued. NOT swaps the masks; AND / OR combine whole masks. The - /// oracle never evaluates an expression row by row. - fn kleene(w: &World, null: &[Vec; 4], e: &SetExpr) -> (Vec, Vec) { - let zip = |a: &[bool], b: &[bool], f: fn(bool, bool) -> bool| -> Vec { - a.iter().zip(b).map(|(x, y)| f(*x, *y)).collect() - }; + /// Kleene evaluation, by definition: `None` is UNKNOWN. A residual is + /// UNKNOWN where its lane is NULL; a `Sym` is two-valued. + fn kleene(w: &World, null: &[Vec; 4], e: &SetExpr, row: usize) -> Option { match e { - SetExpr::All => (vec![true; N], vec![false; N]), - SetExpr::Empty => (vec![false; N], vec![true; N]), - SetExpr::Sym(i) => { - let s = &w.syms[*i as usize]; - (s.clone(), s.iter().map(|b| !b).collect()) - } + SetExpr::All => Some(true), + SetExpr::Empty => Some(false), + SetExpr::Sym(i) => Some(w.syms[*i as usize][row]), SetExpr::Res(r) => { let slot = match *r { Res::I32 { lane, .. } => lane as usize, Res::U32 { lane, .. } => 2 + lane as usize, }; - let holds = eval(w, e); - let known: Vec = null[slot].iter().map(|n| !n).collect(); - ( - zip(&holds, &known, |h, k| h && k), - zip(&holds, &known, |h, k| !h && k), - ) + (!null[slot][row]).then(|| res_holds(w, *r, row)) } - SetExpr::Not(c) => { - let (t, f) = kleene(w, null, c); - (f, t) + SetExpr::Not(c) => kleene(w, null, c, row).map(|b| !b), + SetExpr::And(cs) => { + let mut unknown = false; + for c in cs { + match kleene(w, null, c, row) { + Some(false) => return Some(false), + None => unknown = true, + Some(_) => {} + } + } + (!unknown).then_some(true) + } + SetExpr::Or(cs) => { + let mut unknown = false; + for c in cs { + match kleene(w, null, c, row) { + Some(true) => return Some(true), + None => unknown = true, + Some(_) => {} + } + } + (!unknown).then_some(false) } - SetExpr::And(cs) => cs - .iter() - .fold((vec![true; N], vec![false; N]), |(t, f), c| { - let (ct, cf) = kleene(w, null, c); - (zip(&t, &ct, |a, b| a && b), zip(&f, &cf, |a, b| a || b)) - }), - SetExpr::Or(cs) => cs - .iter() - .fold((vec![false; N], vec![true; N]), |(t, f), c| { - let (ct, cf) = kleene(w, null, c); - (zip(&t, &ct, |a, b| a || b), zip(&f, &cf, |a, b| a && b)) - }), } } /// The rows where `e` is TRUE: what a measure counts and `WHERE` keeps. fn true_rows(w: &World, null: &[Vec; 4], e: &SetExpr) -> Vec { - kleene(w, null, e).0 + (0..N) + .map(|row| kleene(w, null, e, row) == Some(true)) + .collect() } /// FAILS IF: `normalize_kleene` changes the rows where an expression is diff --git a/crates/lance-graph-contract/src/set_rules.rs b/crates/lance-graph-contract/src/set_rules.rs index cc9f77bec..e5627a94a 100644 --- a/crates/lance-graph-contract/src/set_rules.rs +++ b/crates/lance-graph-contract/src/set_rules.rs @@ -629,53 +629,48 @@ mod tests { } } - /// Kleene evaluation as two masks, `(T, F)`: a row set in neither is - /// UNKNOWN. A leaf is one compare per lane slot; NOT swaps the masks; - /// AND / OR combine whole masks. Never evaluated row by row. - fn set_planes(t: &Table, e: &SetExpr) -> (Vec, Vec) { - let n = t.n(); - let zip = |a: &[bool], b: &[bool], f: fn(bool, bool) -> bool| -> Vec { - a.iter().zip(b).map(|(x, y)| f(*x, *y)).collect() - }; - let leaf = |lane: u16, hit: &dyn Fn(i32) -> bool| -> (Vec, Vec) { - let l = &t.lanes[lane as usize]; - ( - l.iter().map(|v| v.is_some_and(hit)).collect(), - l.iter().map(|v| v.is_some_and(|x| !hit(x))).collect(), - ) - }; + /// Kleene evaluation: `None` is UNKNOWN. + fn set_at(t: &Table, e: &SetExpr, row: usize) -> Option { match e { - SetExpr::All => (vec![true; n], vec![false; n]), - SetExpr::Empty => (vec![false; n], vec![true; n]), - SetExpr::Sym(k) => { - let s = &t.syms[*k as usize]; - (s.clone(), s.iter().map(|b| !b).collect()) + SetExpr::All => Some(true), + SetExpr::Empty => Some(false), + SetExpr::Sym(k) => Some(t.syms[*k as usize][row]), + SetExpr::Res(Res::I32 { lane, op, t: c }) => { + t.lanes[*lane as usize][row].map(|v| match op { + Cmp::Lt => v < *c, + Cmp::Le => v <= *c, + Cmp::Gt => v > *c, + Cmp::Ge => v >= *c, + Cmp::Eq => v == *c, + Cmp::Ne => v != *c, + }) } - SetExpr::Res(Res::I32 { lane, op, t: c }) => leaf(*lane, &|v| match op { - Cmp::Lt => v < *c, - Cmp::Le => v <= *c, - Cmp::Gt => v > *c, - Cmp::Ge => v >= *c, - Cmp::Eq => v == *c, - Cmp::Ne => v != *c, - }), - SetExpr::Res(Res::U32 { lane, eq, v }) => leaf(*lane, &|x| (x as u32 == *v) == *eq), - SetExpr::Not(c) => { - let (tp, fp) = set_planes(t, c); - (fp, tp) + SetExpr::Res(Res::U32 { lane, eq, v }) => { + t.lanes[*lane as usize][row].map(|x| (x as u32 == *v) == *eq) + } + SetExpr::Not(c) => set_at(t, c, row).map(|b| !b), + SetExpr::And(cs) => { + let mut acc = Some(true); + for c in cs { + acc = match (acc, set_at(t, c, row)) { + (Some(false), _) | (_, Some(false)) => Some(false), + (Some(true), Some(true)) => Some(true), + _ => None, + }; + } + acc + } + SetExpr::Or(cs) => { + let mut acc = Some(false); + for c in cs { + acc = match (acc, set_at(t, c, row)) { + (Some(true), _) | (_, Some(true)) => Some(true), + (Some(false), Some(false)) => Some(false), + _ => None, + }; + } + acc } - SetExpr::And(cs) => cs - .iter() - .fold((vec![true; n], vec![false; n]), |(a, b), c| { - let (ct, cf) = set_planes(t, c); - (zip(&a, &ct, |x, y| x && y), zip(&b, &cf, |x, y| x || y)) - }), - SetExpr::Or(cs) => cs - .iter() - .fold((vec![false; n], vec![true; n]), |(a, b), c| { - let (ct, cf) = set_planes(t, c); - (zip(&a, &ct, |x, y| x || y), zip(&b, &cf, |x, y| x && y)) - }), } } @@ -730,25 +725,22 @@ mod tests { } } Measure::Count(s) => { - // A fold over the TRUE mask, weighted by the multiplicity lane. - let keep = set_planes(t, s).0; - keep.iter() - .zip(&t.mult) - .filter(|(k, _)| **k) - .fold(fit(acc, 0), |v, (_, m)| { - binop(acc, v, fit(acc, (*m).into()), false) - }) + let mut v = fit(acc, 0); + for r in 0..t.n() { + if set_at(t, s, r) == Some(true) { + v = binop(acc, v, fit(acc, t.mult[r].into()), false); + } + } + v } Measure::Sum { lane, set } => { - // A fold over the TRUE mask and the value lane; the - // accumulator steps once per multiplicity unit so an eager - // overflow surfaces where it would. - let keep = set_planes(t, set).0; let mut v = fit(acc, 0); - for ((k, x), m) in keep.iter().zip(&t.lanes[*lane as usize]).zip(&t.mult) { - if let (true, Some(x)) = (*k, x) { - for _ in 0..*m { - v = binop(acc, v, fit(acc, (*x).into()), false); + for r in 0..t.n() { + if set_at(t, set, r) == Some(true) { + if let Some(x) = t.lanes[*lane as usize][r] { + for _ in 0..t.mult[r] { + v = binop(acc, v, fit(acc, x.into()), false); + } } } } diff --git a/crates/lance-graph-quack/examples/k3_where_pg_parity.rs b/crates/lance-graph-quack/examples/k3_where_pg_parity.rs new file mode 100644 index 000000000..1d5af0b8d --- /dev/null +++ b/crates/lance-graph-quack/examples/k3_where_pg_parity.rs @@ -0,0 +1,427 @@ +//! D-RFT-0 / D-QSP-0: Quack's three-valued `WHERE` against PostgreSQL, row +//! for row. +//! +//! Random [`SetExpr`]s over a table with nullable lanes are mapped to Quack +//! [`Filter`]s and executed through the existing engine: [`Filter::sql_where`] +//! (TRUE / FALSE lowering over each column's validity plane), then [`lower`] +//! and [`lower_fused`], then mask-risc [`execute_into`] with the `Keep` +//! terminal. PostgreSQL is the independent semantic reference: for each +//! expression it returns the rows where the predicate is TRUE and the rows +//! where it is FALSE, as two bit strings in `id` order. +//! +//! Equality is checked on the masks, not on counts: two masks are equal when +//! mask-risc's `Xor` followed by the `Any` terminal says no bit differs. Both +//! the TRUE mask (`sql_where(e)`) and the FALSE mask (`sql_where(NOT e)`) are +//! compared, so an UNKNOWN row (in neither) cannot hide. `normalize_kleene(e)` +//! is held to its own contract, which is WHERE-terminal: its TRUE rows must +//! match exactly; how many expressions it moves FALSE rows on is reported, +//! because NOT over its result is refused, not promised. The two-valued `normalize(e)` must differ on some +//! expressions, which shows the generator reaches the rewrites `normalize_kleene` +//! exists to refuse. +//! +//! Needs `psql` and a server, connection from the usual `PG*` variables. +//! Without one it prints `SKIP` and exits 0, so nothing here gates CI. +//! +//! ```text +//! PGHOST=/var/tmp/k3pg PGPORT=54329 PGUSER=k3 PGDATABASE=postgres \ +//! cargo run --release -p lance-graph-quack --example k3_where_pg_parity +//! ``` + +use lance_graph_contract::set_algebra::{normalize, normalize_kleene, Cmp as SetCmp, Res, SetExpr}; +use lance_graph_mask_risc::{ + execute_into, words_for, Foreign, LaneRef, MaskOp, Operand, Out, Planes, Program, Scratch, + Terminal, Value, +}; +use lance_graph_quack::{lower, lower_fused, Agg, Cmp, Col, Filter, Mask, Query}; +use std::io::Write; +use std::process::{Command, Stdio}; + +const ROWS: usize = 200; +const EXPRS: usize = 2000; + +// Value lanes: a0, a1 (i32), u0 (u32). Mask planes: validity of a0, a1, u0; +// the two symbol columns; the all-rows plane that `All` reads. +const A0: Col = Col(0); +const A1: Col = Col(1); +const U0: Col = Col(2); +const VA0: Mask = Mask(0); +const VA1: Mask = Mask(1); +const VU0: Mask = Mask(2); +const S0: u16 = 3; +const ALL: Mask = Mask(5); +const NULLABLE: [(Col, Mask); 3] = [(A0, VA0), (A1, VA1), (U0, VU0)]; + +struct Rng(u64); +impl Rng { + fn next(&mut self) -> u64 { + self.0 ^= self.0 << 13; + self.0 ^= self.0 >> 7; + self.0 ^= self.0 << 17; + self.0 + } + fn below(&mut self, n: u64) -> u64 { + self.next() % n + } +} + +/// Resident lanes and planes. At a NULL row the value lane holds a payload +/// some predicate tests for (`0`, `1`), so an engine that reads the payload of +/// a NULL row is caught. +struct Table { + a: [Vec; 2], + u: Vec, + planes: [Vec; 6], +} + +fn set(p: &mut [u64], r: usize) { + p[r / 64] |= 1 << (r % 64); +} + +fn table(rng: &mut Rng) -> Table { + let w = words_for(ROWS); + let mut t = Table { + a: [vec![0; ROWS], vec![0; ROWS]], + u: vec![0; ROWS], + planes: std::array::from_fn(|_| vec![0; w]), + }; + for r in 0..ROWS { + for l in 0..2 { + if rng.below(4) != 0 { + t.a[l][r] = rng.below(9) as i32 - 4; + set(&mut t.planes[l], r); + } + } + if rng.below(4) != 0 { + t.u[r] = rng.below(3) as u32; + set(&mut t.planes[2], r); + } else { + t.u[r] = 1; + } + for s in 0..2 { + if rng.below(2) == 0 { + set(&mut t.planes[S0 as usize + s], r); + } + } + set(&mut t.planes[ALL.0 as usize], r); + } + t +} + +fn bit(p: &[u64], r: usize) -> bool { + p[r / 64] >> (r % 64) & 1 == 1 +} + +fn leaf(rng: &mut Rng) -> SetExpr { + const OPS: [SetCmp; 6] = [ + SetCmp::Lt, + SetCmp::Le, + SetCmp::Gt, + SetCmp::Ge, + SetCmp::Eq, + SetCmp::Ne, + ]; + match rng.below(5) { + 0 => SetExpr::Sym(rng.below(2) as u32), + 1 | 2 => SetExpr::i32( + rng.below(2) as u16, + OPS[rng.below(6) as usize], + rng.below(11) as i32 - 5, + ), + 3 => SetExpr::Res(Res::U32 { + lane: 0, + eq: rng.below(2) == 0, + v: rng.below(3) as u32, + }), + _ => SetExpr::i32(rng.below(2) as u16, SetCmp::Ge, i32::MIN), + } +} + +fn expr(rng: &mut Rng, depth: u32) -> SetExpr { + if depth == 0 || rng.below(4) == 0 { + return leaf(rng); + } + match rng.below(3) { + 0 => SetExpr::negated(expr(rng, depth - 1)), + k => { + let cs = (0..1 + rng.below(4)) + .map(|_| expr(rng, depth - 1)) + .collect(); + if k == 1 { + SetExpr::And(cs) + } else { + SetExpr::Or(cs) + } + } + } +} + +/// The contract expression as a Quack filter. A mapping only: every +/// operator below is one Quack already has. +fn filter(e: &SetExpr) -> Filter { + let all = || Filter::Plane(ALL); + match e { + SetExpr::All => all(), + SetExpr::Empty => Filter::Not(Box::new(all())), + SetExpr::Sym(i) => Filter::Plane(Mask(S0 + *i as u16)), + SetExpr::Res(Res::I32 { lane, op, t }) => Filter::Cmp( + if *lane == 0 { A0 } else { A1 }, + match op { + SetCmp::Lt => Cmp::LtI32(*t), + SetCmp::Le => Cmp::LeI32(*t), + SetCmp::Gt => Cmp::GtI32(*t), + SetCmp::Ge => Cmp::GeI32(*t), + SetCmp::Eq => Cmp::EqI32(*t), + SetCmp::Ne => Cmp::NeI32(*t), + }, + ), + SetExpr::Res(Res::U32 { eq, v, .. }) => { + Filter::Cmp(U0, if *eq { Cmp::EqU32(*v) } else { Cmp::NeU32(*v) }) + } + SetExpr::Not(c) => Filter::Not(Box::new(filter(c))), + // `AND()` is TRUE and `OR()` is FALSE, as in SQL's rendering below. + SetExpr::And(cs) if cs.is_empty() => all(), + SetExpr::Or(cs) if cs.is_empty() => Filter::Not(Box::new(all())), + SetExpr::And(cs) => Filter::And(cs.iter().map(filter).collect()), + SetExpr::Or(cs) => Filter::Or(cs.iter().map(filter).collect()), + } +} + +fn op_sql(op: SetCmp) -> &'static str { + match op { + SetCmp::Lt => "<", + SetCmp::Le => "<=", + SetCmp::Gt => ">", + SetCmp::Ge => ">=", + SetCmp::Eq => "=", + SetCmp::Ne => "<>", + } +} + +fn sql(e: &SetExpr) -> String { + let junction = |cs: &[SetExpr], sep: &str, unit: &str| { + if cs.is_empty() { + return unit.to_string(); + } + let parts: Vec = cs.iter().map(sql).collect(); + format!("({})", parts.join(sep)) + }; + match e { + SetExpr::All => "TRUE".into(), + SetExpr::Empty => "FALSE".into(), + SetExpr::Sym(i) => format!("s{i}"), + SetExpr::Res(Res::I32 { lane, op, t }) => { + format!("(a{lane} {} {}::bigint)", op_sql(*op), i64::from(*t)) + } + SetExpr::Res(Res::U32 { lane, eq, v }) => { + format!("(u{lane} {} {v})", if *eq { "=" } else { "<>" }) + } + SetExpr::Not(c) => format!("(NOT {})", sql(c)), + SetExpr::And(cs) => junction(cs, " AND ", "TRUE"), + SetExpr::Or(cs) => junction(cs, " OR ", "FALSE"), + } +} + +/// Which lowering of the engine to run. +#[derive(Clone, Copy)] +enum Mode { + InPlace, + Fused, +} + +/// The rows the engine keeps for `f` read as SQL `WHERE`: Quack's +/// `sql_where`, lowered, executed by mask-risc with the `Keep` terminal. +fn keep(t: &Table, f: &Filter, mode: Mode) -> Vec { + let q = Query { + filter: f.sql_where(&NULLABLE), + agg: Agg::Rows, + }; + let p = match mode { + Mode::InPlace => lower(&q), + Mode::Fused => lower_fused(&q), + } + .expect("lowers"); + let masks: Vec<&[u64]> = t.planes.iter().map(Vec::as_slice).collect(); + let lanes = [ + LaneRef::I32(&t.a[0]), + LaneRef::I32(&t.a[1]), + LaneRef::U32(&t.u), + ]; + let pl = Planes { + n_rows: ROWS, + masks: &masks, + lanes: &lanes, + }; + let mut out = vec![0u64; words_for(ROWS)]; + let mut s = Scratch::for_program(&p, ROWS).expect("carves"); + execute_into(&p, &pl, &Foreign::NONE, &mut s, Out::Mask(&mut out)).expect("runs"); + out +} + +/// Whether two masks differ in any row: mask-risc `Xor`, then `Any`. +fn differ(a: &[u64], b: &[u64]) -> bool { + let p = Program::new( + vec![MaskOp::Xor { + a: Operand::Plane(0), + b: Operand::Plane(1), + dst: 0, + }], + Terminal::Any { + mask: Operand::Scratch(0), + }, + ); + let pl = Planes { + n_rows: ROWS, + masks: &[a, b], + lanes: &[], + }; + let mut s = Scratch::for_program(&p, ROWS).expect("carves"); + match execute_into(&p, &pl, &Foreign::NONE, &mut s, Out::None).expect("runs") { + Value::Bool(any) => any, + v => panic!("Any returned {v:?}"), + } +} + +/// PostgreSQL's bit string (`'1'` per kept row, in `id` order) as a plane. +fn plane(bits: &str) -> Vec { + assert_eq!(bits.len(), ROWS, "one character per row"); + let mut p = vec![0u64; words_for(ROWS)]; + for (r, c) in bits.bytes().enumerate() { + if c == b'1' { + set(&mut p, r); + } + } + p +} + +fn lit(valid: bool, v: impl std::fmt::Display) -> String { + if valid { + v.to_string() + } else { + "NULL".into() + } +} + +fn psql(script: String) -> Result, String> { + let out = Command::new("psql") + .args(["-X", "-q", "-tA", "-F,", "-v", "ON_ERROR_STOP=1", "-f", "-"]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .and_then(|mut child| { + // psql answers while it reads: write on a separate thread so a + // full stdout pipe cannot block the writer. + let mut stdin = child.stdin.take().expect("piped stdin"); + let writer = std::thread::spawn(move || { + let _ = stdin.write_all(script.as_bytes()); + }); + let out = child.wait_with_output(); + let _ = writer.join(); + out + }) + .map_err(|e| format!("could not run psql ({e})"))?; + if out.status.success() { + Ok(out.stdout) + } else { + Err(format!( + "psql failed: {}", + String::from_utf8_lossy(&out.stderr).trim() + )) + } +} + +fn main() { + let mut rng = Rng(0x00D1_5EED_0003_0001); + let t = table(&mut rng); + let exprs: Vec = (0..EXPRS).map(|_| expr(&mut rng, 4)).collect(); + + // The INSERT is PostgreSQL's input: the one place the lanes are read + // side by side. + let mut script = String::from( + "CREATE TEMP TABLE t (id int, a0 int, a1 int, u0 int, s0 bool, s1 bool);\n\ + INSERT INTO t VALUES\n", + ); + let values: Vec = (0..ROWS) + .map(|r| { + format!( + "({r}, {}, {}, {}, {}, {})", + lit(bit(&t.planes[0], r), t.a[0][r]), + lit(bit(&t.planes[1], r), t.a[1][r]), + lit(bit(&t.planes[2], r), t.u[r]), + bit(&t.planes[S0 as usize], r), + bit(&t.planes[S0 as usize + 1], r), + ) + }) + .collect(); + script.push_str(&values.join(",\n")); + script.push_str(";\n"); + for e in &exprs { + let w = sql(e); + script.push_str(&format!( + "SELECT string_agg(CASE WHEN {w} THEN '1' ELSE '0' END, '' ORDER BY id), \ + string_agg(CASE WHEN NOT {w} THEN '1' ELSE '0' END, '' ORDER BY id) FROM t;\n" + )); + } + let stdout = match psql(script) { + Ok(o) => o, + Err(why) => { + println!("SKIP: {why}"); + return; + } + }; + let lines: Vec<&str> = std::str::from_utf8(&stdout) + .expect("utf-8") + .lines() + .filter(|l| !l.is_empty()) + .collect(); + assert_eq!(lines.len(), EXPRS, "one result row per expression"); + + let (mut rewritten, mut two_valued_wrong, mut unknown_rows, mut nk_false_moved) = + (0usize, 0usize, 0usize, 0usize); + for (e, line) in exprs.iter().zip(&lines) { + let (tbits, fbits) = line.split_once(',').expect("two columns"); + let (pg_t, pg_f) = (plane(tbits), plane(fbits)); + unknown_rows += ROWS - tbits.matches('1').count() - fbits.matches('1').count(); + let (f, nk) = (filter(e), filter(&normalize_kleene(e))); + for mode in [Mode::InPlace, Mode::Fused] { + let neg = Filter::Not(Box::new(f.clone())); + assert!( + !differ(&keep(&t, &f, mode), &pg_t), + "TRUE rows differ from PostgreSQL: {e:?}" + ); + assert!( + !differ(&keep(&t, &neg, mode), &pg_f), + "FALSE rows differ from PostgreSQL: {e:?}" + ); + // `normalize_kleene` is WHERE-terminal: it keeps the TRUE rows and + // promises nothing about the FALSE ones (NOT over it is refused). + assert!( + !differ(&keep(&t, &nk, mode), &pg_t), + "normalize_kleene changed the TRUE rows: {e:?}" + ); + } + nk_false_moved += usize::from(differ( + &keep(&t, &Filter::Not(Box::new(nk.clone())), Mode::InPlace), + &pg_f, + )); + rewritten += usize::from(normalize_kleene(e) != *e); + two_valued_wrong += usize::from(differ( + &keep(&t, &filter(&normalize(e)), Mode::InPlace), + &pg_t, + )); + } + let version = Command::new("psql") + .args(["-X", "-tA", "-c", "SHOW server_version"]) + .output() + .map(|o| String::from_utf8_lossy(&o.stdout).trim().to_string()) + .unwrap_or_default(); + println!( + "PostgreSQL {version}: {EXPRS} expressions over {ROWS} rows ({unknown_rows} UNKNOWN \ + row-verdicts). Quack sql_where -> lower / lower_fused -> mask-risc keeps PostgreSQL's \ + TRUE and FALSE rows exactly as written; normalize_kleene ({rewritten} rewritten) keeps \ + the TRUE rows on all and moves FALSE rows on {nk_false_moved} (WHERE-terminal). \ + Two-valued normalize differs on {two_valued_wrong}." + ); + assert!(unknown_rows > 0, "the table never produced UNKNOWN"); + assert!(two_valued_wrong > 0, "the run never reached a lane cover"); +} From ffd7e611c4d312a67b49c63074e8cc89dab7473a Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 21 Oct 2026 20:07:47 +0000 Subject: [PATCH 5/5] board: the PostgreSQL differential runs through Quack and mask-risc Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01G22yT6htkcdyXsihxxXdrg --- .../2026-10-21-coresearch-plans-mapping-foresight.md | 6 +++--- .../entries/2026-10-21-eval-egglog-datafrog-ascent.md | 8 ++++++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.claude/board/entries/2026-10-21-coresearch-plans-mapping-foresight.md b/.claude/board/entries/2026-10-21-coresearch-plans-mapping-foresight.md index 651310f43..dc7f06710 100644 --- a/.claude/board/entries/2026-10-21-coresearch-plans-mapping-foresight.md +++ b/.claude/board/entries/2026-10-21-coresearch-plans-mapping-foresight.md @@ -96,8 +96,8 @@ Bridge findings worth a probe: - **Quack `Filter::Semijoin` is a value-directed foreign gather** (`lance-graph-quack/src/lib.rs:152, 366-391`, lowering to `MaskOp::Gather`). ADMISSION B lists it as pending call-site classification. No `NOT IN`, `EXCEPT` or anti-join lowering may build on it until that classification lands. - **Oracle scope.** `CLAUDE.md` already rules that answers are compared by folds, "for tests and oracles too". So every oracle inside the workspace is bound. Some existing oracles walk decoded rows: - - `k3_pg_parity`'s Kleene reference evaluator (converted to (T, F) masks 2026-10-21); - - the K3 unit tests in `set_algebra` / `set_rules` (converted 2026-10-21); + - `k3_pg_parity`'s Kleene reference evaluator; + - the K3 unit tests in `set_algebra` / `set_rules`; - `join_probe.rs`; - `walk_mask.rs:361-372`. @@ -106,7 +106,7 @@ Bridge findings worth a probe: ## Open (operator) -1. ~~Does "oracles compare by folds" admit a reference engine that holds tuples internally?~~ **Answered 2026-10-21:** no (*"It's not smart to compare tuples they are a million times more expensive"*). The datafrog oracle role is withdrawn (see the eval entry's update). The three Kleene oracles named above now evaluate (T, F) masks. Still row-shaped and unconverted: `join_probe.rs` and `walk_mask.rs:361-372`. +1. ~~Does "oracles compare by folds" admit a reference engine that holds tuples internally?~~ **Answered 2026-10-21:** no (*"It's not smart to compare tuples they are a million times more expensive"*). The datafrog oracle role is withdrawn (see the eval entry's update). The PostgreSQL differential now executes through Quack `sql_where` and mask-risc and compares TRUE and FALSE masks by `Xor` + `Any` (`lance-graph-quack/examples/k3_where_pg_parity.rs`). 2. Adopt the reconciled D-RFT table (A1) before anyone claims D-RFT-2/4/5 on EXO-50? 3. Who classifies `Semijoin` under ADMISSION B? diff --git a/.claude/board/entries/2026-10-21-eval-egglog-datafrog-ascent.md b/.claude/board/entries/2026-10-21-eval-egglog-datafrog-ascent.md index b9c8165a4..0148bbaa9 100644 --- a/.claude/board/entries/2026-10-21-eval-egglog-datafrog-ascent.md +++ b/.claude/board/entries/2026-10-21-eval-egglog-datafrog-ascent.md @@ -6,8 +6,12 @@ - **Role (b) is withdrawn for datafrog and Ascent.** Both hold their relations as owned tuples (§2), so neither can serve as an oracle. P-D1, P-A1 and P-A2 are SUPERSEDED. - **The replacement oracle is an independent fold.** For D-RFT-3 / D-TGB-10 recursion: a second closure algorithm over the same mask planes (for example Warshall over a ≤256 tile, against iterated frontier masks), compared terminal against terminal. Independence comes from the algorithm, not from a second data model. This is a working model, not yet a probe. -- **P-E1 keeps its egglog half** (offline, counts only), but its Kleene check now runs against the mask oracle: the per-row `set_at` it names was replaced by `set_planes`, which evaluates (T, F) masks. -- Done in the same change: the three Kleene oracles (`k3_pg_parity`, `set_algebra` tests, `set_rules` tests) now evaluate whole TRUE/FALSE masks and count by popcount or fold. Each was disable-checked: a two-valued NOT turns its tests red. +- **P-E1 keeps its egglog half** (offline, counts only). +- **The PostgreSQL differential runs through the engine.** `lance-graph-quack/examples/k3_where_pg_parity.rs`: + - Each expression is executed by Quack: `sql_where` → `lower` / `lower_fused` → mask-risc `execute_into` with `Keep`. + - PostgreSQL returns the TRUE and FALSE rows. + - Equality is mask-risc `Xor` then `Any`, on both masks, never on counts. + - No new evaluator, counting loop or equality helper is added. The contract-side oracles are unchanged. **Evidence grades.** Claims about our own tree are marked `VERIFIED-IN-CODE`, with `file:line`. Claims from outside use these grades: