From d8bfe8b2dda43b2867892e7f79bfa012d6a96927 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 17:59:35 +0000 Subject: [PATCH 1/6] D-SRAM-2 spike: crepe_plan! as a cold optimizer in front of relalg New excluded lab crate crates/crepe-plan-probe: - crepe_plan! (AdaWorldAPI/crepe 9fd5785) reports crepe's own checks, strata and left-to-right bound/free analysis as data; no HashMap index, HashSet store or fixpoint loop is generated. - logic: labels to ids; flat positive non-recursive rules over @input relations, everything else refused by name. - optimize: binding propagation (head unification with the demand, the adornment step of Souffle's magic sets), duplicate-atom and duplicate-rule elimination gated on duplicate-insensitive answers. - mork_api: each rule as a typed query_multi pattern; nested, repeated variable and cyclic shapes refused (D-MPS-9). - native: a star on one shared variable laid out by the D-MPS-8 table, joint mask = relalg And of presence planes, one mask-risc program; coordinate witness, domain fingerprint and generation checks. - reference: nested-loop evaluator over the unoptimized rules. relalg: run_with_program() returns the program the fold ran (additive; run and run_as_written unchanged, 13/13 relalg tests). 16 tests (A-M of the brief): answers equal crepe!, datafrog, MORK and the reference; set and bag kept apart; refusals by name; no foreign hop. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01VncKoyG6JkpsoqhqTGTv3f --- Cargo.toml | 4 + crates/crepe-plan-probe/.cargo/config.toml | 5 + crates/crepe-plan-probe/Cargo.toml | 34 + crates/crepe-plan-probe/src/lib.rs | 32 + crates/crepe-plan-probe/src/logic.rs | 214 +++++++ crates/crepe-plan-probe/src/mork_api.rs | 120 ++++ crates/crepe-plan-probe/src/native.rs | 410 ++++++++++++ crates/crepe-plan-probe/src/optimize.rs | 224 +++++++ crates/crepe-plan-probe/src/pipeline.rs | 161 +++++ crates/crepe-plan-probe/src/plan.rs | 59 ++ crates/crepe-plan-probe/src/programs.rs | 118 ++++ crates/crepe-plan-probe/src/reference.rs | 64 ++ crates/crepe-plan-probe/tests/pipeline.rs | 706 +++++++++++++++++++++ crates/lance-graph-relalg/src/lib.rs | 2 +- crates/lance-graph-relalg/src/lower.rs | 33 +- 15 files changed, 2179 insertions(+), 7 deletions(-) create mode 100644 crates/crepe-plan-probe/.cargo/config.toml create mode 100644 crates/crepe-plan-probe/Cargo.toml create mode 100644 crates/crepe-plan-probe/src/lib.rs create mode 100644 crates/crepe-plan-probe/src/logic.rs create mode 100644 crates/crepe-plan-probe/src/mork_api.rs create mode 100644 crates/crepe-plan-probe/src/native.rs create mode 100644 crates/crepe-plan-probe/src/optimize.rs create mode 100644 crates/crepe-plan-probe/src/pipeline.rs create mode 100644 crates/crepe-plan-probe/src/plan.rs create mode 100644 crates/crepe-plan-probe/src/programs.rs create mode 100644 crates/crepe-plan-probe/src/reference.rs create mode 100644 crates/crepe-plan-probe/tests/pipeline.rs diff --git a/Cargo.toml b/Cargo.toml index d23ff96a4..6f5de67e4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -170,6 +170,10 @@ exclude = [ # on the AdaWorldAPI forks, results only (D-ASM-10). Run from its directory # so its `.cargo/config.toml` applies: `cd crates/mork-pathmap-probe && cargo test`. "crates/mork-pathmap-probe", + # Crepe-plan spike (D-SRAM-2 research, lab): crepe_plan! -> binding propagation + # -> MORK query_multi -> relalg over the D-MPS-8 table. Own workspace; run + # from its directory so `.cargo/config.toml` applies. + "crates/crepe-plan-probe", # D-TGB-0 thought-git pre-registration: fixtures, the from-scratch oracle, # and signal checks over lance-graph-contract only. Lab probe; run from # its directory: `cd crates/thought-git-probe && cargo test`. diff --git a/crates/crepe-plan-probe/.cargo/config.toml b/crates/crepe-plan-probe/.cargo/config.toml new file mode 100644 index 000000000..94aff8253 --- /dev/null +++ b/crates/crepe-plan-probe/.cargo/config.toml @@ -0,0 +1,5 @@ +# MORK (through mork-pathmap-probe) depends on a gxhash fork that refuses to +# compile without AES/SSE2 intrinsics. Scoped to this probe; never set +# RUSTFLAGS in the environment. +[build] +rustflags = ["-C", "target-feature=+aes,+sse2"] diff --git a/crates/crepe-plan-probe/Cargo.toml b/crates/crepe-plan-probe/Cargo.toml new file mode 100644 index 000000000..b3a0fba2c --- /dev/null +++ b/crates/crepe-plan-probe/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "crepe-plan-probe" +version = "0.0.0" +edition = "2024" +publish = false +description = "Lab spike: Crepe's rule analysis as a cold optimizer in front of relalg. crepe_plan! reports the rules as data; binding propagation and set-only dedup rewrite them; a MORK query_multi pattern carries them; relalg ANDs the D-MPS-8 presence planes through mask-risc. Results only; no production path." + +# Its own workspace root, excluded from the lance-graph workspace, like +# mork-pathmap-probe (which it reuses for the shared-y relationship table and +# the MORK oracle) and holo-h14d-probe (relalg + mask-risc). Run from THIS +# directory so `.cargo/config.toml` applies: +# cd crates/crepe-plan-probe && cargo test +[workspace] + +[dependencies] +# Proc macros only: `crepe_plan!` (the plan) and, in the oracle module only, +# the unmodified `crepe!` evaluator. AdaWorldAPI fork, by branch (no pins). +crepe = { git = "https://github.com/AdaWorldAPI/crepe", branch = "ccr-d3ce1263-ied9yh" } +lance-graph-relalg = { path = "../lance-graph-relalg" } +lance-graph-mask-risc = { path = "../lance-graph-mask-risc" } +# The D-MPS-8 versioned relationship table (shared y domain, presence planes, +# generations) and the MORK/PathMap oracle behind boundary.rs. +mork-pathmap-probe = { path = "../mork-pathmap-probe" } + +[dev-dependencies] +datafrog = { git = "https://github.com/AdaWorldAPI/datafrog", branch = "master" } + +[patch."https://github.com/AdaWorldAPI/lance-graph"] +lance-graph-contract = { path = "../lance-graph-contract" } + +# Same reason as in mork-pathmap-probe: MORK's workspace names PathMap by a +# path outside its repository, so the git dependency needs this to resolve. +[patch."https://github.com/AdaWorldAPI/MORK"] +pathmap = { git = "https://github.com/AdaWorldAPI/PathMap", rev = "ef02591" } diff --git a/crates/crepe-plan-probe/src/lib.rs b/crates/crepe-plan-probe/src/lib.rs new file mode 100644 index 000000000..15bf6bebf --- /dev/null +++ b/crates/crepe-plan-probe/src/lib.rs @@ -0,0 +1,32 @@ +//! Lab spike: Crepe's rule analysis as a cold optimizer in front of relalg. +//! +//! STATUS: PROBE (results only). Not wired into Quack, R2IL or any production +//! path. Plan: `.claude/plans/crepe-rule-optimizer-spike-v1.md`. +//! +//! The pipeline, one stage per module: +//! +//! 1. `crepe_plan!` (AdaWorldAPI/crepe fork) checks the rules exactly as +//! `crepe!` does (names, arity, strata, negation cycles) and reports crepe's +//! own left-to-right bound/free analysis as data ([`plan`]). No HashMap +//! index, HashSet store or fixpoint loop is generated. +//! 2. [`logic`]: labels to ids; the admitted subset (flat, positive, +//! non-recursive, over `@input` relations) or a named refusal. +//! 3. [`optimize`]: binding propagation, then duplicate-atom and duplicate-rule +//! elimination where the answer is duplicate-insensitive. +//! 4. [`mork_api`]: each rule as a MORK `query_multi` pattern (D-MPS-9 shape), +//! admitted or refused by name. +//! 5. [`native`]: the star read out of the pattern, laid out by the D-MPS-8 +//! relationship table, the joint mask computed by relalg (`And` of presence +//! planes) as one mask-risc program; terminals read the mask. +//! 6. [`reference`]: an independent nested-loop evaluator over the resolved, +//! unoptimized rules. The unmodified `crepe!` evaluator, datafrog and MORK +//! are the other oracles (tests only). + +pub mod logic; +pub mod mork_api; +pub mod native; +pub mod optimize; +pub mod pipeline; +pub mod plan; +pub mod programs; +pub mod reference; diff --git a/crates/crepe-plan-probe/src/logic.rs b/crates/crepe-plan-probe/src/logic.rs new file mode 100644 index 000000000..1593253b2 --- /dev/null +++ b/crates/crepe-plan-probe/src/logic.rs @@ -0,0 +1,214 @@ +//! The resolved rules: relation and variable ids, no strings. Built once from a +//! [`Plan`] through the label table, then rewritten by [`crate::optimize`]. +//! +//! The admitted subset is flat, positive, non-recursive conjunctions of atoms +//! over `@input` relations, with integer constants. Everything else is refused +//! by name at resolution. + +use crate::plan::{Plan, PlanClause, PlanField, PlanRelKind}; + +/// A relation id: its position in the label table. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct RelId(pub u16); + +/// A variable id, numbered by first occurrence within one rule. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct VarId(pub u16); + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum Term { + Var(VarId), + Const(u64), +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Atom { + pub rel: RelId, + pub terms: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Rule { + pub head: Atom, + pub body: Vec, +} + +/// The label table: relation names to ids, built from the plan's relations. +/// The probe's stand-in for the label codebook; nothing after resolution keeps +/// a name. +#[derive(Debug, Clone)] +pub struct Labels { + names: Vec<&'static str>, + arity: Vec, + input: Vec, +} + +impl Labels { + pub fn of(plan: &Plan) -> Labels { + Labels { + names: plan.relations.iter().map(|r| r.name).collect(), + arity: plan.relations.iter().map(|r| r.arity).collect(), + input: plan + .relations + .iter() + .map(|r| r.kind == PlanRelKind::Input) + .collect(), + } + } + + pub fn id(&self, name: &str) -> Option { + self.names + .iter() + .position(|n| *n == name) + .map(|i| RelId(i as u16)) + } + + /// The label, for diagnostics at the test boundary only. + pub fn name(&self, rel: RelId) -> &'static str { + self.names[rel.0 as usize] + } + + pub fn arity(&self, rel: RelId) -> usize { + self.arity[rel.0 as usize] + } + + pub fn is_input(&self, rel: RelId) -> bool { + self.input[rel.0 as usize] + } +} + +/// Why a rule is outside the admitted subset. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Refusal { + /// A condition, `let` or `for` clause. + NonAtomClause(&'static str), + /// A negated atom: needs an explicit universe; not in this spike. + Negation, + /// The rule reads its own stratum (recursion): needs the D-MTP-3 driver. + Recursive, + /// A body atom over a derived relation: needs rule unfolding, not in this + /// spike. + DerivedBodyRelation(&'static str), + /// A `ref` binding or a non-literal expression in a field. + UnsupportedField(&'static str), + /// A head variable the body does not bind. + UnboundHeadVar(&'static str), + /// A relation name the label table does not know. + UnknownRelation(&'static str), +} + +/// Resolves every rule of the plan: each one is a rule in the subset or a +/// named refusal. +pub fn resolve(plan: &Plan, labels: &Labels) -> Vec> { + plan.rules + .iter() + .map(|rule| { + if rule.recursive { + return Err(Refusal::Recursive); + } + let mut vars: Vec<&'static str> = Vec::new(); + let var = |name: &'static str, vars: &mut Vec<&'static str>| { + let i = vars.iter().position(|v| *v == name).unwrap_or_else(|| { + vars.push(name); + vars.len() - 1 + }); + Term::Var(VarId(i as u16)) + }; + let mut body = Vec::new(); + for clause in &rule.body { + let atom = match clause { + PlanClause::Unsupported(what) => return Err(Refusal::NonAtomClause(what)), + PlanClause::Atom(a) => a, + }; + if atom.negated { + return Err(Refusal::Negation); + } + let rel = labels + .id(atom.relation) + .ok_or(Refusal::UnknownRelation(atom.relation))?; + if !labels.is_input(rel) { + return Err(Refusal::DerivedBodyRelation(atom.relation)); + } + let mut terms = Vec::new(); + for f in &atom.fields { + terms.push(match *f { + PlanField::NewVar(n) | PlanField::BoundVar(n) => var(n, &mut vars), + // `_` binds nothing: a variable no other field names. + PlanField::Ignored => { + vars.push("_"); + Term::Var(VarId((vars.len() - 1) as u16)) + } + PlanField::Const(c) => Term::Const(c), + PlanField::Ref(n) | PlanField::Expr(n) => { + return Err(Refusal::UnsupportedField(n)); + } + }); + } + body.push(Atom { rel, terms }); + } + let head_rel = labels + .id(rule.goal.relation) + .ok_or(Refusal::UnknownRelation(rule.goal.relation))?; + let mut head_terms = Vec::new(); + for f in &rule.goal.fields { + head_terms.push(match *f { + PlanField::BoundVar(n) => { + let i = vars + .iter() + .position(|v| *v == n) + .ok_or(Refusal::UnboundHeadVar(n))?; + Term::Var(VarId(i as u16)) + } + PlanField::Const(c) => Term::Const(c), + PlanField::Ignored => return Err(Refusal::UnsupportedField("_")), + PlanField::NewVar(n) | PlanField::Ref(n) | PlanField::Expr(n) => { + return Err(Refusal::UnsupportedField(n)); + } + }); + } + Ok(Rule { + head: Atom { + rel: head_rel, + terms: head_terms, + }, + body, + }) + }) + .collect() +} + +impl Rule { + /// The same rule with variables renumbered by first occurrence (body + /// first, then head). Two rules that differ only in variable names are + /// equal after this. + pub fn canonical(&self) -> Rule { + let mut map: Vec<(VarId, VarId)> = Vec::new(); + let mut rn = |t: &Term| match *t { + Term::Const(c) => Term::Const(c), + Term::Var(v) => { + let n = match map.iter().find(|(a, _)| *a == v) { + Some(&(_, b)) => b, + None => { + let b = VarId(map.len() as u16); + map.push((v, b)); + b + } + }; + Term::Var(n) + } + }; + let body: Vec = self + .body + .iter() + .map(|a| Atom { + rel: a.rel, + terms: a.terms.iter().map(&mut rn).collect(), + }) + .collect(); + let head = Atom { + rel: self.head.rel, + terms: self.head.terms.iter().map(&mut rn).collect(), + }; + Rule { head, body } + } +} diff --git a/crates/crepe-plan-probe/src/mork_api.rs b/crates/crepe-plan-probe/src/mork_api.rs new file mode 100644 index 000000000..5a85f1bb5 --- /dev/null +++ b/crates/crepe-plan-probe/src/mork_api.rs @@ -0,0 +1,120 @@ +//! The logical interface: MORK's `query_multi (pattern, effect)` shape +//! (`kernel/src/space.rs`; D-MPS-9, §7c of the Schnellverschluss plan). An +//! optimized rule becomes one conjunctive pattern and an output template. No +//! MORK code is copied or called here; only the shape of the verb is matched. +//! MORK itself stays the oracle behind `mork_pathmap_probe::boundary`. +//! +//! The admitted subset is D-MPS-9's bounded first scope: flat patterns of +//! fixed arity whose variables are shared between atoms. Nested patterns, a +//! variable repeated inside one atom and cyclic bodies are refused by name. + +use lance_graph_relalg::join_body::is_alpha_acyclic; + +use crate::logic::{RelId, Rule, Term, VarId}; + +/// One S-expression element. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum Expr { + /// The relation symbol at the head of an atom. + Sym(RelId), + /// `$v`. + Var(VarId), + /// An integer constant. + Int(u64), + /// A sub-expression. Never produced by [`query_multi`]; it exists so a + /// caller's nested pattern can be refused instead of flattened. + Nested(Vec), +} + +/// `(rel a b …)`. +pub type Pattern = Vec; + +/// `query_multi`: every binding of the conjunction `patterns`, written through +/// `template`. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct QueryMulti { + pub patterns: Vec, + pub template: Pattern, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum MorkRefusal { + /// A pattern contains a sub-expression. + NestedPattern { pattern: usize }, + /// One atom names the same variable twice (`(R $x $x)`). + RepeatedVarInAtom { pattern: usize, var: VarId }, + /// The body hypergraph is cyclic (needs `EqU32ViaLane` / `RunsMeet`, + /// ratified but not authorized). + Cyclic, + /// A pattern does not start with a relation symbol. + NoRelation { pattern: usize }, +} + +/// The pattern of one rule. +pub fn query_multi(rule: &Rule) -> QueryMulti { + let term = |t: &Term| match *t { + Term::Var(v) => Expr::Var(v), + Term::Const(c) => Expr::Int(c), + }; + let atom = |rel: RelId, terms: &[Term]| -> Pattern { + std::iter::once(Expr::Sym(rel)) + .chain(terms.iter().map(term)) + .collect() + }; + QueryMulti { + patterns: rule.body.iter().map(|a| atom(a.rel, &a.terms)).collect(), + template: atom(rule.head.rel, &rule.head.terms), + } +} + +/// Refuses a pattern outside D-MPS-9's first scope. +pub fn admit(q: &QueryMulti) -> Result<(), MorkRefusal> { + let mut edges = Vec::new(); + for (i, p) in q.patterns.iter().enumerate() { + if !matches!(p.first(), Some(Expr::Sym(_))) { + return Err(MorkRefusal::NoRelation { pattern: i }); + } + let mut vars: Vec = Vec::new(); + for e in &p[1..] { + match e { + Expr::Nested(_) | Expr::Sym(_) => { + return Err(MorkRefusal::NestedPattern { pattern: i }); + } + Expr::Var(v) => { + if vars.contains(&v.0) { + return Err(MorkRefusal::RepeatedVarInAtom { + pattern: i, + var: *v, + }); + } + vars.push(v.0); + } + Expr::Int(_) => {} + } + } + edges.push(vars); + } + if !is_alpha_acyclic(&edges) { + return Err(MorkRefusal::Cyclic); + } + Ok(()) +} + +/// The pattern as MORK text, `(Parent 17 $1)`, for reports and test output +/// only. Nothing reads it back. +pub fn render(q: &QueryMulti, name: impl Fn(RelId) -> &'static str) -> String { + fn expr(e: &Expr, name: &dyn Fn(RelId) -> &'static str) -> String { + match e { + Expr::Sym(r) => name(*r).to_string(), + Expr::Var(v) => format!("${}", v.0), + Expr::Int(c) => c.to_string(), + Expr::Nested(es) => list(es, name), + } + } + fn list(es: &[Expr], name: &dyn Fn(RelId) -> &'static str) -> String { + let parts: Vec = es.iter().map(|e| expr(e, name)).collect(); + format!("({})", parts.join(" ")) + } + let pats: Vec = q.patterns.iter().map(|p| list(p, &name)).collect(); + format!("(, {}) -> {}", pats.join(" "), list(&q.template, &name)) +} diff --git a/crates/crepe-plan-probe/src/native.rs b/crates/crepe-plan-probe/src/native.rs new file mode 100644 index 000000000..c6566933a --- /dev/null +++ b/crates/crepe-plan-probe/src/native.rs @@ -0,0 +1,410 @@ +//! Native lowering of an admitted `query_multi` star onto the D-MPS-8 +//! versioned relationship table, through relalg into one mask-risc program. +//! +//! STATUS: PROBE. The table (`mork_pathmap_probe::join`) is measured but not +//! admitted (D-RMAP-1's binding membrane waits on EXO-46). Nothing here is a +//! production path. +//! +//! # The only shape lowered +//! +//! A star on one shared variable `c`: every pattern names `c`, and every other +//! variable occurs in exactly one pattern. Each pattern becomes one *side*, laid +//! out by the slot of `c` over the shared domain. A constant on the side's other +//! column restricts the rows the side is prepared from (cold, at preparation, +//! once per prepared table). Then +//! +//! ```text +//! joint = present_0 ∧ present_1 ∧ … ∧ present_k (relalg: And of Plane(i)) +//! ``` +//! +//! is the set of `c` values every side has. That mask is all relalg computes. +//! Existence and distinct-`c` counts read only it; tuple emission and bag +//! counts are a separate terminal that reads each side at the slot the mask +//! names, never at a position taken from another side's values. +//! +//! Anything else (a chain, a pattern of arity > 2, a side without `c`) is +//! refused by name. + +use lance_graph_mask_risc::{MaskOp, Planes, Pred, Program, Terminal, words_for}; +use lance_graph_relalg::{Answer, Measure, Query, Rel, RelError, Run, run_with_program}; +use mork_pathmap_probe::RelationView; +use mork_pathmap_probe::join::{JoinRefusal, JoinTable, Side, for_each_set}; + +use crate::logic::{RelId, VarId}; +use crate::mork_api::{Expr, QueryMulti}; + +/// Which rows of which relation a side holds, and how they are keyed. Two +/// sides are interchangeable only when their keys are equal. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct SideKey { + pub rel: RelId, + /// The column holding the shared variable (`0` for a unary relation). + pub key_col: u8, + /// `Some(k)`: only rows whose other column is `k`. + pub restrict: Option, + pub unary: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SidePlan { + pub key: SideKey, + /// The variable this side's other column binds, if the template uses it. + pub emit: Option, +} + +/// Where an output column comes from. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HeadTerm { + Const(u64), + /// The shared variable. + Center, + /// The value column of side `i`. + Side(usize), +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StarPlan { + pub center: VarId, + pub sides: Vec, + pub head: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum NativeRefusal { + /// No variable occurs in every pattern. + NoSharedVariable, + /// A variable other than the center occurs in two patterns (a chain or a + /// second join variable). + NotAStar { + var: VarId, + }, + /// The D-MPS-8 sides are binary or unary. + Arity { + pattern: usize, + arity: usize, + }, + /// A template variable no pattern binds. + UnboundTemplateVar(VarId), + /// The prepared table was built for other sides than this plan reads. + NoCoordinateWitness, + /// A side was laid out over another domain than the table's. + DomainMismatch { + side: usize, + }, + /// From the table: a stale generation, or rows outside the domain. + Table(JoinRefusal), + Relalg(RelError), +} + +/// Reads the star out of an admitted pattern. +pub fn star(q: &QueryMulti) -> Result { + let args: Vec<&[Expr]> = q.patterns.iter().map(|p| &p[1..]).collect(); + let rel = |i: usize| match q.patterns[i][0] { + Expr::Sym(r) => r, + _ => unreachable!("admitted patterns start with a symbol"), + }; + let vars_of = |a: &[Expr]| -> Vec { + a.iter() + .filter_map(|e| match e { + Expr::Var(v) => Some(*v), + _ => None, + }) + .collect() + }; + let first = vars_of(args.first().copied().unwrap_or(&[])); + let center = first + .into_iter() + .find(|v| args.iter().all(|a| vars_of(a).contains(v))) + .ok_or(NativeRefusal::NoSharedVariable)?; + let mut sides = Vec::new(); + for (i, a) in args.iter().enumerate() { + let key_col = a + .iter() + .position(|e| *e == Expr::Var(center)) + .expect("center") as u8; + let (restrict, emit, unary) = match a.len() { + 1 => (None, None, true), + 2 => match &a[1 - key_col as usize] { + Expr::Int(k) => (Some(*k), None, false), + Expr::Var(v) => (None, Some(*v), false), + _ => unreachable!("admitted patterns are flat"), + }, + n => { + return Err(NativeRefusal::Arity { + pattern: i, + arity: n, + }); + } + }; + sides.push(SidePlan { + key: SideKey { + rel: rel(i), + key_col, + restrict, + unary, + }, + emit, + }); + } + for (i, s) in sides.iter().enumerate() { + if let Some(v) = s.emit + && sides + .iter() + .enumerate() + .any(|(j, t)| j != i && t.emit == Some(v)) + { + return Err(NativeRefusal::NotAStar { var: v }); + } + } + let head = q.template[1..] + .iter() + .map(|e| match *e { + Expr::Int(c) => Ok(HeadTerm::Const(c)), + Expr::Var(v) if v == center => Ok(HeadTerm::Center), + Expr::Var(v) => sides + .iter() + .position(|s| s.emit == Some(v)) + .map(HeadTerm::Side) + .ok_or(NativeRefusal::UnboundTemplateVar(v)), + _ => unreachable!("templates are flat"), + }) + .collect::>()?; + Ok(StarPlan { + center, + sides, + head, + }) +} + +/// The relations a plan reads, by id, with their generations. +pub struct Store<'a> { + /// `(relation, rows, generation)`. A unary relation's rows are `[v, v]`. + pub relations: Vec<(RelId, &'a [[u32; 2]], u64)>, +} + +impl<'a> Store<'a> { + pub fn view(&self, rel: RelId) -> (&'a [[u32; 2]], u64) { + let (_, rows, g) = self + .relations + .iter() + .find(|(r, _, _)| *r == rel) + .expect("relation in store"); + (rows, *g) + } +} + +/// A relationship table prepared for one plan's sides, with the witness that +/// says which sides those are and which domain they share. +pub struct PreparedStar { + pub table: JoinTable, + pub keys: Vec, + /// Fingerprint of the domain every side is laid out over. + pub domain_fp: u64, + /// Rows laid out, over all sides (after restriction). + pub rows_prepared: usize, +} + +/// Lays the plan's sides out over one shared domain. A side's restriction is +/// applied to the rows it is prepared from. +pub fn prepare(plan: &StarPlan, store: &Store<'_>) -> PreparedStar { + let owned: Vec<(Vec<[u32; 2]>, u64)> = plan + .sides + .iter() + .map(|s| { + let (rows, g) = store.view(s.key.rel); + let other = 1 - s.key.key_col as usize; + let kept = rows + .iter() + .filter(|r| s.key.unary || s.key.restrict.is_none_or(|k| u64::from(r[other]) == k)) + .copied() + .collect(); + (kept, g) + }) + .collect(); + let inputs: Vec<(RelationView<'_>, usize)> = owned + .iter() + .zip(&plan.sides) + .map(|((rows, g), s)| { + ( + RelationView { + rows, + generation: *g, + }, + s.key.key_col as usize, + ) + }) + .collect(); + let table = JoinTable::prepare_star(&inputs); + PreparedStar { + domain_fp: fingerprint(&table), + keys: plan.sides.iter().map(|s| s.key).collect(), + rows_prepared: owned.iter().map(|(r, _)| r.len()).sum(), + table, + } +} + +/// FNV-1a over the domain's values: equal domains, equal fingerprints. +pub fn fingerprint(table: &JoinTable) -> u64 { + let mut h = 0xcbf2_9ce4_8422_2325_u64; + for slot in 0..table.domain.len() { + for b in table.domain.id(slot).to_le_bytes() { + h = (h ^ u64::from(b)).wrapping_mul(0x100_0000_01b3); + } + } + h ^ table.domain.len() as u64 +} + +impl PreparedStar { + /// Replaces side `i` with one prepared elsewhere. Refused unless that side + /// was laid out over this table's domain: equal slot numbers over two + /// different domains name different values. + pub fn graft( + &mut self, + i: usize, + side: Side, + key: SideKey, + side_domain_fp: u64, + ) -> Result<(), NativeRefusal> { + if side_domain_fp != self.domain_fp { + return Err(NativeRefusal::DomainMismatch { side: i }); + } + self.table.sides[i] = side; + self.keys[i] = key; + Ok(()) + } + + /// The coordinate witness: this table holds exactly the plan's sides. + pub fn witness(&self, plan: &StarPlan) -> Result<(), NativeRefusal> { + let wanted: Vec = plan.sides.iter().map(|s| s.key).collect(); + if wanted != self.keys { + return Err(NativeRefusal::NoCoordinateWitness); + } + Ok(()) + } +} + +/// What the native path answers. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum NativeAnswer { + Exists(bool), + Count(u64), + /// Output tuples with their multiplicities (all 1 under set semantics). + Bindings(Vec<(Vec, u64)>), +} + +/// The joint mask and its relalg run, after the witness and generation checks. +pub fn joint( + prep: &PreparedStar, + plan: &StarPlan, + current: &[u64], + measure: Measure, + keep: Option<&mut [u64]>, +) -> Result<(Run, Option), NativeRefusal> { + prep.witness(plan)?; + prep.table.check(current).map_err(NativeRefusal::Table)?; + let masks: Vec<&[u64]> = prep.table.sides.iter().map(|s| s.present()).collect(); + let planes = Planes { + n_rows: prep.table.domain.len(), + masks: &masks, + lanes: &[], + }; + let rel = Rel::And((0..masks.len() as u16).map(Rel::Plane).collect()); + run_with_program(&Query { rel, measure }, &planes, keep).map_err(NativeRefusal::Relalg) +} + +/// Number of distinct center values every side has: relalg's popcount of the +/// joint mask, no payload read. +pub fn distinct_centers( + prep: &PreparedStar, + plan: &StarPlan, + current: &[u64], +) -> Result { + match joint(prep, plan, current, Measure::Count, None)?.0.answer { + Answer::Count(n) => Ok(n), + other => unreachable!("Count measure answered {other:?}"), + } +} + +/// Tuples over the joint mask. Reads each side at the slot the mask names. +/// A side whose value the template does not use still multiplies the count of +/// every tuple under bag semantics: each of its rows at that slot is one more +/// derivation. +pub fn bindings( + prep: &PreparedStar, + plan: &StarPlan, + current: &[u64], + bag: bool, +) -> Result, u64)>, NativeRefusal> { + let n = prep.table.domain.len(); + let mut mask = vec![0u64; words_for(n)]; + joint(prep, plan, current, Measure::Keep, Some(&mut mask))?; + let mut out: Vec<(Vec, u64)> = Vec::new(); + for_each_set(&mask, n, |slot| { + let center = u64::from(prep.table.domain.id(slot)); + let mut tuples: Vec<(Vec>, u64)> = vec![(vec![None; plan.sides.len()], 1)]; + for (i, s) in plan.sides.iter().enumerate() { + let vals = prep.table.sides[i].at(slot); + if s.emit.is_some() { + tuples = tuples + .into_iter() + .flat_map(|(t, m)| { + vals.iter().map(move |&v| { + let mut t = t.clone(); + t[i] = Some(u64::from(v)); + (t, m) + }) + }) + .collect(); + } else { + let k = vals.len() as u64; + for t in &mut tuples { + t.1 *= k; + } + } + } + for (t, m) in tuples { + let row: Vec = plan + .head + .iter() + .map(|h| match *h { + HeadTerm::Const(c) => c, + HeadTerm::Center => center, + HeadTerm::Side(i) => t[i].expect("emitting side"), + }) + .collect(); + out.push((row, m)); + } + }); + out.sort(); + let mut merged: Vec<(Vec, u64)> = Vec::new(); + for (row, m) in out { + match merged.last_mut() { + Some((r, c)) if *r == row => *c += m, + _ => merged.push((row, m)), + } + } + if !bag { + for e in &mut merged { + e.1 = 1; + } + } + Ok(merged) +} + +/// True when the program contains an op that reads at a position taken from +/// another lane's values, or writes by one (the 2026-10-16 ruling): a +/// `Gather`, an `EqU32Via` predicate, a scatter terminal or `GroupSumViaI32`. +pub fn has_foreign_hop(p: &Program) -> bool { + let op_hops = p.ops.iter().any(|op| match op { + MaskOp::Gather { .. } => true, + MaskOp::Pred { pred, .. } => matches!(pred, Pred::EqU32Via { .. }), + _ => false, + }); + let terminal_hops = matches!( + p.terminal, + Terminal::ScatterOrU32 { .. } + | Terminal::ScatterCountU32 { .. } + | Terminal::GroupSumViaI32 { .. } + ); + op_hops || terminal_hops +} diff --git a/crates/crepe-plan-probe/src/optimize.rs b/crates/crepe-plan-probe/src/optimize.rs new file mode 100644 index 000000000..4bb1962bd --- /dev/null +++ b/crates/crepe-plan-probe/src/optimize.rs @@ -0,0 +1,224 @@ +//! The cold rewrites that run before anything reaches relalg. Each one is a +//! proven equivalence under the semantics it is gated on; none reads a row. +//! +//! 1. **Binding propagation** (the adornment step of Soufflé's magic-set +//! transformation, without the magic predicates: the spike has no +//! recursion, so the demand never has to travel through a derived +//! relation). A query that binds head positions is unified with each rule +//! head. A head constant that differs from the bound value makes the rule +//! unreachable; a head variable is replaced by the bound value everywhere +//! in the rule, so the constant reaches the body atoms. +//! 2. **Duplicate atom** (`A ∩ A = A`). Only where the answer is a set or an +//! existence: under bag semantics a repeated atom multiplies every +//! derivation, so removing it changes the counts. +//! 3. **Duplicate rule** (`A ∪ A = A`). Same gate, same reason: two equal +//! rules derive every answer twice. +//! +//! relalg's own rewrites (flattening, `All`/`Empty`, duplicate planes, +//! absorption, intervals) run later on the plane expression; nothing here +//! repeats them. + +use mork_pathmap_probe::Semantics; + +use crate::logic::{Atom, RelId, Rule, Term, VarId}; + +/// What the caller asks for. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ResultKind { + /// Whether any answer exists. + Exists, + /// How many answers, counted as a set or as a bag. + Count(Semantics), + /// The answers themselves, as a set or as a bag. + Bindings(Semantics), +} + +impl ResultKind { + /// True when duplicate derivations cannot change the answer. + pub fn duplicate_insensitive(self) -> bool { + matches!( + self, + ResultKind::Exists + | ResultKind::Count(Semantics::Set) + | ResultKind::Bindings(Semantics::Set) + ) + } +} + +/// A query: the goal relation and, per head position, the value it is bound +/// to (or `None`). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Demand { + pub goal: RelId, + pub bound: Vec>, +} + +/// Which rewrites run. Turning one off may change the plan shape and the cost, +/// never the answer. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Options { + pub propagate: bool, + pub dedup: bool, +} + +impl Options { + pub const ALL: Options = Options { + propagate: true, + dedup: true, + }; + pub const NONE: Options = Options { + propagate: false, + dedup: false, + }; +} + +/// One rewrite that happened, for the plan report. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Rewrite { + /// The rule's head constant at `pos` contradicts the bound value. + DroppedRule { rule: usize, pos: usize }, + /// The head variable at `pos` was replaced by the bound value. + BoundHeadVar { rule: usize, pos: usize, value: u64 }, + /// Body atom `atom` repeated an earlier one. + DuplicateAtom { rule: usize, atom: usize }, + /// The rule repeated an earlier rule. + DuplicateRule { rule: usize }, +} + +/// The rules to evaluate, plus what is still to be applied to the answers. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Optimized { + /// The rules for the goal, rewritten. Each keeps its source index. + pub rules: Vec<(usize, Rule)>, + /// Head positions still to be filtered after evaluation (what propagation + /// did not push into the rules). + pub post_filter: Vec>, + pub rewrites: Vec, +} + +/// Rewrites the rules for `demand`. +pub fn optimize(rules: &[Rule], demand: &Demand, kind: ResultKind, opts: Options) -> Optimized { + let mut rewrites = Vec::new(); + let mut out: Vec<(usize, Rule)> = Vec::new(); + for (i, rule) in rules.iter().enumerate() { + if rule.head.rel != demand.goal { + continue; + } + let mut rule = rule.clone(); + if opts.propagate { + match specialize(&rule, &demand.bound) { + None => { + let pos = conflict(&rule, &demand.bound).expect("a conflict"); + rewrites.push(Rewrite::DroppedRule { rule: i, pos }); + continue; + } + Some((r, bound)) => { + for (pos, value) in bound { + rewrites.push(Rewrite::BoundHeadVar { + rule: i, + pos, + value, + }); + } + rule = r; + } + } + } + if opts.dedup && kind.duplicate_insensitive() { + let mut kept: Vec = Vec::new(); + for (a, atom) in rule.body.iter().enumerate() { + if kept.contains(atom) { + rewrites.push(Rewrite::DuplicateAtom { rule: i, atom: a }); + } else { + kept.push(atom.clone()); + } + } + rule.body = kept; + } + out.push((i, rule)); + } + if opts.dedup && kind.duplicate_insensitive() { + let mut seen: Vec = Vec::new(); + out.retain(|(i, r)| { + let c = r.canonical(); + if seen.contains(&c) { + rewrites.push(Rewrite::DuplicateRule { rule: *i }); + false + } else { + seen.push(c); + true + } + }); + } + let post_filter = if opts.propagate { + vec![None; demand.bound.len()] + } else { + demand.bound.clone() + }; + Optimized { + rules: out, + post_filter, + rewrites, + } +} + +/// `None` when a head constant contradicts the demand; otherwise the rule with +/// every bound head variable replaced, and the replacements made. +fn specialize(rule: &Rule, bound: &[Option]) -> Option<(Rule, Vec<(usize, u64)>)> { + let mut subst: Vec<(VarId, u64)> = Vec::new(); + let mut made = Vec::new(); + for (pos, (t, b)) in rule.head.terms.iter().zip(bound).enumerate() { + let Some(d) = *b else { continue }; + match *t { + Term::Const(c) if c != d => return None, + Term::Const(_) => {} + Term::Var(v) => match subst.iter().find(|(w, _)| *w == v) { + // `Answer(x, x)` asked as `(1, 2)`: the head cannot match. + Some(&(_, prev)) if prev != d => return None, + Some(_) => {} + None => { + subst.push((v, d)); + made.push((pos, d)); + } + }, + } + } + let apply = |a: &Atom| Atom { + rel: a.rel, + terms: a + .terms + .iter() + .map(|t| match *t { + Term::Var(v) => subst + .iter() + .find(|(w, _)| *w == v) + .map_or(*t, |&(_, d)| Term::Const(d)), + c => c, + }) + .collect(), + }; + Some(( + Rule { + head: apply(&rule.head), + body: rule.body.iter().map(apply).collect(), + }, + made, + )) +} + +fn conflict(rule: &Rule, bound: &[Option]) -> Option { + let mut seen: Vec<(VarId, u64)> = Vec::new(); + for (pos, (t, b)) in rule.head.terms.iter().zip(bound).enumerate() { + let Some(d) = *b else { continue }; + match *t { + Term::Const(c) if c != d => return Some(pos), + Term::Var(v) => match seen.iter().find(|(w, _)| *w == v) { + Some(&(_, prev)) if prev != d => return Some(pos), + Some(_) => {} + None => seen.push((v, d)), + }, + _ => {} + } + } + None +} diff --git a/crates/crepe-plan-probe/src/pipeline.rs b/crates/crepe-plan-probe/src/pipeline.rs new file mode 100644 index 000000000..31c361031 --- /dev/null +++ b/crates/crepe-plan-probe/src/pipeline.rs @@ -0,0 +1,161 @@ +//! The stages wired together, with every intermediate product kept for the +//! report: rules, rewrites, MORK patterns, star plans, mask-risc programs. + +use std::collections::BTreeMap; +use std::time::{Duration, Instant}; + +use lance_graph_mask_risc::Program; +use mork_pathmap_probe::Semantics; + +use crate::logic::{RelId, Rule}; +use crate::mork_api::{MorkRefusal, QueryMulti, admit, query_multi}; +use crate::native::{NativeRefusal, PreparedStar, StarPlan, Store, bindings, joint, prepare, star}; +use crate::optimize::{Demand, Optimized, Options, ResultKind, optimize}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PipelineRefusal { + Mork { rule: usize, why: MorkRefusal }, + Native { rule: usize, why: NativeRefusal }, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Outcome { + Exists(bool), + Count(u64), + /// Sorted tuples with multiplicities (1 under set semantics). + Bindings(Vec<(Vec, u64)>), +} + +/// Costs, each measured on its own. +#[derive(Debug, Clone, Copy, Default)] +pub struct Costs { + pub optimize: Duration, + pub map_and_admit: Duration, + pub prepare: Duration, + pub execute: Duration, +} + +/// Everything one evaluation produced. +#[derive(Debug, Clone)] +pub struct Report { + pub optimized: Optimized, + pub patterns: Vec, + pub stars: Vec, + pub programs: Vec, + /// Rows laid out over all prepared tables. + pub rows_prepared: usize, + /// mask-risc programs run (0 for a rule relalg answered symbolically). + pub programs_run: usize, + pub costs: Costs, +} + +/// Rewrites, maps, prepares and runs `rules` for `demand`. +pub fn evaluate( + rules: &[Rule], + demand: &Demand, + kind: ResultKind, + opts: Options, + store: &Store<'_>, +) -> Result<(Outcome, Report), PipelineRefusal> { + let t = Instant::now(); + let optimized = optimize(rules, demand, kind, opts); + let optimize_cost = t.elapsed(); + + let t = Instant::now(); + let mut patterns = Vec::new(); + let mut stars = Vec::new(); + for (i, rule) in &optimized.rules { + let q = query_multi(rule); + admit(&q).map_err(|why| PipelineRefusal::Mork { rule: *i, why })?; + stars.push(star(&q).map_err(|why| PipelineRefusal::Native { rule: *i, why })?); + patterns.push(q); + } + let map_cost = t.elapsed(); + + let t = Instant::now(); + let prepared: Vec = stars.iter().map(|s| prepare(s, store)).collect(); + let prepare_cost = t.elapsed(); + + let t = Instant::now(); + let generations = |s: &StarPlan| -> Vec { + s.sides + .iter() + .map(|side| store.view(side.key.rel).1) + .collect() + }; + let native = |rule: usize, why| PipelineRefusal::Native { rule, why }; + let mut programs = Vec::new(); + let mut programs_run = 0; + let filtered = optimized.post_filter.iter().any(Option::is_some); + let outcome = if kind == ResultKind::Exists && !filtered { + // Mask-only terminal: no payload is read. + let mut any = false; + for ((i, _), (s, p)) in optimized.rules.iter().zip(stars.iter().zip(&prepared)) { + let (run, prog) = joint( + p, + s, + &generations(s), + lance_graph_relalg::Measure::Count, + None, + ) + .map_err(|w| native(*i, w))?; + programs_run += run.programs; + programs.extend(prog); + any |= run.answer != lance_graph_relalg::Answer::Count(0); + } + Outcome::Exists(any) + } else { + let mut bag: BTreeMap, u64> = BTreeMap::new(); + for ((i, _), (s, p)) in optimized.rules.iter().zip(stars.iter().zip(&prepared)) { + let (run, prog) = joint( + p, + s, + &generations(s), + lance_graph_relalg::Measure::Count, + None, + ) + .map_err(|w| native(*i, w))?; + programs_run += run.programs; + programs.extend(prog); + for (row, m) in bindings(p, s, &generations(s), true).map_err(|w| native(*i, w))? { + *bag.entry(row).or_insert(0) += m; + } + } + bag.retain(|row, _| { + row.iter() + .zip(&optimized.post_filter) + .all(|(v, f)| f.is_none_or(|f| f == *v)) + }); + match kind { + ResultKind::Exists => Outcome::Exists(!bag.is_empty()), + ResultKind::Count(Semantics::Set) => Outcome::Count(bag.len() as u64), + ResultKind::Count(Semantics::Bag) => Outcome::Count(bag.values().sum()), + ResultKind::Bindings(Semantics::Set) => { + Outcome::Bindings(bag.into_keys().map(|r| (r, 1)).collect()) + } + ResultKind::Bindings(Semantics::Bag) => Outcome::Bindings(bag.into_iter().collect()), + } + }; + let execute_cost = t.elapsed(); + + let report = Report { + rows_prepared: prepared.iter().map(|p| p.rows_prepared).sum(), + programs_run, + optimized, + patterns, + stars, + programs, + costs: Costs { + optimize: optimize_cost, + map_and_admit: map_cost, + prepare: prepare_cost, + execute: execute_cost, + }, + }; + Ok((outcome, report)) +} + +/// The goal's relation id, for building a [`Demand`]. +pub fn demand(goal: RelId, bound: Vec>) -> Demand { + Demand { goal, bound } +} diff --git a/crates/crepe-plan-probe/src/plan.rs b/crates/crepe-plan-probe/src/plan.rs new file mode 100644 index 000000000..4fac79823 --- /dev/null +++ b/crates/crepe-plan-probe/src/plan.rs @@ -0,0 +1,59 @@ +//! The shape `crepe_plan!` builds. TEMPORARY: a cold compiler representation +//! private to this probe, not a global rule IR. Who owns the rule IR is D-RA-2, +//! an open maintainer decision; this type takes no position on it. +//! +//! Names are source labels here. [`crate::resolve`] maps them to ids through the +//! label table once; nothing after resolution holds a string. + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Plan { + pub relations: Vec, + pub strata: Vec>, + pub rules: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PlanRelation { + pub name: &'static str, + pub arity: usize, + pub kind: PlanRelKind, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PlanRelKind { + Input, + Output, + Intermediate, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PlanRule { + pub goal: PlanAtom, + pub body: Vec, + pub stratum: usize, + pub recursive: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PlanAtom { + pub relation: &'static str, + pub negated: bool, + pub fields: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PlanClause { + Atom(PlanAtom), + Unsupported(&'static str), +} + +/// One field, as crepe's binding analysis classified it from left to right. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PlanField { + Ignored, + Ref(&'static str), + NewVar(&'static str), + BoundVar(&'static str), + Const(u64), + Expr(&'static str), +} diff --git a/crates/crepe-plan-probe/src/programs.rs b/crates/crepe-plan-probe/src/programs.rs new file mode 100644 index 000000000..8e925742e --- /dev/null +++ b/crates/crepe-plan-probe/src/programs.rs @@ -0,0 +1,118 @@ +//! The fixture programs. Each is written once as `crepe_plan!` (the plan this +//! probe optimizes) and, where it is used as an oracle, once more as the +//! unmodified `crepe!` evaluator with the same rules. + +/// `Answer(x, z)`: grandchildren of `x` through an allowed middle generation, +/// plus a sponsor branch that only ever answers for `x = 5`. +pub mod family { + use crate::plan::*; + use crepe::crepe_plan; + crepe_plan! { + @input + struct Parent(u32, u32); + @input + struct Allowed(u32); + @input + struct Sponsor(u32); + @output + struct Answer(u32, u32); + Answer(x, z) <- Parent(x, y), Parent(y, z), Allowed(y); + Answer(5, z) <- Sponsor(z); + } + + /// The same rules through crepe's own evaluator (test oracle only). + pub mod oracle { + use crepe::crepe; + crepe! { + @input + pub struct Parent(pub u32, pub u32); + @input + pub struct Allowed(pub u32); + @input + pub struct Sponsor(pub u32); + @output + pub struct Answer(pub u32, pub u32); + Answer(x, z) <- Parent(x, y), Parent(y, z), Allowed(y); + Answer(5, z) <- Sponsor(z); + } + } +} + +/// The `family` program with renamed variables and a duplicated atom and rule. +pub mod family_noisy { + use crate::plan::*; + use crepe::crepe_plan; + crepe_plan! { + @input + struct Parent(u32, u32); + @input + struct Allowed(u32); + @input + struct Sponsor(u32); + @output + struct Answer(u32, u32); + Answer(a, c) <- Parent(a, b), Allowed(b), Parent(b, c), Allowed(b); + Answer(p, q) <- Parent(p, m), Allowed(m), Parent(m, q); + Answer(5, w) <- Sponsor(w); + } +} + +/// The two-relation join MORK's oracle answers: `T(x, z) <- R(x, y), S(y, z)`. +pub mod two_way { + use crate::plan::*; + use crepe::crepe_plan; + crepe_plan! { + @input + struct R(u32, u32); + @input + struct S(u32, u32); + @output + struct T(u32, u32, u32); + T(x, y, z) <- R(x, y), S(y, z); + } +} + +/// A unary star `R(y), S(y), Q(y)`. +pub mod star3 { + use crate::plan::*; + use crepe::crepe_plan; + crepe_plan! { + @input + struct R(u32); + @input + struct S(u32); + @input + struct Q(u32); + @output + struct Hit(u32); + Hit(y) <- R(y), S(y), Q(y); + } +} + +/// Shapes the spike refuses. +pub mod refused { + use crate::plan::*; + use crepe::crepe_plan; + crepe_plan! { + @input + struct E(u32, u32); + @input + struct B(u32); + @output + struct Chain(u32, u32); + @output + struct Tri(u32); + @output + struct Loop(u32); + @output + struct Neg(u32); + @output + struct Reach(u32, u32); + Chain(a, d) <- E(a, b), E(b, c), E(c, d); + Tri(a) <- E(a, b), E(b, c), E(c, a); + Loop(a) <- E(a, a); + Neg(a) <- E(a, _), !B(a); + Reach(a, b) <- E(a, b); + Reach(a, c) <- E(a, b), Reach(b, c); + } +} diff --git a/crates/crepe-plan-probe/src/reference.rs b/crates/crepe-plan-probe/src/reference.rs new file mode 100644 index 000000000..636f35898 --- /dev/null +++ b/crates/crepe-plan-probe/src/reference.rs @@ -0,0 +1,64 @@ +//! Independent oracle: the resolved, UNOPTIMIZED rules evaluated by nested +//! loops over the raw rows, bag semantics. Shares no code with +//! [`crate::optimize`] or [`crate::native`]. + +use std::collections::BTreeMap; + +use crate::logic::{Atom, RelId, Rule, Term}; + +/// Bag of head tuples for every rule whose head is `goal`, then filtered by the +/// demand. +pub fn eval( + rules: &[Rule], + goal: RelId, + demand: &[Option], + rows: &dyn Fn(RelId) -> Vec>, +) -> BTreeMap, u64> { + let mut out = BTreeMap::new(); + for rule in rules.iter().filter(|r| r.head.rel == goal) { + let mut env: Vec> = vec![None; 64]; + walk(rule, 0, &mut env, rows, &mut out); + } + out.retain(|t, _| t.iter().zip(demand).all(|(v, d)| d.is_none_or(|d| d == *v))); + out +} + +fn walk( + rule: &Rule, + i: usize, + env: &mut Vec>, + rows: &dyn Fn(RelId) -> Vec>, + out: &mut BTreeMap, u64>, +) { + if i == rule.body.len() { + let t = rule + .head + .terms + .iter() + .map(|t| match *t { + Term::Const(c) => c, + Term::Var(v) => env[v.0 as usize].expect("head var bound"), + }) + .collect(); + *out.entry(t).or_insert(0) += 1; + return; + } + let Atom { rel, terms } = &rule.body[i]; + for row in rows(*rel) { + let saved = env.clone(); + let ok = terms.iter().zip(&row).all(|(t, &v)| match *t { + Term::Const(c) => c == v, + Term::Var(x) => match env[x.0 as usize] { + Some(b) => b == v, + None => { + env[x.0 as usize] = Some(v); + true + } + }, + }); + if ok { + walk(rule, i + 1, env, rows, out); + } + *env = saved; + } +} diff --git a/crates/crepe-plan-probe/tests/pipeline.rs b/crates/crepe-plan-probe/tests/pipeline.rs new file mode 100644 index 000000000..2612954b9 --- /dev/null +++ b/crates/crepe-plan-probe/tests/pipeline.rs @@ -0,0 +1,706 @@ +//! Tests A–M of the spike brief. Every answer is compared against the +//! independent nested-loop reference over the UNOPTIMIZED rules, and where the +//! shape allows, against crepe's own evaluator, datafrog and MORK. + +use std::collections::BTreeMap; + +use crepe_plan_probe::logic::{Labels, RelId, Rule, resolve}; +use crepe_plan_probe::mork_api::{Expr, MorkRefusal, QueryMulti, admit, query_multi, render}; +use crepe_plan_probe::native::{ + NativeRefusal, SideKey, Store, distinct_centers, fingerprint, has_foreign_hop, joint, prepare, + star, +}; +use crepe_plan_probe::optimize::{Demand, Options, ResultKind, Rewrite}; +use crepe_plan_probe::pipeline::{Outcome, PipelineRefusal, evaluate}; +use crepe_plan_probe::plan::Plan; +use crepe_plan_probe::programs::{family, family_noisy, refused, star3, two_way}; +use crepe_plan_probe::{logic, reference}; +use lance_graph_mask_risc::{MaskOp, Operand, Program, Terminal}; +use lance_graph_relalg::{Measure, Rel, normalize}; +use mork_pathmap_probe::join::{JoinRefusal, JoinTable}; +use mork_pathmap_probe::{Backend, JoinInput, RelationView, Semantics, bind, capability}; + +const SET: Semantics = Semantics::Set; +const BAG: Semantics = Semantics::Bag; + +/// The family fixture. `Allowed(2)` is stored twice: a bag of facts, so bag +/// answers through `y = 2` are counted twice. +struct Family { + parent: Vec<[u32; 2]>, + allowed: Vec<[u32; 2]>, + sponsor: Vec<[u32; 2]>, +} + +fn family_data() -> Family { + let mut parent = vec![ + [17, 1], + [17, 2], + [17, 3], + [1, 10], + [1, 11], + [2, 10], + [2, 12], + [3, 13], + [5, 1], + [4, 2], + ]; + // Unrelated families: they enlarge the shared domain, never the answer. + for i in 0..200u32 { + parent.push([1000 + i, 2000 + i]); + parent.push([2000 + i, 3000 + i]); + } + Family { + parent, + allowed: vec![[1, 1], [2, 2], [2, 2], [2000, 2000]], + sponsor: vec![[20, 20], [21, 21]], + } +} + +fn resolved(plan: &Plan) -> (Labels, Vec) { + let labels = Labels::of(plan); + let rules = resolve(plan, &labels) + .into_iter() + .map(|r| r.expect("admitted rule")) + .collect(); + (labels, rules) +} + +fn store<'a>(labels: &Labels, f: &'a Family, gens: [u64; 3]) -> Store<'a> { + Store { + relations: vec![ + (labels.id("Parent").unwrap(), &f.parent, gens[0]), + (labels.id("Allowed").unwrap(), &f.allowed, gens[1]), + (labels.id("Sponsor").unwrap(), &f.sponsor, gens[2]), + ], + } +} + +/// Raw rows of a relation at its declared arity, for the reference. +fn raw<'a>(labels: &'a Labels, store: &'a Store<'a>) -> impl Fn(RelId) -> Vec> + 'a { + move |rel| { + let (rows, _) = store.view(rel); + rows.iter() + .map(|r| { + r[..labels.arity(rel)] + .iter() + .map(|&v| u64::from(v)) + .collect() + }) + .collect() + } +} + +fn answer_demand(labels: &Labels, x: u64) -> Demand { + Demand { + goal: labels.id("Answer").unwrap(), + bound: vec![Some(x), None], + } +} + +fn reference_bag( + labels: &Labels, + rules: &[Rule], + store: &Store<'_>, + demand: &Demand, +) -> BTreeMap, u64> { + reference::eval(rules, demand.goal, &demand.bound, &raw(labels, store)) +} + +fn as_set(bag: &BTreeMap, u64>) -> Vec<(Vec, u64)> { + bag.keys().map(|k| (k.clone(), 1)).collect() +} + +fn bindings(o: Outcome) -> Vec<(Vec, u64)> { + match o { + Outcome::Bindings(b) => b, + other => panic!("expected bindings, got {other:?}"), + } +} + +/// crepe's own evaluator on the same rules, filtered to `x`. +fn crepe_oracle(f: &Family, x: u32) -> Vec<(Vec, u64)> { + use family::oracle::{Allowed, Answer, Crepe, Parent, Sponsor}; + let mut rt = Crepe::new(); + rt.extend(f.parent.iter().map(|&[a, b]| Parent(a, b))); + rt.extend(f.allowed.iter().map(|&[a, _]| Allowed(a))); + rt.extend(f.sponsor.iter().map(|&[a, _]| Sponsor(a))); + let (answers,) = rt.run(); + let mut out: Vec<(Vec, u64)> = answers + .into_iter() + .filter(|Answer(a, _)| *a == x) + .map(|Answer(a, b)| (vec![u64::from(a), u64::from(b)], 1)) + .collect(); + out.sort(); + out +} + +/// datafrog, for the family rule only (the sponsor branch never answers for +/// `x = 17`): `Answer(x, z) <- Parent(x, y), Parent(y, z), Allowed(y)`. +fn datafrog_family(f: &Family, x: u32) -> Vec<(Vec, u64)> { + use datafrog::Relation; + let by_y: Relation<(u32, u32)> = f.parent.iter().map(|&[a, b]| (b, a)).collect(); + let by_parent: Relation<(u32, u32)> = f.parent.iter().map(|&[a, b]| (a, b)).collect(); + let allowed: Relation<(u32, ())> = f.allowed.iter().map(|&[a, _]| (a, ())).collect(); + let xz: Relation<(u32, (u32, u32))> = + Relation::from_join(&by_y, &by_parent, |&y, &x, &z| (y, (x, z))); + let ans: Relation<(u32, u32)> = Relation::from_join(&xz, &allowed, |_, &(x, z), _| (x, z)); + ans.elements + .iter() + .filter(|(a, _)| *a == x) + .map(|&(a, b)| (vec![u64::from(a), u64::from(b)], 1)) + .collect() +} + +// --------------------------------------------------------------------------- +// A. Basic equivalence +// --------------------------------------------------------------------------- + +#[test] +fn a_answers_equal_every_oracle() { + let f = family_data(); + let (labels, rules) = resolved(&family::crepe_plan()); + let st = store(&labels, &f, [1, 1, 1]); + let d = answer_demand(&labels, 17); + let want = as_set(&reference_bag(&labels, &rules, &st, &d)); + assert_eq!( + want, + vec![(vec![17, 10], 1), (vec![17, 11], 1), (vec![17, 12], 1)], + "fixture answer" + ); + for opts in [Options::ALL, Options::NONE] { + let (o, _) = evaluate(&rules, &d, ResultKind::Bindings(SET), opts, &st).unwrap(); + assert_eq!(bindings(o), want, "native, {opts:?}"); + } + assert_eq!(crepe_oracle(&f, 17), want, "unmodified crepe!"); + assert_eq!(datafrog_family(&f, 17), want, "datafrog"); +} + +// --------------------------------------------------------------------------- +// B. Binding propagation +// --------------------------------------------------------------------------- + +#[test] +fn b_the_bound_constant_reaches_the_body_and_the_dead_branch_is_dropped() { + let f = family_data(); + let (labels, rules) = resolved(&family::crepe_plan()); + let st = store(&labels, &f, [1, 1, 1]); + let d = answer_demand(&labels, 17); + let (on, rep_on) = evaluate(&rules, &d, ResultKind::Bindings(SET), Options::ALL, &st).unwrap(); + let (off, rep_off) = + evaluate(&rules, &d, ResultKind::Bindings(SET), Options::NONE, &st).unwrap(); + + // Same answer. + assert_eq!(on, off); + // Plan with propagation: one rule, the constant in its first pattern, the + // sponsor branch dropped for its head constant 5. + assert!( + rep_on + .optimized + .rewrites + .contains(&Rewrite::DroppedRule { rule: 1, pos: 0 }) + ); + assert!(rep_on.optimized.rewrites.contains(&Rewrite::BoundHeadVar { + rule: 0, + pos: 0, + value: 17 + })); + assert_eq!(rep_on.optimized.rules.len(), 1); + let text = render(&rep_on.patterns[0], |r| labels.name(r)); + assert_eq!( + text, + "(, (Parent 17 $1) (Parent $1 $2) (Allowed $1)) -> (Answer 17 $2)" + ); + assert_eq!(rep_on.stars[0].sides[0].key.restrict, Some(17)); + assert_eq!(rep_on.optimized.post_filter, vec![None, None]); + // Without it: both rules evaluated, the demand applied to the output. + assert_eq!(rep_off.optimized.rules.len(), 2); + assert_eq!(rep_off.optimized.post_filter, vec![Some(17), None]); + assert_eq!(rep_off.stars[0].sides[0].key.restrict, None); + // The plan shape is cheaper with it: fewer rows laid out, fewer programs. + assert!( + rep_on.rows_prepared < rep_off.rows_prepared, + "{} vs {}", + rep_on.rows_prepared, + rep_off.rows_prepared + ); + assert!(rep_on.programs_run < rep_off.programs_run); +} + +/// The other demand keeps the sponsor branch and specializes the family rule +/// to `x = 5`. +#[test] +fn b_a_matching_head_constant_keeps_its_branch() { + let f = family_data(); + let (labels, rules) = resolved(&family::crepe_plan()); + let st = store(&labels, &f, [1, 1, 1]); + let d = answer_demand(&labels, 5); + let (o, rep) = evaluate(&rules, &d, ResultKind::Bindings(SET), Options::ALL, &st).unwrap(); + assert_eq!(rep.optimized.rules.len(), 2); + assert_eq!( + bindings(o), + as_set(&reference_bag(&labels, &rules, &st, &d)) + ); + assert_eq!( + crepe_oracle(&f, 5), + as_set(&reference_bag(&labels, &rules, &st, &d)) + ); +} + +// --------------------------------------------------------------------------- +// C. Algebraic elimination, gated on the semantics +// --------------------------------------------------------------------------- + +#[test] +fn c_duplicate_atoms_and_rules_are_removed_only_for_set_answers() { + let f = family_data(); + let (labels, rules) = resolved(&family_noisy::crepe_plan()); + let st = store(&labels, &f, [1, 1, 1]); + let d = answer_demand(&labels, 17); + + let (set, rep_set) = + evaluate(&rules, &d, ResultKind::Bindings(SET), Options::ALL, &st).unwrap(); + assert!( + rep_set + .optimized + .rewrites + .contains(&Rewrite::DuplicateAtom { rule: 0, atom: 3 }) + ); + assert!( + rep_set + .optimized + .rewrites + .contains(&Rewrite::DuplicateRule { rule: 1 }) + ); + assert_eq!( + rep_set.optimized.rules.len(), + 1, + "rule 1 equals rule 0; rule 2 is dead" + ); + assert_eq!( + bindings(set), + as_set(&reference_bag(&labels, &rules, &st, &d)) + ); + + // Under bag semantics both rewrites would change the counts, so neither runs. + let (bag, rep_bag) = + evaluate(&rules, &d, ResultKind::Bindings(BAG), Options::ALL, &st).unwrap(); + assert!(!rep_bag.optimized.rewrites.iter().any(|r| matches!( + r, + Rewrite::DuplicateAtom { .. } | Rewrite::DuplicateRule { .. } + ))); + let want = reference_bag(&labels, &rules, &st, &d) + .into_iter() + .collect::>(); + assert_eq!(bindings(bag), want); + // The duplicate Allowed(b) squares the Allowed(2) multiplicity: the counts + // really are different from the set. + assert!(want.iter().any(|(_, m)| *m > 1)); +} + +// --------------------------------------------------------------------------- +// D. Coordinate identity +// --------------------------------------------------------------------------- + +#[test] +fn d_a_side_from_another_domain_is_not_anded() { + let f = family_data(); + let (labels, rules) = resolved(&family::crepe_plan()); + let st = store(&labels, &f, [1, 1, 1]); + let q = query_multi(&rules[0]); + let plan = star(&q).unwrap(); + let mut prep = prepare(&plan, &st); + + // Same schema, other ordinal domain: Parent keyed by column 0 over a + // domain built from a different relation. + let other_rows: Vec<[u32; 2]> = (0..50).map(|i| [i * 7, i]).collect(); + let other = JoinTable::prepare_star(&[( + RelationView { + rows: &other_rows, + generation: 1, + }, + 0, + )]); + let foreign = other.sides[0].clone(); + let key = prep.keys[1]; + assert_eq!( + prep.graft(1, foreign, key, fingerprint(&other)), + Err(NativeRefusal::DomainMismatch { side: 1 }) + ); + + // A table prepared for another plan's sides is no witness for this one. + let other_plan = star(&query_multi(&rules[1])).unwrap(); + assert_eq!( + joint(&prep, &other_plan, &[1], Measure::Count, None).map(|_| ()), + Err(NativeRefusal::NoCoordinateWitness) + ); +} + +// --------------------------------------------------------------------------- +// E. Generation staleness +// --------------------------------------------------------------------------- + +#[test] +fn e_a_stale_generation_and_a_value_outside_the_domain_are_refused() { + let f = family_data(); + let (labels, rules) = resolved(&family::crepe_plan()); + let st = store(&labels, &f, [1, 1, 1]); + let plan = star(&query_multi(&rules[0])).unwrap(); + let mut prep = prepare(&plan, &st); + assert!(distinct_centers(&prep, &plan, &[1, 1, 1]).is_ok()); + assert_eq!( + distinct_centers(&prep, &plan, &[1, 2, 1]), + Err(NativeRefusal::Table(JoinRefusal::Stale { + side: 1, + bound: 1, + current: 2 + })) + ); + // A reseal of Allowed that names a y the domain never had. + let resealed: Vec<[u32; 2]> = vec![[1, 1], [999_999, 999_999]]; + let r = prep.table.replace_side( + 2, + RelationView { + rows: &resealed, + generation: 2, + }, + 0, + ); + assert_eq!( + r, + Err(JoinRefusal::OutsideDomain { + side: 2, + y: 999_999 + }) + ); +} + +// --------------------------------------------------------------------------- +// F. Bag multiplicity +// --------------------------------------------------------------------------- + +#[test] +fn f_set_and_bag_answers_are_not_confused() { + let f = family_data(); + let (labels, rules) = resolved(&family::crepe_plan()); + let st = store(&labels, &f, [1, 1, 1]); + let d = answer_demand(&labels, 17); + let bag = reference_bag(&labels, &rules, &st, &d); + let derivations: u64 = bag.values().sum(); + // (17,10) through y = 1 once and through y = 2 twice (Allowed(2) twice): + // 3 derivations; (17,11) 1; (17,12) 2. + assert_eq!( + bag.into_iter().collect::>(), + vec![(vec![17, 10], 3), (vec![17, 11], 1), (vec![17, 12], 2)] + ); + for opts in [Options::ALL, Options::NONE] { + let count = |k| evaluate(&rules, &d, k, opts, &st).unwrap().0; + assert_eq!(count(ResultKind::Count(BAG)), Outcome::Count(derivations)); + assert_eq!(count(ResultKind::Count(SET)), Outcome::Count(3)); + assert_eq!(count(ResultKind::Exists), Outcome::Exists(true)); + } + // MORK stores sets: asked for a bag it refuses, never answers 1s. + assert!(capability(Backend::Mork, BAG).is_err()); + assert!(capability(Backend::PathMapZipper, BAG).is_err()); +} + +// --------------------------------------------------------------------------- +// G. Unsupported patterns are refused by name +// --------------------------------------------------------------------------- + +#[test] +fn g_unsupported_shapes_are_refused_not_flattened() { + let plan = refused::crepe_plan(); + let labels = Labels::of(&plan); + let resolved = resolve(&plan, &labels); + let by_goal = |name: &str| -> Vec<_> { + plan.rules + .iter() + .zip(&resolved) + .filter(|(r, _)| r.goal.relation == name) + .map(|(_, res)| res.clone()) + .collect() + }; + assert_eq!(by_goal("Neg"), vec![Err(logic::Refusal::Negation)]); + let reach = by_goal("Reach"); + assert!(reach[0].is_ok()); + assert_eq!(reach[1], Err(logic::Refusal::Recursive)); + + let pattern = |name: &str| query_multi(by_goal(name)[0].as_ref().unwrap()); + assert!(matches!( + admit(&pattern("Loop")), + Err(MorkRefusal::RepeatedVarInAtom { pattern: 0, .. }) + )); + assert_eq!(admit(&pattern("Tri")), Err(MorkRefusal::Cyclic)); + // A chain is acyclic, so MORK's shape admits it; the native star does not. + let chain = pattern("Chain"); + assert_eq!(admit(&chain), Ok(())); + assert!(matches!(star(&chain), Err(NativeRefusal::NoSharedVariable))); + + // A nested sub-expression is refused, never flattened. + let mut nested: QueryMulti = pattern("Chain"); + nested.patterns[0][2] = Expr::Nested(vec![Expr::Int(1), Expr::Int(2)]); + assert_eq!( + admit(&nested), + Err(MorkRefusal::NestedPattern { pattern: 0 }) + ); +} + +/// The pipeline surfaces a refusal; it does not answer. +#[test] +fn g_the_pipeline_refuses_a_chain() { + let plan = refused::crepe_plan(); + let labels = Labels::of(&plan); + let rules: Vec = resolve(&plan, &labels) + .into_iter() + .filter_map(Result::ok) + .collect(); + let e = labels.id("E").unwrap(); + let rows: Vec<[u32; 2]> = vec![[1, 2], [2, 3], [3, 4]]; + let st = Store { + relations: vec![(e, &rows, 1), (labels.id("B").unwrap(), &rows, 1)], + }; + let d = Demand { + goal: labels.id("Chain").unwrap(), + bound: vec![None, None], + }; + let r = evaluate(&rules, &d, ResultKind::Bindings(SET), Options::ALL, &st); + assert!(matches!(r, Err(PipelineRefusal::Native { .. }))); +} + +// --------------------------------------------------------------------------- +// H. No forbidden physical operation +// --------------------------------------------------------------------------- + +#[test] +fn h_the_programs_have_no_foreign_hop() { + let f = family_data(); + let (labels, rules) = resolved(&family::crepe_plan()); + let st = store(&labels, &f, [1, 1, 1]); + for kind in [ + ResultKind::Exists, + ResultKind::Count(BAG), + ResultKind::Bindings(SET), + ] { + for d in [answer_demand(&labels, 17), answer_demand(&labels, 5)] { + let (_, rep) = evaluate(&rules, &d, kind, Options::ALL, &st).unwrap(); + assert!(!rep.programs.is_empty(), "a program ran"); + for p in &rep.programs { + assert!(!has_foreign_hop(p), "{p:?}"); + assert!( + p.ops.iter().all(|op| matches!(op, MaskOp::And { .. })), + "only plane ANDs: {p:?}" + ); + } + } + } + // The detector can fire. + let gather = Program::new( + vec![MaskOp::Gather { + lane: 0, + foreign: 0, + under: None, + dst: 0, + }], + Terminal::Count { + mask: Operand::Scratch(0), + }, + ); + assert!(has_foreign_hop(&gather)); +} + +// --------------------------------------------------------------------------- +// I. Common-coordinate star +// --------------------------------------------------------------------------- + +#[test] +fn i_the_star_mask_is_the_intersection_of_all_three_planes() { + let plan = star3::crepe_plan(); + let (labels, rules) = resolved(&plan); + let r: Vec<[u32; 2]> = (0..300).map(|v| [v, v]).collect(); + let s: Vec<[u32; 2]> = (0..300).filter(|v| v % 2 == 0).map(|v| [v, v]).collect(); + let q: Vec<[u32; 2]> = (0..300).filter(|v| v % 3 == 0).map(|v| [v, v]).collect(); + let st = Store { + relations: vec![ + (labels.id("R").unwrap(), &r, 1), + (labels.id("S").unwrap(), &s, 1), + (labels.id("Q").unwrap(), &q, 1), + ], + }; + let sp = star(&query_multi(&rules[0])).unwrap(); + assert_eq!(sp.sides.len(), 3); + let prep = prepare(&sp, &st); + let n = distinct_centers(&prep, &sp, &[1, 1, 1]).unwrap(); + let all_three = (0..300).filter(|v| v % 6 == 0).count() as u64; + let two_only = (0..300).filter(|v| v % 2 == 0).count() as u64; + assert_eq!(n, all_three); + assert_ne!(n, two_only, "the third plane must restrict"); +} + +// --------------------------------------------------------------------------- +// J. Reference parity, including MORK +// --------------------------------------------------------------------------- + +#[test] +fn j_two_way_join_equals_mork_and_the_reference() { + let plan = two_way::crepe_plan(); + let (labels, rules) = resolved(&plan); + let r: Vec<[u32; 2]> = (0..120u32).map(|i| [i, i % 17]).collect(); + let s: Vec<[u32; 2]> = (0..90u32).map(|i| [i % 23, 500 + i]).collect(); + let st = Store { + relations: vec![ + (labels.id("R").unwrap(), &r, 1), + (labels.id("S").unwrap(), &s, 1), + ], + }; + let d = Demand { + goal: labels.id("T").unwrap(), + bound: vec![None, None, None], + }; + let want_bag = reference::eval(&rules, d.goal, &d.bound, &raw(&labels, &st)); + let (set, _) = evaluate(&rules, &d, ResultKind::Bindings(SET), Options::ALL, &st).unwrap(); + let (bag, _) = evaluate(&rules, &d, ResultKind::Bindings(BAG), Options::ALL, &st).unwrap(); + assert_eq!(bindings(set.clone()), as_set(&want_bag)); + assert_eq!( + bindings(bag), + want_bag.clone().into_iter().collect::>() + ); + + let input = JoinInput { + r: RelationView { + rows: &r, + generation: 1, + }, + s: RelationView { + rows: &s, + generation: 1, + }, + }; + let mut mork = bind(Backend::Mork, &input, SET).expect("MORK builds"); + let mork_rows: Vec<(Vec, u64)> = mork + .answers(&input) + .expect("MORK answers") + .rows + .keys() + .map(|t| (t.iter().map(|&v| u64::from(v)).collect(), 1)) + .collect(); + assert_eq!(bindings(set), mork_rows, "MORK"); +} + +// --------------------------------------------------------------------------- +// K. No runtime container in the optimized path +// --------------------------------------------------------------------------- + +#[test] +fn k_the_native_path_names_no_crepe_runtime_container() { + for (name, src) in [ + ("native.rs", include_str!("../src/native.rs")), + ("pipeline.rs", include_str!("../src/pipeline.rs")), + ("optimize.rs", include_str!("../src/optimize.rs")), + ("mork_api.rs", include_str!("../src/mork_api.rs")), + ] { + for banned in ["HashMap", "HashSet", "crepe!", "Crepe::", "oracle::"] { + assert!(!src.contains(banned), "{name} names {banned}"); + } + } +} + +// --------------------------------------------------------------------------- +// L. Deterministic lowering +// --------------------------------------------------------------------------- + +#[test] +fn l_renamed_and_noisy_rules_give_the_same_plan() { + let f = family_data(); + let (la, ra) = resolved(&family::crepe_plan()); + let (lb, rb) = resolved(&family_noisy::crepe_plan()); + let sa = store(&la, &f, [1, 1, 1]); + let sb = store(&lb, &f, [1, 1, 1]); + let (_, a) = evaluate( + &ra, + &answer_demand(&la, 17), + ResultKind::Bindings(SET), + Options::ALL, + &sa, + ) + .unwrap(); + let (_, b) = evaluate( + &rb, + &answer_demand(&lb, 17), + ResultKind::Bindings(SET), + Options::ALL, + &sb, + ) + .unwrap(); + // Both label tables sort the same names, so the ids agree. + assert_eq!(la.id("Parent"), lb.id("Parent")); + let canon = |r: &crepe_plan_probe::pipeline::Report| -> Vec { + r.optimized + .rules + .iter() + .map(|(_, r)| r.canonical()) + .collect() + }; + // The noisy rule's atoms are in another order; compare as sets of atoms. + let atoms = |rs: Vec| -> Vec> { + rs.into_iter() + .map(|r| { + let mut b = r.body; + b.sort(); + b + }) + .collect() + }; + assert_eq!(atoms(canon(&a)).len(), atoms(canon(&b)).len()); + assert_eq!(a.programs, b.programs, "same mask-risc programs"); + assert_eq!(a.stars.len(), b.stars.len()); +} + +// --------------------------------------------------------------------------- +// M. The H14d absorption case +// --------------------------------------------------------------------------- + +/// `normalize` still misses `¬(x ∧ (x ∨ y)) = ¬x` (H14d, 2026-10-16) — but +/// only when `x` is itself an `And` or `Or`. Measured here: with `x` a plane or +/// a comparison it reduces correctly; with `x = p0 ∧ p2` De Morgan and +/// flattening run first and the absorption no longer matches. A relalg defect, +/// pinned here, not fixed in the optimizer. This spike never builds the shape: +/// it only emits an `And` of planes (no `Not`, no `Or`). +#[test] +fn m_the_known_absorption_gap_is_untouched_and_not_produced() { + let absorbed = |x: Rel| { + let e = Rel::Not(Box::new(Rel::And(vec![ + x.clone(), + Rel::Or(vec![x.clone(), Rel::Plane(1)]), + ]))); + normalize(&e) == normalize(&Rel::Not(Box::new(x))) + }; + assert!(absorbed(Rel::Plane(0)), "leaf x reduces"); + assert!( + !absorbed(Rel::And(vec![Rel::Plane(0), Rel::Plane(2)])), + "compound x: fixed upstream? update this pin" + ); + assert!( + !absorbed(Rel::Or(vec![Rel::Plane(0), Rel::Plane(2)])), + "compound x: fixed upstream? update this pin" + ); + // What this spike produces: duplicates collapse, nothing widens. + let ours = Rel::And(vec![Rel::Plane(0), Rel::Plane(1), Rel::Plane(0)]); + assert_eq!( + normalize(&ours), + Rel::And(vec![Rel::Plane(0), Rel::Plane(1)]) + ); +} + +#[test] +fn keys_name_relation_and_column() { + // A side key is equality on all four fields; two sides of the same + // relation with other key columns are different coordinates. + let a = SideKey { + rel: RelId(0), + key_col: 1, + restrict: None, + unary: false, + }; + let b = SideKey { key_col: 0, ..a }; + assert_ne!(a, b); +} diff --git a/crates/lance-graph-relalg/src/lib.rs b/crates/lance-graph-relalg/src/lib.rs index 9944d524d..06917a5eb 100644 --- a/crates/lance-graph-relalg/src/lib.rs +++ b/crates/lance-graph-relalg/src/lib.rs @@ -34,5 +34,5 @@ pub mod lower; pub mod normalize; pub use ir::{Atom, Cmp, Measure, Query, Rel}; -pub use lower::{run, run_as_written, Answer, RelError, Run}; +pub use lower::{run, run_as_written, run_with_program, Answer, RelError, Run}; pub use normalize::{normalize, symbolic}; diff --git a/crates/lance-graph-relalg/src/lower.rs b/crates/lance-graph-relalg/src/lower.rs index bbd91483c..401090299 100644 --- a/crates/lance-graph-relalg/src/lower.rs +++ b/crates/lance-graph-relalg/src/lower.rs @@ -2,7 +2,7 @@ //! builds and runs one mask-risc program at the measure. Nothing here loops //! over rows. -use lance_graph_mask_risc::{Foreign, Planes}; +use lance_graph_mask_risc::{Foreign, Planes, Program}; use lance_graph_r2il::{ FoldDialect, FoldError, Val, INT_AND, INT_EQUAL, INT_NOT, INT_OR, INT_S_LESS, LOAD, POP_COUNT, }; @@ -61,15 +61,27 @@ impl From for RelError { /// Rewrite the query, then lower and run it. pub fn run(q: &Query, planes: &Planes<'_>, keep: Option<&mut [u64]>) -> Result { + run_with_program(q, planes, keep).map(|(run, _)| run) +} + +/// [`run`], also returning the mask-risc program the fold ran, so a caller can +/// inspect its ops (for example, that no `Gather` or `Scatter` was emitted). +/// `None` when the description alone answered and no program ran. +pub fn run_with_program( + q: &Query, + planes: &Planes<'_>, + keep: Option<&mut [u64]>, +) -> Result<(Run, Option), RelError> { let rel = normalize(&q.rel); if let Some(answer) = symbolic(&rel, q.measure, planes.n_rows) { - return Ok(Run { + let run = Run { answer, passes: 0, programs: 0, - }); + }; + return Ok((run, None)); } - run_as_written( + lower_and_run( &Query { rel, measure: q.measure, @@ -86,6 +98,14 @@ pub fn run_as_written( planes: &Planes<'_>, keep: Option<&mut [u64]>, ) -> Result { + lower_and_run(q, planes, keep).map(|(run, _)| run) +} + +fn lower_and_run( + q: &Query, + planes: &Planes<'_>, + keep: Option<&mut [u64]>, +) -> Result<(Run, Option), RelError> { let mut d = match (q.measure, keep) { (Measure::Keep, Some(sink)) => FoldDialect::with_keep(planes, &Foreign::NONE, sink), (Measure::Keep, None) => return Err(RelError::NoKeepSink), @@ -120,11 +140,12 @@ pub fn run_as_written( Answer::Kept } }; - Ok(Run { + let run = Run { answer, passes: d.facade_passes(), programs: d.programs_run(), - }) + }; + Ok((run, d.last_program().cloned())) } fn call( From 2aad5d323e802256bf3e270329affabfab625976 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 18:06:54 +0000 Subject: [PATCH 2/6] D-SRAM-2 spike: measure example, plan, board entry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stage costs per size with and without the rewrites, parity against datafrog and the nested-loop reference; Soufflé harvest table and follow-ups in the plan. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01VncKoyG6JkpsoqhqTGTv3f --- .claude/board/STATUS_BOARD.md | 2 +- .../2026-10-20-crepe-rule-optimizer-spike.md | 20 ++ .claude/board/entries/README.md | 3 +- .../plans/crepe-rule-optimizer-spike-v1.md | 115 ++++++++++ crates/crepe-plan-probe/examples/measure.rs | 211 ++++++++++++++++++ 5 files changed, 349 insertions(+), 2 deletions(-) create mode 100644 .claude/board/entries/2026-10-20-crepe-rule-optimizer-spike.md create mode 100644 .claude/plans/crepe-rule-optimizer-spike-v1.md create mode 100644 crates/crepe-plan-probe/examples/measure.rs diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index 5ab306fed..3042116ef 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -56,7 +56,7 @@ These PRs merged with no board rows. Every row below is a PROPOSAL (documents an | D-RMAP-2 | context-declaration reuse and GraphBLAS/BF16 tile probe (`d-rmap-2-context-tile-algebra-probe-v1.md`) | LAB ONLY, not admitted to production; PARTIAL: §5a MEASURED on the polyfill tier and on the AMX tile tier (§5a-AMX); `VDPBF16PS`-only tier and GraphBLAS arm not run | arms 1–5 in the plan; 297× (D-GTM-0j) is not evidence | | D-SRAM-0 | inventory Soufflé AST/ast2ram/RAM passes against D-ASM/D-RA/D-TGA owners | Queued | evidence per mapped pass; no code imported | | D-SRAM-1 | bound AST → RAM-like schedule for pure conjunctive selection | Queued | same answers and read sets on hand-built fixtures; unsupported ops typed-refused | -| D-SRAM-2 | cold optimizer rewrites (SCC/strata, magic sets, ordering, dedup) | Queued (research until D-ASM AST placement and D-RA-2) | optimized == unoptimized == oracle; rewrite off removes only the plan shape; disabling a semantic guard fails | +| D-SRAM-2 | cold optimizer rewrites (SCC/strata, magic sets, ordering, dedup) | Queued (research until D-ASM AST placement and D-RA-2); lab spike measured 2026-10-20, `entries/2026-10-20-crepe-rule-optimizer-spike.md`, not authorization | optimized == unoptimized == oracle; rewrite off removes only the plan shape; disabling a semantic guard fails | | D-SRAM-3 | co-addressed physical selection, R2IL `Program` reproducibility | Queued | P1/P2 == scalar reference; P3 refuses; no foreign gather | | **D-SRAM-4** | recursive region scheduling | **BLOCKED** — its target (D-RA-4's d′ driver) is a banned gather; no lawful driver exists | waits for a co-addressed recursion design | | D-SRAM-5 | speculative-thought liveness over 0+63 slots | Queued (after D-TGA-1/3) | exhaustion refused; no tombstoned branch reused before receipt | diff --git a/.claude/board/entries/2026-10-20-crepe-rule-optimizer-spike.md b/.claude/board/entries/2026-10-20-crepe-rule-optimizer-spike.md new file mode 100644 index 000000000..6ec648b7e --- /dev/null +++ b/.claude/board/entries/2026-10-20-crepe-rule-optimizer-spike.md @@ -0,0 +1,20 @@ +# 2026-10-20 — D-SRAM-2 spike: crepe as a cold rule optimizer in front of relalg + +**Status:** MEASURED (lab probe `crates/crepe-plan-probe`, own workspace, excluded). Research only: does not authorize D-SRAM-2 and does not own D-RA-2. Plan: `.claude/plans/crepe-rule-optimizer-spike-v1.md`. + +## What landed + +- crepe fork (`AdaWorldAPI/crepe` branch `ccr-d3ce1263-ied9yh`, `9fd5785`): `bind_fields` extracted from `make_clause` and shared by `crepe!` and a new `crepe_plan!`, which runs `Context::new` (arity, `@input`, strata, negation cycles) and emits a plan as data, no HashSet/HashMap runtime. +- relalg: `run_with_program` (additive) returns the lowered mask-risc program for inspection. +- Probe: label resolution, binding propagation (Soufflé adornment step, no magic predicates), duplicate-atom and duplicate-rule elimination gated on duplicate-insensitive answers, MORK `query_multi` mapping with D-MPS-9 refusals, star lowering over the D-MPS-8 table with coordinate witness and domain fingerprint. + +## Measured + +- 16 differential tests (A–M) against `crepe!`, datafrog, the MORK shape and a nested-loop bag reference; 9 disables each red at their named tests. +- `Answer(17, z)`, 50,000 Parent rows, release: terminal 3.6 µs with rewrites vs 11.8 ms without; prepared rows 56,286 vs 118,802; summed cold cost lower with the rewrites at every size (break-even reuse count 0). Parity ok at 100/1k/10k/50k. +- H14d absorption miss holds only when `x` is itself an `And`/`Or` (pinned, test M). + +## OPEN + +- Dynamic MeTTa rules need crepe's analysis as a library: crepe is a proc-macro crate (follow-up A, `crepe-core` split). +- Preparation still reads the unrestricted side in full; semijoin reduction before preparation is the next step. diff --git a/.claude/board/entries/README.md b/.claude/board/entries/README.md index 5df1eb7fa..c131c948c 100644 --- a/.claude/board/entries/README.md +++ b/.claude/board/entries/README.md @@ -25,7 +25,7 @@ index row, (3) no duplicate entry id. Checks 1 and 2 are deliberately opposite directions; the stranding this convention prevents shows up in exactly one of them, never both. -388 entries, 2026-08-06 .. 2026-10-20. +389 entries, 2026-08-06 .. 2026-10-20. | date | entry id | finding | file | |---|---|---|---| @@ -42,6 +42,7 @@ exactly one of them, never both. | 2026-10-20 | `D-RMAP-2` | | [2026-10-20-d-rmap-2-tile-gemm-cell.md](2026-10-20-d-rmap-2-tile-gemm-cell.md) | | 2026-10-20 | `D-RMAP-2-AMX` | | [2026-10-20-d-rmap-2-amx-tier.md](2026-10-20-d-rmap-2-amx-tier.md) | | 2026-10-20 | `D-RMAP-2-AMX-RERUN` | §5a-AMX re-measured on a second AMX host | [2026-10-20-d-rmap-2-amx-remeasured.md](2026-10-20-d-rmap-2-amx-remeasured.md) | +| 2026-10-20 | `D-SRAM-2` | | [2026-10-20-crepe-rule-optimizer-spike.md](2026-10-20-crepe-rule-optimizer-spike.md) | | 2026-10-20 | `ce64-isa-not-storage` | | [2026-10-20-ce64-isa-not-storage.md](2026-10-20-ce64-isa-not-storage.md) | | 2026-10-19 | `post-merge-recovery-1601-1617` | | [2026-10-19-post-merge-recovery-1601-1617.md](2026-10-19-post-merge-recovery-1601-1617.md) | | 2026-10-19 | `native-join-relationship-table` | | [2026-10-19-native-join-relationship-table.md](2026-10-19-native-join-relationship-table.md) | diff --git a/.claude/plans/crepe-rule-optimizer-spike-v1.md b/.claude/plans/crepe-rule-optimizer-spike-v1.md new file mode 100644 index 000000000..b59ae4dd5 --- /dev/null +++ b/.claude/plans/crepe-rule-optimizer-spike-v1.md @@ -0,0 +1,115 @@ +# Crepe as a cold rule optimizer in front of relalg (D-SRAM-2 research spike) + +**Status:** MEASURED 2026-10-20 (lab spike, not authorization of D-SRAM-2; D-RA-2 still gates any IR ownership). Probe crate `crates/crepe-plan-probe`; crepe fork branch `AdaWorldAPI/crepe` `ccr-d3ce1263-ied9yh` (`9fd5785`). +**Answers:** can Crepe's compiler front end optimize MeTTa-over-MORK queries before relalg without bringing Crepe's runtime into the path? **PARTIALLY — yes for the admitted subset, by a second macro in the fork; no library reuse without a crate split** (§4). + +## Overview + +The spike tests one hypothesis: most of the physical computation already exists (D-MPS-8 relationship table, relalg, mask-risc), and the missing piece is a small rule compiler that rewrites the question before relalg sees it. + +Pipeline, one stage per module of the probe: + +| stage | code | what it does | +|---|---|---| +| 1 | crepe fork, `crepe_plan!` | `Context::new` (names, arity, `@input` goals, negation cycles, strata), then crepe's own left-to-right bound/free analysis as data. No `HashSet` store, `HashMap` index or fixpoint loop is emitted. | +| 2 | `logic.rs` | labels to ids through the label table; flat, positive, non-recursive rules over `@input` relations, or a named refusal | +| 3 | `optimize.rs` | binding propagation; duplicate-atom and duplicate-rule elimination gated on duplicate-insensitive answers | +| 4 | `mork_api.rs` | each rule as a typed `query_multi (pattern, template)` (D-MPS-9 shape); nested, repeated-variable and cyclic shapes refused | +| 5 | `native.rs` | a star on one shared variable laid out by the D-MPS-8 table; joint mask = relalg `And` of presence planes, one mask-risc program; terminals read the mask | +| 6 | `reference.rs` | nested-loop evaluator over the unoptimized rules (bag) | + +## Checklist + +- [x] Phase 0: sources read (crepe `b26a580`, lance-graph `a7c9c6ef`/`010729b9`, MORK oracle `de9312e3` via mork-pathmap-probe, Soufflé `f53dab8`); claim on EXO-50 (`session_01VncKoyG6JkpsoqhqTGTv3f/009`) +- [x] Phase 1: compiler seam — `bind_fields` extracted from `make_clause`, shared by `crepe!` and `crepe_plan!` +- [x] Phase 2: binding propagation +- [x] Phase 3: MORK `query_multi` mapping and admission +- [x] Phase 4: relalg + mask-risc over the D-MPS-8 table; `relalg::run_with_program` (additive) to inspect the program +- [x] Phase 5: differential tests A–M against `crepe!`, datafrog, MORK, reference; 9 disable runs +- [x] Phase 6 (limited): Soufflé harvest table (§3) +- [ ] Follow-up (not built): semijoin reduction of the unrestricted sides before preparation (§6) + +## 1. Integration point + +``` +crepe_plan! { … } crepe fork src/lib.rs crepe_plan → Context::new → make_plan + └─ bind_fields(fact, vars) shared with make_clause (the crepe! evaluator) +crepe_plan_probe::logic::resolve labels → RelId / VarId +crepe_plan_probe::optimize::optimize Demand → Optimized { rules, post_filter, rewrites } +crepe_plan_probe::mork_api::query_multi + admit QueryMulti { patterns, template } +crepe_plan_probe::native::star → prepare StarPlan → PreparedStar (JoinTable::prepare_star) +crepe_plan_probe::native::joint witness + JoinTable::check + └─ lance_graph_relalg::run_with_program(And[Plane(0..k)], Count|Keep) + └─ FoldDialect → mask-risc Program (And ops only) +``` + +Coordinate witness: a `PreparedStar` carries the `SideKey` of every side (relation, key column, restriction, arity) and a fingerprint of the shared domain. `joint` refuses a plan whose sides differ (`NoCoordinateWitness`); `graft` refuses a side laid out over another domain (`DomainMismatch`); `JoinTable::check` refuses a stale generation. + +## 2. Measured (release, rustc 1.99.0, avx512f, medians of 31) + +Query `Answer(17, z)` over `Answer(x, z) <- Parent(x, y), Parent(y, z), Allowed(y); Answer(5, z) <- Sponsor(z);`, set answers, `n` Parent rows. `cargo run --release --example measure`. + +| n | opts | optimize ns | map+admit ns | normalize ns | prepare µs | mask ns | terminal µs | rows prepared | rules | answers | parity | +|---|---|---|---|---|---|---|---|---|---|---|---| +| 100 | all | 743 | 902 | 702 | 6.2 | 1,295 | 3.7 | 152 | 1 | 12 | ok | +| 100 | none | 270 | 1,325 | 1,008 | 11.2 | 2,244 | 26.5 | 293 | 2 | 12 | ok | +| 1,000 | all | 501 | 538 | 474 | 34.2 | 872 | 2.4 | 1,161 | 1 | 12 | ok | +| 1,000 | none | 178 | 841 | 708 | 62.5 | 1,266 | 192.2 | 2,427 | 2 | 12 | ok | +| 10,000 | all | 489 | 513 | 449 | 489.0 | 896 | 2.5 | 11,286 | 1 | 12 | ok | +| 10,000 | none | 172 | 857 | 698 | 1,128.0 | 1,447 | 2,095.3 | 23,802 | 2 | 12 | ok | +| 50,000 | all | 480 | 514 | 445 | 3,502.9 | 1,104 | 3.6 | 56,286 | 1 | 12 | ok | +| 50,000 | none | 271 | 867 | 718 | 6,910.3 | 1,863 | 11,813.2 | 118,802 | 2 | 12 | ok | + +Plan construction (`crepe_plan()`) 164–261 ns, resolution 273–458 ns, independent of `n`. Each mask-risc program has 2 `And` ops (3 planes). Parity: datafrog at every size, nested-loop reference up to 10,000. + +Reading: +- **Break-even reuse count is 0.** The rewrite itself costs ~210–470 ns more, but it removes a rule, so mapping and normalization get cheaper by more than that: summed cold cost (optimize + map + normalize) is lower with the rewrites at every size (2,347 vs 2,603 ns at n = 100; 1,439 vs 1,856 ns at 50,000). +- **The win is in preparation and in the terminal, not the mask.** With the constant pushed into the body, the `Parent(17, y)` side is prepared from 8 rows instead of n, and the sponsor branch is never prepared: rows halve. The terminal reads one restricted side instead of enumerating every `x` and filtering: 3.6 µs vs 11.8 ms at 50,000 (≈3,300×). The joint mask itself is ~1 µs either way. +- **Preparation still dominates** (3.5 ms at 50,000) because `Parent(y, z)` is prepared in full. That is the next rewrite (§6), not a physical gap. + +## 3. Soufflé harvest (souffle-lang/souffle `f53dab8`, UPL-1.0; algorithms only, no source copied) + +| Soufflé | transformation | precondition | in crepe | in lance-graph | taken | +|---|---|---|---|---|---| +| `src/ast/transform/MagicSet.cpp` `AdornDatabaseTransformer::adornClause` (l. 624) | bound head fields mark variables (weakly) bound; adorned rule per binding pattern | query binding pattern; head constants are normalised to variables first (`NormaliseDatabaseTransformer::normaliseArguments`, l. 510) | no (crepe evaluates bottom-up, no demand) | no | **yes, the adornment step only**: `optimize::specialize` unifies the head with the demand, drops a rule on a constant conflict, substitutes bound variables into the body. No magic predicates (no recursion in scope) | +| `MagicSet.cpp` `MagicSetCoreTransformer::transform` (l. 1139) | magic predicates carry the demand through derived relations | recursion or derived body relations | no | no | no — follow-up B | +| `MinimiseProgram.cpp` `reduceClauseBodies` (l. 375) | drop a body literal equal to an earlier one | Soufflé relations are sets | no | relalg drops duplicate planes after lowering | **yes, gated**: only for `Exists` / set answers; under bag semantics a repeated atom multiplies derivations (test C) | +| `MinimiseProgram.cpp` `removeRedundantClauses` (l. 349) + `areBijectivelyEquivalent` (l. 203) | drop a clause equal to another under a variable bijection and body permutation | set semantics | no | no | **partly, gated**: equality after first-occurrence renumbering; body permutations are not searched (§5) | +| `ast2ram/utility/SipsMetric.h` | literal ordering by bound arguments | — | crepe keeps written order | the star lowering ANDs all planes, order-free | no — nothing to order in a star | +| SCC / stratification | evaluation order, negation check | — | `strata.rs` (petgraph Kosaraju) | — | reused through `crepe_plan!` | + +## 4. Why PARTIALLY + +- crepe is a `proc-macro = true` crate: it can only export macros. `Context`, `Strata` and the binding analysis cannot be called from another crate as a library. The spike therefore adds a second macro to the fork; the plan is fixed at Rust compile time. +- Dynamically arriving MeTTa rules need the analysis as a library: split the fork into `crepe-core` (parse to typed rules, `Context`, `Strata`, `bind_fields`, with `abort!` replaced by `Result`) and the two macros on top. Not needed for the static proof; filed as follow-up A. +- The binding analysis is shared, not copied: disabling bound-variable recognition in `bind_fields` turns both the plan test and `crepe!`'s own transitive-closure tests red. + +## 5. Tests and disables + +16 tests (`tests/pipeline.rs`, A–M of the brief), all green. Every guard turned its tests red under a disable run (made after commit, restored after): + +| disable | red | +|---|---| +| ignore a head-constant conflict | A, B, C, F | +| dedup without the set gate | C | +| skip the domain fingerprint | D | +| skip the coordinate witness | D | +| AND only two of three planes | A, F, I | +| let a nested pattern through | G | +| foreign-hop detector always false | H | +| prepare without the constant restriction | A, B ×2, C, F | +| cap bag multiplicity at 1 | C, F | + +Turning the rewrites off (`Options::NONE`) changes plan shape and cost only; tests A and F compare both settings. + +Test M (H14d absorption): `normalize(¬(x ∧ (x ∨ y)))` reduces correctly when `x` is a plane or a comparison and misses when `x` is itself an `And` or `Or` — narrower than the H14d note states. Pinned, not fixed; the spike never emits `Not` or `Or`. + +Known limits: rule dedup does not search body permutations; the star must have one shared variable and binary or unary atoms; chains are refused by the native star (MORK's shape admits them); the label table stands in for the label codebook. + +## 6. Follow-ups (not built) + +- **Next (recommended):** semijoin reduction before preparation — restrict the unrestricted sides of a star to the center values the restricted side has (here: prepare `Parent(y, z)` only for `y ∈ {children of 17}`). Same answers, preparation from O(n) to O(k). Needs a demand-aware layout step, not a new mask op. +- A: `crepe-core` library split in the fork for dynamic rules. +- B: full magic sets for derived body relations and recursion (waits on D-MTP-3 / D-SRAM-4). +- C: cost-informed ordering once a non-star shape is admitted. +- D–G as in the brief (recursion, 115-strategy corpus, 0+63 thought reuse, richer MORK patterns). diff --git a/crates/crepe-plan-probe/examples/measure.rs b/crates/crepe-plan-probe/examples/measure.rs new file mode 100644 index 000000000..925869190 --- /dev/null +++ b/crates/crepe-plan-probe/examples/measure.rs @@ -0,0 +1,211 @@ +//! Stage costs of the spike, each measured on its own, with and without the +//! rewrites. Run in release from this crate's directory: +//! +//! ```text +//! cargo run --release --example measure +//! ``` +//! +//! The query is `Answer(17, z)` over the `family` program. Each size `n` is the +//! number of `Parent` rows. `x = 17` has `k = 8` children; the other parents +//! are random, so the shared domain grows with `n` while the answer does not. +//! Parity is checked against datafrog at every size and against the nested-loop +//! reference up to 10,000 rows. Timings are medians of `REPS` runs. + +use std::collections::BTreeMap; +use std::hint::black_box; +use std::time::{Duration, Instant}; + +use crepe_plan_probe::logic::{Labels, Rule, resolve}; +use crepe_plan_probe::mork_api::{admit, query_multi}; +use crepe_plan_probe::native::{StarPlan, Store, bindings, joint, prepare, star}; +use crepe_plan_probe::optimize::{Demand, Options, ResultKind, optimize}; +use crepe_plan_probe::programs::family; +use crepe_plan_probe::reference; +use lance_graph_relalg::{Measure, Rel, normalize}; +use mork_pathmap_probe::Semantics; + +const REPS: usize = 31; + +fn median(mut v: Vec) -> Duration { + v.sort(); + v[v.len() / 2] +} + +fn time(mut f: impl FnMut() -> T) -> Duration { + let mut v = Vec::with_capacity(REPS); + for _ in 0..REPS { + let t = Instant::now(); + black_box(f()); + v.push(t.elapsed()); + } + median(v) +} + +fn data(n: usize) -> (Vec<[u32; 2]>, Vec<[u32; 2]>, Vec<[u32; 2]>) { + let mut s = 0x9E37_79B9_7F4A_7C15_u64; + let mut next = move || { + s = s.wrapping_mul(6_364_136_223_846_793_005).wrapping_add(1); + (s >> 33) as u32 + }; + let mut parent = Vec::with_capacity(n); + for c in 0..8u32 { + parent.push([17, 100 + c]); + } + let people = (n as u32 / 2).max(64); + while parent.len() < n { + let a = 1000 + next() % people; + let b = 1000 + next() % people; + parent.push([a, b]); + } + for c in 0..8u32 { + for g in 0..3u32 { + parent.push([100 + c, 1000 + (c * 3 + g) % people]); + } + } + let allowed: Vec<[u32; 2]> = (0..8u32) + .filter(|c| c % 2 == 0) + .map(|c| [100 + c, 100 + c]) + .chain((0..people / 4).map(|i| [1000 + i, 1000 + i])) + .collect(); + let sponsor: Vec<[u32; 2]> = (0..n as u32 / 4).map(|i| [i, i]).collect(); + (parent, allowed, sponsor) +} + +fn main() { + let tier = if cfg!(target_feature = "avx512f") { + "avx512f" + } else if cfg!(target_feature = "avx2") { + "avx2" + } else { + "baseline x86-64 (compile-time features)" + }; + println!("# crepe-plan-probe measure, release, compile-time target features: {tier}"); + println!( + "n,opts,plan_ns,resolve_ns,optimize_ns,map_admit_ns,normalize_ns,prepare_us,mask_ns,terminal_us,rows_prepared,rules,programs,mask_ops,answers,parity" + ); + for n in [100usize, 1_000, 10_000, 50_000] { + let (parent, allowed, sponsor) = data(n); + let plan_t = time(family::crepe_plan); + let plan = family::crepe_plan(); + let labels = Labels::of(&plan); + let resolve_t = time(|| resolve(&plan, &labels)); + let rules: Vec = resolve(&plan, &labels).into_iter().map(|r| r.unwrap()).collect(); + let st = Store { + relations: vec![ + (labels.id("Parent").unwrap(), &parent, 1), + (labels.id("Allowed").unwrap(), &allowed, 1), + (labels.id("Sponsor").unwrap(), &sponsor, 1), + ], + }; + let demand = Demand { + goal: labels.id("Answer").unwrap(), + bound: vec![Some(17), None], + }; + let kind = ResultKind::Bindings(Semantics::Set); + + // Oracles. + let df = datafrog_answer(&parent, &allowed); + let reference = (n <= 10_000).then(|| { + let rows = |rel| { + let (r, _) = st.view(rel); + r.iter() + .map(|x| x[..labels.arity(rel)].iter().map(|&v| u64::from(v)).collect()) + .collect() + }; + reference::eval(&rules, demand.goal, &demand.bound, &rows) + .into_keys() + .collect::>() + }); + + for (name, opts) in [("all", Options::ALL), ("none", Options::NONE)] { + let optimize_t = time(|| optimize(&rules, &demand, kind, opts)); + let opt = optimize(&rules, &demand, kind, opts); + let map_t = time(|| { + opt.rules + .iter() + .map(|(_, r)| { + let q = query_multi(r); + admit(&q).unwrap(); + star(&q).unwrap() + }) + .collect::>() + }); + let stars: Vec = opt + .rules + .iter() + .map(|(_, r)| star(&query_multi(r)).unwrap()) + .collect(); + let rels: Vec = stars + .iter() + .map(|s| Rel::And((0..s.sides.len() as u16).map(Rel::Plane).collect())) + .collect(); + let normalize_t = time(|| rels.iter().map(normalize).collect::>()); + let prepare_t = time(|| stars.iter().map(|s| prepare(s, &st)).collect::>()); + let preps: Vec<_> = stars.iter().map(|s| prepare(s, &st)).collect(); + let gens = |s: &StarPlan| -> Vec { vec![1; s.sides.len()] }; + let mask_t = time(|| { + stars + .iter() + .zip(&preps) + .map(|(s, p)| joint(p, s, &gens(s), Measure::Count, None).unwrap().0) + .collect::>() + }); + let mut programs = 0; + let mut ops = 0; + for (s, p) in stars.iter().zip(&preps) { + let (run, prog) = joint(p, s, &gens(s), Measure::Count, None).unwrap(); + programs += run.programs; + ops += prog.map_or(0, |p| p.ops.len()); + } + let answer = || { + let mut out: BTreeMap, u64> = BTreeMap::new(); + for (s, p) in stars.iter().zip(&preps) { + for (row, m) in bindings(p, s, &gens(s), false).unwrap() { + *out.entry(row).or_insert(0) += m; + } + } + out.retain(|row, _| { + row.iter() + .zip(&opt.post_filter) + .all(|(v, f)| f.is_none_or(|f| f == *v)) + }); + out.into_keys().collect::>() + }; + let terminal_t = time(answer); + let got = answer(); + let parity = got == df && reference.as_ref().is_none_or(|r| *r == got); + println!( + "{n},{name},{},{},{},{},{},{:.1},{},{:.1},{},{},{programs},{ops},{},{}", + plan_t.as_nanos(), + resolve_t.as_nanos(), + optimize_t.as_nanos(), + map_t.as_nanos(), + normalize_t.as_nanos(), + prepare_t.as_secs_f64() * 1e6, + mask_t.as_nanos(), + terminal_t.as_secs_f64() * 1e6, + preps.iter().map(|p| p.rows_prepared).sum::(), + opt.rules.len(), + got.len(), + if parity { "ok" } else { "MISMATCH" }, + ); + assert!(parity, "answers differ from the oracles at n = {n}, {name}"); + } + } +} + +/// `Answer(17, z)` by datafrog (set), the family rule only. +fn datafrog_answer(parent: &[[u32; 2]], allowed: &[[u32; 2]]) -> Vec> { + use datafrog::Relation; + let by_y: Relation<(u32, u32)> = parent.iter().map(|&[a, b]| (b, a)).collect(); + let by_parent: Relation<(u32, u32)> = parent.iter().map(|&[a, b]| (a, b)).collect(); + let allowed: Relation<(u32, ())> = allowed.iter().map(|&[a, _]| (a, ())).collect(); + let xz: Relation<(u32, (u32, u32))> = + Relation::from_join(&by_y, &by_parent, |&y, &x, &z| (y, (x, z))); + let ans: Relation<(u32, u32)> = Relation::from_join(&xz, &allowed, |_, &(x, z), _| (x, z)); + ans.elements + .iter() + .filter(|(a, _)| *a == 17) + .map(|&(a, b)| vec![u64::from(a), u64::from(b)]) + .collect() +} From 94c3cf59aaf452e15665ddf239ac653166729dbf Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 18:07:08 +0000 Subject: [PATCH 3/6] crepe-plan-probe: rustfmt the measure example Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01VncKoyG6JkpsoqhqTGTv3f --- crates/crepe-plan-probe/examples/measure.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/crates/crepe-plan-probe/examples/measure.rs b/crates/crepe-plan-probe/examples/measure.rs index 925869190..1b9cbcc39 100644 --- a/crates/crepe-plan-probe/examples/measure.rs +++ b/crates/crepe-plan-probe/examples/measure.rs @@ -89,7 +89,10 @@ fn main() { let plan = family::crepe_plan(); let labels = Labels::of(&plan); let resolve_t = time(|| resolve(&plan, &labels)); - let rules: Vec = resolve(&plan, &labels).into_iter().map(|r| r.unwrap()).collect(); + let rules: Vec = resolve(&plan, &labels) + .into_iter() + .map(|r| r.unwrap()) + .collect(); let st = Store { relations: vec![ (labels.id("Parent").unwrap(), &parent, 1), @@ -109,7 +112,12 @@ fn main() { let rows = |rel| { let (r, _) = st.view(rel); r.iter() - .map(|x| x[..labels.arity(rel)].iter().map(|&v| u64::from(v)).collect()) + .map(|x| { + x[..labels.arity(rel)] + .iter() + .map(|&v| u64::from(v)) + .collect() + }) .collect() }; reference::eval(&rules, demand.goal, &demand.bound, &rows) From 8c13c4e1af95482c2555cb92091e2c67dfed9a40 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 18:07:34 +0000 Subject: [PATCH 4/6] crepe-plan-probe: name the measure fixture tuple (clippy type_complexity) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01VncKoyG6JkpsoqhqTGTv3f --- crates/crepe-plan-probe/examples/measure.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/crepe-plan-probe/examples/measure.rs b/crates/crepe-plan-probe/examples/measure.rs index 1b9cbcc39..4d7b7a0a8 100644 --- a/crates/crepe-plan-probe/examples/measure.rs +++ b/crates/crepe-plan-probe/examples/measure.rs @@ -41,7 +41,10 @@ fn time(mut f: impl FnMut() -> T) -> Duration { median(v) } -fn data(n: usize) -> (Vec<[u32; 2]>, Vec<[u32; 2]>, Vec<[u32; 2]>) { +/// `(Parent, Allowed, Sponsor)` rows. +type Rows = (Vec<[u32; 2]>, Vec<[u32; 2]>, Vec<[u32; 2]>); + +fn data(n: usize) -> Rows { let mut s = 0x9E37_79B9_7F4A_7C15_u64; let mut next = move || { s = s.wrapping_mul(6_364_136_223_846_793_005).wrapping_add(1); From 8b6b585f6d865d6dd53385a336ea2a7443155411 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 18:09:57 +0000 Subject: [PATCH 5/6] thought-git-probe: never squash away the branch parent Under pressure with a full pool, a root candidate was squashed with its oldest child as the source. When that child was the new thought's parent, the branch succeeded but the caller's parent handle went stale. The root is now squashed into that child instead, and the child takes the root's place in the tree. Found by review on #1630. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01VncKoyG6JkpsoqhqTGTv3f --- crates/thought-git-probe/src/slots.rs | 19 ++++++++++++++--- crates/thought-git-probe/tests/slots.rs | 27 +++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/crates/thought-git-probe/src/slots.rs b/crates/thought-git-probe/src/slots.rs index 3e46d94ec..f57ae06c3 100644 --- a/crates/thought-git-probe/src/slots.rs +++ b/crates/thought-git-probe/src/slots.rs @@ -164,15 +164,22 @@ impl SlotPool { Ok(()) } - /// `keep`: the new thought's parent, never the candidate (branching from - /// a thought must not be what removes it). + /// `keep`: the new thought's parent, never the candidate and never the + /// slot a squash releases (branching from a thought must not be what + /// removes it). fn make_room(&mut self, keep: Option) -> (usize, Option) { let c = self.candidate(keep); let cand = self.live[c].expect("pool is full"); let (from, into) = if let Some(p) = cand.parent { (c, p) } else if let Some(child) = self.oldest_child(c) { - (child, c) + // When that child is the new thought's parent, squash the root + // into it instead, so the parent handle stays live. + if Some(child) == keep { + (c, child) + } else { + (child, c) + } } else { let slot = self.handle(c, cand.generation); self.release(c); @@ -196,6 +203,12 @@ impl SlotPool { target.uses += f.uses; } self.reparent(from, Some(into)); + // A child absorbing its parent takes the parent's place in the tree. + if let Some(target) = self.live[into].as_mut() + && target.parent == Some(into) + { + target.parent = f.parent; + } self.release(from); (from, Some(pressure)) } diff --git a/crates/thought-git-probe/tests/slots.rs b/crates/thought-git-probe/tests/slots.rs index 02d724155..37a313bf2 100644 --- a/crates/thought-git-probe/tests/slots.rs +++ b/crates/thought-git-probe/tests/slots.rs @@ -101,6 +101,33 @@ fn branching_from_the_candidate_keeps_it() { assert_eq!(p.thought(s), Ok(100)); } +/// Branching from the oldest child of the candidate root must not squash that +/// child away: the root is squashed into it instead. +#[test] +fn branching_from_the_candidates_child_keeps_the_child() { + let (mut p, h) = full(Eviction::Fifo, &[(1, 0)]); + let (s, e) = p.branch(100, Some(h[1])).unwrap(); + assert_eq!( + p.thought(h[1]), + Ok(1), + "the branch parent was removed: {e:?}" + ); + assert!( + matches!(e, Some(Pressure::Squashed { thought: 0, into, .. }) if into == h[1]), + "{e:?}" + ); + assert_eq!(p.thought(h[0]), Err(Stale(h[0]))); + assert_eq!(p.thought(s), Ok(100)); + // Thought 1 is a root now: under the next pressure it squashes its own + // child (100) into itself rather than being dropped. + let (_, e) = p.branch(101, None).unwrap(); + assert!( + matches!(e, Some(Pressure::Squashed { thought: 100, .. })), + "{e:?}" + ); + assert_eq!(p.thought(h[1]), Ok(1)); +} + /// Freeing makes room without pressure, and the freed handle is stale. #[test] fn a_freed_slot_is_reused_without_pressure() { From 743b3a1036d041a5c71d3bec02839761bfe509eb Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 18:16:53 +0000 Subject: [PATCH 6/6] Address #1631 review: one program per rule, full generation vectors, merge order - crepe-plan-probe: the bindings path ran a Count pass and then a Keep pass, and counted only the first. It now runs the Keep program once and counts that run. - crepe-plan-probe: joint refuses a generation vector whose length differs from the table's side count; JoinTable::check zips and would skip sides. - thought-git-probe: a root squashed into its child (the branch-parent case) is the older thought, so Pressure::Squashed now carries from_is_older and the caller merges the younger writes on top. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01VncKoyG6JkpsoqhqTGTv3f --- crates/crepe-plan-probe/src/native.rs | 29 +++++++++++++++++++++-- crates/crepe-plan-probe/src/pipeline.rs | 18 +++++++------- crates/crepe-plan-probe/tests/pipeline.rs | 15 ++++++++++++ crates/thought-git-probe/src/slots.rs | 11 ++++++--- crates/thought-git-probe/tests/slots.rs | 21 +++++++++++++++- 5 files changed, 78 insertions(+), 16 deletions(-) diff --git a/crates/crepe-plan-probe/src/native.rs b/crates/crepe-plan-probe/src/native.rs index c6566933a..005dd4972 100644 --- a/crates/crepe-plan-probe/src/native.rs +++ b/crates/crepe-plan-probe/src/native.rs @@ -91,6 +91,12 @@ pub enum NativeRefusal { DomainMismatch { side: usize, }, + /// The caller passed a generation vector of another length than the + /// table has sides; `JoinTable::check` would skip the missing ones. + Generations { + sides: usize, + given: usize, + }, /// From the table: a stale generation, or rows outside the domain. Table(JoinRefusal), Relalg(RelError), @@ -301,6 +307,12 @@ pub fn joint( keep: Option<&mut [u64]>, ) -> Result<(Run, Option), NativeRefusal> { prep.witness(plan)?; + if current.len() != prep.table.sides.len() { + return Err(NativeRefusal::Generations { + sides: prep.table.sides.len(), + given: current.len(), + }); + } prep.table.check(current).map_err(NativeRefusal::Table)?; let masks: Vec<&[u64]> = prep.table.sides.iter().map(|s| s.present()).collect(); let planes = Planes { @@ -335,9 +347,22 @@ pub fn bindings( current: &[u64], bag: bool, ) -> Result, u64)>, NativeRefusal> { + bindings_run(prep, plan, current, bag).map(|(rows, _, _)| rows) +} + +/// Rows with multiplicities, the relalg run, and the program behind the mask. +pub type BindingsRun = (Vec<(Vec, u64)>, Run, Option); + +/// [`bindings`], plus the relalg run and the program that produced the mask. +pub fn bindings_run( + prep: &PreparedStar, + plan: &StarPlan, + current: &[u64], + bag: bool, +) -> Result { let n = prep.table.domain.len(); let mut mask = vec![0u64; words_for(n)]; - joint(prep, plan, current, Measure::Keep, Some(&mut mask))?; + let (run, program) = joint(prep, plan, current, Measure::Keep, Some(&mut mask))?; let mut out: Vec<(Vec, u64)> = Vec::new(); for_each_set(&mask, n, |slot| { let center = u64::from(prep.table.domain.id(slot)); @@ -388,7 +413,7 @@ pub fn bindings( e.1 = 1; } } - Ok(merged) + Ok((merged, run, program)) } /// True when the program contains an op that reads at a position taken from diff --git a/crates/crepe-plan-probe/src/pipeline.rs b/crates/crepe-plan-probe/src/pipeline.rs index 31c361031..ecbdfbe27 100644 --- a/crates/crepe-plan-probe/src/pipeline.rs +++ b/crates/crepe-plan-probe/src/pipeline.rs @@ -9,7 +9,9 @@ use mork_pathmap_probe::Semantics; use crate::logic::{RelId, Rule}; use crate::mork_api::{MorkRefusal, QueryMulti, admit, query_multi}; -use crate::native::{NativeRefusal, PreparedStar, StarPlan, Store, bindings, joint, prepare, star}; +use crate::native::{ + NativeRefusal, PreparedStar, StarPlan, Store, bindings_run, joint, prepare, star, +}; use crate::optimize::{Demand, Optimized, Options, ResultKind, optimize}; #[derive(Debug, Clone, PartialEq, Eq)] @@ -107,17 +109,13 @@ pub fn evaluate( } else { let mut bag: BTreeMap, u64> = BTreeMap::new(); for ((i, _), (s, p)) in optimized.rules.iter().zip(stars.iter().zip(&prepared)) { - let (run, prog) = joint( - p, - s, - &generations(s), - lance_graph_relalg::Measure::Count, - None, - ) - .map_err(|w| native(*i, w))?; + // One program per rule: the Keep mask that the bindings are read + // from; no separate count pass. + let (rows, run, prog) = + bindings_run(p, s, &generations(s), true).map_err(|w| native(*i, w))?; programs_run += run.programs; programs.extend(prog); - for (row, m) in bindings(p, s, &generations(s), true).map_err(|w| native(*i, w))? { + for (row, m) in rows { *bag.entry(row).or_insert(0) += m; } } diff --git a/crates/crepe-plan-probe/tests/pipeline.rs b/crates/crepe-plan-probe/tests/pipeline.rs index 2612954b9..127092980 100644 --- a/crates/crepe-plan-probe/tests/pipeline.rs +++ b/crates/crepe-plan-probe/tests/pipeline.rs @@ -224,6 +224,11 @@ fn b_the_bound_constant_reaches_the_body_and_the_dead_branch_is_dropped() { rep_off.rows_prepared ); assert!(rep_on.programs_run < rep_off.programs_run); + // Each rule runs exactly one mask-risc program, even when bindings are read. + for rep in [&rep_on, &rep_off] { + assert_eq!(rep.programs_run, rep.optimized.rules.len()); + assert_eq!(rep.programs.len(), rep.optimized.rules.len()); + } } /// The other demand keeps the sponsor branch and specializes the family rule @@ -347,6 +352,16 @@ fn e_a_stale_generation_and_a_value_outside_the_domain_are_refused() { let plan = star(&query_multi(&rules[0])).unwrap(); let mut prep = prepare(&plan, &st); assert!(distinct_centers(&prep, &plan, &[1, 1, 1]).is_ok()); + // One generation per side: a short vector does not skip the check. + for short in [&[][..], &[1, 1][..]] { + assert_eq!( + distinct_centers(&prep, &plan, short), + Err(NativeRefusal::Generations { + sides: 3, + given: short.len() + }) + ); + } assert_eq!( distinct_centers(&prep, &plan, &[1, 2, 1]), Err(NativeRefusal::Table(JoinRefusal::Stale { diff --git a/crates/thought-git-probe/src/slots.rs b/crates/thought-git-probe/src/slots.rs index f57ae06c3..f10f76b90 100644 --- a/crates/thought-git-probe/src/slots.rs +++ b/crates/thought-git-probe/src/slots.rs @@ -15,8 +15,9 @@ //! 2. If the candidate is in a dependency chain, it is squashed instead of //! lost: into its live parent, or, if it has none, its live child is //! squashed into it (operator, same day: "Squashing if the least used are -//! in a dependency chain"). Nothing is discarded; the caller merges the -//! writes (`oracle::squash`). +//! in a dependency chain"). When that child is the new thought's parent, +//! the root is squashed into the child instead. Nothing is discarded; the +//! caller merges the writes, younger over older (`oracle::squash`). //! 3. Only a candidate with no live parent and no live child is evicted. //! //! A [`Slot`] handle carries the generation it was issued under. After its @@ -47,11 +48,14 @@ pub struct Slot { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Pressure { /// `from` was squashed into `into`; `from`'s handle is now stale. The - /// caller merges `from`'s writes over `into`'s. + /// caller merges the younger thought's writes over the older's: `from` + /// over `into`, unless `from_is_older` (a root squashed into its child + /// because that child is the new thought's parent). Squashed { thought: u32, from: Slot, into: Slot, + from_is_older: bool, }, /// The thought had no live parent or child and gave way. Evicted { thought: u32, slot: Slot, uses: u64 }, @@ -198,6 +202,7 @@ impl SlotPool { thought: f.thought, from: self.handle(from, f.generation), into: self.handle(into, t.generation), + from_is_older: f.born < t.born, }; if let Some(target) = self.live[into].as_mut() { target.uses += f.uses; diff --git a/crates/thought-git-probe/tests/slots.rs b/crates/thought-git-probe/tests/slots.rs index 37a313bf2..a8707ff2c 100644 --- a/crates/thought-git-probe/tests/slots.rs +++ b/crates/thought-git-probe/tests/slots.rs @@ -1,6 +1,8 @@ //! D-TGB-1: the slot pool under its two starting policies, with squashing //! along dependency chains before anything is evicted. +use thought_git_probe::fixtures::{Outcome, Thought, cell}; +use thought_git_probe::oracle; use thought_git_probe::slots::{Eviction, LIVE_SLOTS, Pressure, Slot, SlotPool, Stale}; /// Fill the pool with root thoughts (branched from main); `parents[t]` names @@ -68,6 +70,7 @@ fn a_least_used_thought_in_a_chain_is_squashed_into_its_parent() { thought: 20, from: h[20], into: h[5], + from_is_older: false, }) ); assert_eq!(p.thought(h[20]), Err(Stale(h[20]))); @@ -86,6 +89,7 @@ fn a_candidate_with_a_child_absorbs_it() { thought: 1, from: h[1], into: h[0], + from_is_older: false, }) ); assert_eq!(p.thought(h[0]), Ok(0)); @@ -113,9 +117,24 @@ fn branching_from_the_candidates_child_keeps_the_child() { "the branch parent was removed: {e:?}" ); assert!( - matches!(e, Some(Pressure::Squashed { thought: 0, into, .. }) if into == h[1]), + matches!( + e, + Some(Pressure::Squashed { thought: 0, into, from_is_older: true, .. }) if into == h[1] + ), "{e:?}" ); + // The child is the younger thought: its writes win the merge. + let Some(Pressure::Squashed { from_is_older, .. }) = e else { + unreachable!() + }; + let root = Thought::new(0, vec![(cell(0, 1), 7)], Outcome::Commit); + let child = Thought::new(1, vec![(cell(0, 1), 9)], Outcome::Commit); + let stack = if from_is_older { + [root, child] + } else { + [child, root] + }; + assert_eq!(oracle::squash(2, &stack).writes, vec![(cell(0, 1), 9)]); assert_eq!(p.thought(h[0]), Err(Stale(h[0]))); assert_eq!(p.thought(s), Ok(100)); // Thought 1 is a root now: under the next pressure it squashes its own