diff --git a/crates/ogar-elk/Cargo.toml b/crates/ogar-elk/Cargo.toml index 323b7a4a..2c5e6b27 100644 --- a/crates/ogar-elk/Cargo.toml +++ b/crates/ogar-elk/Cargo.toml @@ -6,6 +6,9 @@ license.workspace = true repository.workspace = true authors.workspace = true rust-version.workspace = true -description = "The EL subsumption closure as a PRE-BAKE join validator. Answers two questions and no others: does `A ⊑ B` follow from what is asserted, and is adding a set of axioms to an existing closure sound (does it introduce an equivalence cycle). Entailments are facts — they follow necessarily — so nothing here is graded, ranked, or scored. Runs once, upstream of the bake, over the joined sources; nothing it defines survives into the baked substrate. Zero dependencies, and no serialization surface at all." +description = "The EL subsumption closure as a POST-BAKE observer. Answers two questions and no others: does `A ⊑ B` follow from the baked edges, and is adding a set of axioms to an existing closure sound (does it introduce an equivalence cycle). Entailments are facts — they follow necessarily — so nothing here is graded, ranked, or scored. Reads the `is_a` links carried in a baked row's own edge lanes; the joins happened upstream and are not its business. Zero dependencies, and no serialization surface at all." [dependencies] + +[dev-dependencies] +ogar-obo = { path = "../ogar-obo" } diff --git a/crates/ogar-elk/examples/elk_cross_ontology.rs b/crates/ogar-elk/examples/elk_cross_ontology.rs new file mode 100644 index 00000000..50aa76c0 --- /dev/null +++ b/crates/ogar-elk/examples/elk_cross_ontology.rs @@ -0,0 +1,229 @@ +//! S4b — the typed cross-predicate walk, verified on **seeded, local** queries. +//! +//! `meet` over `is_a` alone always returns nothing across ontologies: +//! subsumption never leaves the ontology it belongs to. Crossing needs a +//! **role** — `has_phenotype` into HPO, `has_anatomy` into Uberon — and then +//! R∃ generalization of the filler up the grounding spine. +//! +//! # Scale: seeded from facts, never a global saturation +//! +//! A query starts from facts that already exist — an observed lab code, a +//! dispensed drug, an imaging study, a coded diagnosis. That is a handful of +//! addresses. Saturating the whole ontology to answer something local is a +//! theory-of-everything computation nobody asked for, and this example +//! deliberately does not do it. +//! +//! # Why the check is NOT against `reason::saturate` +//! +//! Measured this session: the combined `is_a ∪ part_of` grounding spine carries +//! **2 277 back edges** (is_a alone has none), and `ancestor_closure` SKIPS a +//! back edge when it meets one (`!stack.iter().any(|(x, _)| *x == p)`). Its +//! memo is therefore an incomplete closure whose value depends on DFS start +//! order — the same code over the same triples in two orders produced +//! 9 522 084 and 9 437 429. A number that moves cannot falsify anything. +//! +//! So this verifies the R∃ *semantics* on bounded queries against an +//! independent computation sharing no code with [`fillers_closed`]. +//! +//! Usage: `cargo run --release -p ogar-elk --example elk_cross_ontology -- [slab.soa]` + +use ogar_elk::{ClassAddr, LensClosure, Role, fillers_closed, meet_via, most_specific}; +use ogar_obo::{NODE_ROW_STRIDE, Predicate, Row512, edges::EdgeLanes, spine::SpineLens}; + +fn main() { + let path = std::env::args() + .nth(1) + .unwrap_or_else(|| "/tmp/obo/obo-core.soa".to_string()); + let bytes = std::fs::read(&path).unwrap_or_else(|e| panic!("read {path}: {e}")); + let n = bytes.len() / NODE_ROW_STRIDE; + let mut rows = vec![Row512::zeroed(); n]; + for (i, r) in rows.iter_mut().enumerate() { + r.0.copy_from_slice(&bytes[i * NODE_ROW_STRIDE..(i + 1) * NODE_ROW_STRIDE]); + } + let lens = SpineLens::new(&rows); + assert!(lens.is_sorted()); + println!("slab {path}\n rows {n}"); + + // The GROUNDING spine — is_a ∪ part_of. Deliberately NOT the subsumption + // source: a filler generalizes through mereology, a subject must not. + let grounding = |a: ClassAddr, visit: &mut dyn FnMut(ClassAddr)| { + lens.grounding_parents_of(a.classid, a.identity, &mut |c, num| { + visit(ClassAddr::new(c, num)); + }); + }; + let fillers = |a: ClassAddr, r: Role, visit: &mut dyn FnMut(ClassAddr)| { + let p = match r.0 { + 3 => Predicate::HasPhenotype, + 4 => Predicate::HasLocation, + 5 => Predicate::HasAnatomy, + 6 => Predicate::HasQuality, + _ => Predicate::PartOf, + }; + lens.fillers_of(a.classid, a.identity, p, &mut |c, num| { + visit(ClassAddr::new(c, num)); + }); + }; + let has_phenotype = Role(Predicate::HasPhenotype as u8); + + // ── seeds: the facts a query would actually start from ── + let mut seeds: Vec = Vec::new(); + for row in &rows { + if EdgeLanes::new(&row.0).degree(Predicate::HasPhenotype) > 0 { + let s = ogar_obo::edges::subject(&row.0); + seeds.push(ClassAddr::new(s.classid, s.num)); + if seeds.len() == 64 { + break; + } + } + } + println!(" seeds (diseases asserting a phenotype): {}", seeds.len()); + assert!(!seeds.is_empty(), "no seed carries a phenotype"); + + // ── FALSIFIER: R∃ against an independent local computation ── + let t = std::time::Instant::now(); + let mut checked = 0usize; + let mut widened = 0usize; + for &d in &seeds { + let got = fillers_closed(d, has_phenotype, &fillers, &grounding, 64); + + // Independent path: asserted fillers off the row, then a hand-rolled + // BFS up the grounding spine. No shared code with `fillers_closed`. + let mut asserted: Vec = Vec::new(); + lens.fillers_of( + d.classid, + d.identity, + Predicate::HasPhenotype, + &mut |c, num| asserted.push(ClassAddr::new(c, num)), + ); + let mut want: Vec = Vec::new(); + for &f in &asserted { + let mut queue = std::collections::VecDeque::from([(f, 0usize)]); + let mut seen: Vec = vec![f]; + if let Err(at) = want.binary_search(&f) { + want.insert(at, f); + } + while let Some((x, depth)) = queue.pop_front() { + if depth >= 64 { + continue; + } + lens.grounding_parents_of(x.classid, x.identity, &mut |c, num| { + let p = ClassAddr::new(c, num); + if let Err(at) = seen.binary_search(&p) { + seen.insert(at, p); + queue.push_back((p, depth + 1)); + if let Err(w) = want.binary_search(&p) { + want.insert(w, p); + } + } + }); + } + } + assert_eq!( + got, want, + "R∃ diverges for 0x{:08x}:{}", + d.classid, d.identity + ); + checked += 1; + if got.len() > asserted.len() { + widened += 1; + } + } + let el = t.elapsed(); + + println!("\n=== FALSIFIER (R∃, local) ==="); + println!(" seeds checked : {checked} (0 divergences) in {el:?}"); + println!(" seeds where R∃ widened : {widened} (must be > 0, else generalization is inert)"); + assert!( + widened > 0, + "R∃ never added an ancestor — the rule is doing nothing" + ); + + // ── the Schnittpunkt is_a alone cannot find ── + println!("\n=== CROSS-ONTOLOGY SCHNITTPUNKTE ==="); + + // control: is_a alone must never leave the seed's own ontology + let isa = |a: ClassAddr, visit: &mut dyn FnMut(ClassAddr)| { + lens.parents_of(a.classid, a.identity, &mut |c, num| { + visit(ClassAddr::new(c, num)); + }); + }; + let isa_lens = LensClosure::new(&isa); + let mut isa_cross = 0usize; + let k = seeds.len().min(16); + for i in 0..k { + for j in (i + 1)..k { + if isa_lens + .meet(&[seeds[i], seeds[j]]) + .iter() + .any(|a| a.classid != seeds[i].classid) + { + isa_cross += 1; + } + } + } + println!(" via is_a : {isa_cross} pairs meeting OUTSIDE their own ontology (expected 0)"); + assert_eq!(isa_cross, 0, "subsumption must not cross an ontology"); + + let mut found = 0usize; + let mut empty = 0usize; + let mut root_only = 0usize; + let mut specific = 0usize; + for i in 0..k { + for j in (i + 1)..k { + let m = meet_via( + &[seeds[i], seeds[j]], + has_phenotype, + &fillers, + &grounding, + 64, + ); + if m.is_empty() { + empty += 1; + continue; + } + found += 1; + // The raw meet is sound but vacuous: every phenotype generalizes to + // the HPO root, so every pair "shares" it. Narrow to the most + // specific common filler — the answer that carries information. + let narrowed = most_specific(&m, &grounding, 64); + let is_root_only = narrowed.len() == 1 + && lens + .resolve(narrowed[0].classid, narrowed[0].identity) + .is_some_and(|r| EdgeLanes::new(&rows[r].0).degree(Predicate::IsA) == 0); + if is_root_only { + root_only += 1; + } else { + specific += 1; + if specific <= 3 { + println!( + " 0x{:08x}:{} ∩ 0x{:08x}:{} -> {} shared, {} most-specific, e.g. 0x{:08x}:{}", + seeds[i].classid, + seeds[i].identity, + seeds[j].classid, + seeds[j].identity, + m.len(), + narrowed.len(), + narrowed[0].classid, + narrowed[0].identity + ); + } + } + } + } + println!(" via role : {found} pairs share something | {empty} disjoint"); + println!(" narrowed : {specific} pairs share something SPECIFIC | {root_only} only the root"); + assert!( + found > 0, + "the role walk found no Schnittpunkt — is_a-only behaviour" + ); + assert!( + specific > 0, + "no pair shared anything below the root — narrowing found nothing informative" + ); + assert!( + root_only > 0 || empty > 0, + "every pair sharing something specific would mean the walk discriminates nothing" + ); + + println!("\nS4b VERIFIED: R∃ sound on seeded queries, roles cross, meets discriminate."); +} diff --git a/crates/ogar-elk/examples/elk_lens_vs_closure.rs b/crates/ogar-elk/examples/elk_lens_vs_closure.rs new file mode 100644 index 00000000..740d8d65 --- /dev/null +++ b/crates/ogar-elk/examples/elk_lens_vs_closure.rs @@ -0,0 +1,149 @@ +//! The falsifier for the lazy lens, plus an honest cost comparison. +//! +//! The lens exists to reason over a hydrated table without copying it. That is +//! only worth anything if it gives the **same answers** as the indexed +//! [`Closure`], so this runs both over the real bake and compares them class by +//! class — not on a sample, on every class that has a parent. +//! +//! It also reports what the lens costs. The lens is not free and is not +//! uniformly faster; the numbers below are printed whether or not they flatter +//! it, because a lens that quietly lost on every axis would still pass a +//! correctness check. +//! +//! Usage: `cargo run --release -p ogar-elk --example elk_lens_vs_closure -- [slab.soa]` + +use ogar_elk::{ClassAddr, Closure, LensClosure, Subsumption}; +use ogar_obo::{NODE_ROW_STRIDE, Row512, edges, spine::SpineLens}; + +fn main() { + let path = std::env::args() + .nth(1) + .unwrap_or_else(|| "/tmp/obo/obo-core.soa".to_string()); + let bytes = std::fs::read(&path).unwrap_or_else(|e| panic!("read {path}: {e}")); + assert!(bytes.len().is_multiple_of(NODE_ROW_STRIDE), "not a slab"); + + // Harness copy into an aligned buffer. On the real path Lance hands over an + // arrow-aligned FixedSizeBinary(512) column and this step does not exist. + let n = bytes.len() / NODE_ROW_STRIDE; + let mut rows = vec![Row512::zeroed(); n]; + for (i, r) in rows.iter_mut().enumerate() { + r.0.copy_from_slice(&bytes[i * NODE_ROW_STRIDE..(i + 1) * NODE_ROW_STRIDE]); + } + println!("slab {path}\n rows {n}"); + + // ── the lens: borrow, assert the precondition, wrap in a Parents closure ── + let lens = SpineLens::new(&rows); + assert!( + lens.is_sorted(), + "binary-search resolve requires ascending (classid, num)" + ); + let parents = |a: ClassAddr, visit: &mut dyn FnMut(ClassAddr)| { + lens.parents_of(a.classid, a.identity, &mut |c, num| { + visit(ClassAddr::new(c, num)); + }); + }; + let lc = LensClosure::new(&parents); + + // ── the indexed closure, for comparison ── + let t = std::time::Instant::now(); + let cl = Closure::from_asserted(edges::subsumptions(&rows).map(|(s, p)| { + Subsumption::new( + ClassAddr::new(s.classid, s.num), + ClassAddr::new(p.classid, p.num), + ) + })); + let build = t.elapsed(); + let subs: Vec = cl.subclasses().collect(); + println!( + " indexed closure: {} asserted edges over {} subclasses, built in {build:?}", + cl.asserted_len(), + subs.len() + ); + println!(" lens: 0 edges indexed, 0 build time — it borrows the rows\n"); + + // ── THE FALSIFIER: identical ancestor sets, every class, no sampling ── + let t = std::time::Instant::now(); + let mut total = 0usize; + let mut diverged = 0usize; + let mut first_bad: Option = None; + for &c in &subs { + let a = lc.supers_of(c); + let mut b: Vec = cl.supers_of(c).into_iter().collect(); + b.sort_unstable(); + if a != b { + diverged += 1; + if first_bad.is_none() { + first_bad = Some(c); + } + } + total += a.len(); + } + let lens_walk = t.elapsed(); + + let t = std::time::Instant::now(); + let control: usize = subs.iter().map(|&c| cl.supers_of(c).len()).sum(); + let idx_walk = t.elapsed(); + + println!("=== FALSIFIER ==="); + println!(" classes compared : {}", subs.len()); + println!( + " ancestor sets differing: {diverged} (must be 0){}", + first_bad.map_or(String::new(), |c| format!(" first: {c:?}")) + ); + println!(" transitive ancestors : lens {total} | indexed {control}"); + assert_eq!(diverged, 0, "the lens disagrees with the indexed closure"); + assert_eq!(total, control, "ancestor totals differ"); + + // Cycles: the lens is per-class by design (it has no membership roll). + let lens_cycles = subs.iter().filter(|&&c| lc.reaches_itself(c)).count(); + let idx_cycles = cl.equivalence_cycles().len(); + println!(" equivalence cycles : lens {lens_cycles} | indexed {idx_cycles}"); + assert_eq!(lens_cycles, idx_cycles); + + println!("\n=== COST (honest) ==="); + println!(" build : lens 0s | indexed {build:?}"); + println!(" full sweep : lens {lens_walk:?} | indexed {idx_walk:?}"); + let ratio = lens_walk.as_secs_f64() / idx_walk.as_secs_f64().max(1e-9); + println!(" sweep ratio : lens is {ratio:.1}x the indexed walk"); + println!( + " residency : lens holds a &[Row512] + a depth cap; the index holds a \n\ + \x20 HashMap over {} edges for the life of the query set", + cl.asserted_len() + ); + + // ── Schnittpunkte: the cross-ontology x-hop lookup ── + println!("\n=== SCHNITTPUNKTE ==="); + let mut seeds: Vec = Vec::new(); + let mut by_class: std::collections::BTreeMap = Default::default(); + for &c in &subs { + by_class.entry(c.classid).or_insert(c); + } + for (_, c) in by_class.iter().take(2) { + seeds.push(*c); + } + if seeds.len() == 2 { + let m = lc.meet(&seeds); + println!( + " meet({:?}) -> {} shared ancestor(s)", + seeds + .iter() + .map(|s| format!("0x{:08x}:{}", s.classid, s.identity)) + .collect::>(), + m.len() + ); + for a in m.iter().take(5) { + println!(" 0x{:08x}:{}", a.classid, a.identity); + } + // a class always meets itself in its own ancestry — a control that the + // primitive is not simply returning empty for everything + let self_meet = lc.meet(&[seeds[0], seeds[0]]); + println!( + " control meet(x,x) -> {} (equals |ancestors(x)| = {})", + self_meet.len(), + lc.supers_of(seeds[0]).len() + ); + assert_eq!(self_meet.len(), lc.supers_of(seeds[0]).len()); + } + + println!("\nLENS VERIFIED: identical answers, zero index built."); +} diff --git a/crates/ogar-elk/examples/elk_over_bake.rs b/crates/ogar-elk/examples/elk_over_bake.rs new file mode 100644 index 00000000..461e51d4 --- /dev/null +++ b/crates/ogar-elk/examples/elk_over_bake.rs @@ -0,0 +1,115 @@ +//! ELK over the **baked** substrate — the post-bake observation path. +//! +//! Reads a `.soa` slab, peeks the `is_a` links out of each row's own edge lanes +//! (`ogar_obo::edges::subsumptions`), closes over them, and reports what +//! follows. No `.obo` file is opened; no CURIE is parsed. The only input is +//! positions in baked rows, which is what makes this observation rather than a +//! second harvest. +//! +//! Usage: `cargo run -p ogar-elk --example elk_over_bake -- ` + +use ogar_elk::{ClassAddr, Closure, Subsumption}; +use ogar_obo::{NODE_ROW_STRIDE, Row512, edges}; + +fn main() { + let path = std::env::args() + .nth(1) + .unwrap_or_else(|| "/tmp/obo/obo-core.soa".to_string()); + let bytes = std::fs::read(&path).unwrap_or_else(|e| panic!("read {path}: {e}")); + assert!( + bytes.len().is_multiple_of(NODE_ROW_STRIDE), + "not a slab: {} bytes is not a multiple of {NODE_ROW_STRIDE}", + bytes.len() + ); + + // A harness copy into an aligned buffer. On the real path Lance hands over + // an arrow-aligned FixedSizeBinary(512) column and this step does not exist. + let n = bytes.len() / NODE_ROW_STRIDE; + let mut rows = vec![Row512::zeroed(); n]; + for (i, r) in rows.iter_mut().enumerate() { + r.0.copy_from_slice(&bytes[i * NODE_ROW_STRIDE..(i + 1) * NODE_ROW_STRIDE]); + } + println!("slab: {path}\n rows: {n}"); + + // ── the peek: is_a links straight out of the lanes ─────────────────────── + let t0 = std::time::Instant::now(); + let closure = Closure::from_asserted(edges::subsumptions(&rows).map(|(sub, sup)| { + Subsumption::new( + ClassAddr::new(sub.classid, sub.num), + ClassAddr::new(sup.classid, sup.num), + ) + })); + let build = t0.elapsed(); + println!( + " asserted subsumptions read from lanes: {} in {build:?}", + closure.asserted_len() + ); + + // Per-class census of what the closure indexed, so the numbers are + // attributable to an ontology rather than a single total. + let mut by_class: std::collections::BTreeMap = Default::default(); + for c in closure.subclasses() { + *by_class.entry(c.classid).or_default() += 1; + } + println!(" classes with at least one parent:"); + for (cid, count) in &by_class { + println!(" 0x{cid:08x}: {count}"); + } + + // ── question 1: what follows ───────────────────────────────────────────── + // Depth and breadth of the spine, sampled over every indexed subclass. + let t1 = std::time::Instant::now(); + let mut max_ancestors = 0usize; + let mut deepest: Option = None; + let mut total = 0usize; + for c in closure.subclasses() { + let a = closure.supers_of(c).len(); + total += a; + if a > max_ancestors { + max_ancestors = a; + deepest = Some(c); + } + } + let walk = t1.elapsed(); + let indexed = by_class.values().sum::(); + println!( + "\n transitive ancestors: {total} over {indexed} subclasses \ + (mean {:.1}, max {max_ancestors} at {deepest:?}) in {walk:?}", + total as f64 / indexed.max(1) as f64 + ); + + // ── question 2: is the baked spine sound ───────────────────────────────── + let cycles = closure.equivalence_cycles(); + println!( + "\n equivalence cycles in the baked spine: {} (depth cap {})", + cycles.len(), + closure.depth_cap() + ); + for c in cycles.iter().take(10) { + println!(" 0x{:08x}:{}", c.classid, c.identity); + } + if cycles.len() > 10 { + println!(" ... {} total", cycles.len()); + } + + // A worked entailment, printed so the run is not merely a count: the + // deepest class and one of its derived ancestors, plus a control that must + // NOT be entailed. + if let Some(d) = deepest { + let supers = closure.supers_of(d); + if let Some(top) = supers.iter().max() { + assert!(closure.entails(d, *top)); + println!( + "\n entails: 0x{:08x}:{} ⊑ 0x{:08x}:{} (derived)", + d.classid, d.identity, top.classid, top.identity + ); + } + // control: an identity no ontology mints (numeric 0 never occurs) + let ghost = ClassAddr::new(d.classid, 0); + assert!( + !closure.entails(d, ghost), + "entailment must be able to say no" + ); + println!(" and does NOT entail a class that was never minted — the walk discriminates"); + } +} diff --git a/crates/ogar-elk/src/lens.rs b/crates/ogar-elk/src/lens.rs new file mode 100644 index 00000000..6df5ec57 --- /dev/null +++ b/crates/ogar-elk/src/lens.rs @@ -0,0 +1,635 @@ +//! The **lazy lens** seam — reasoning against borrowed rows, with no index built. +//! +//! [`Closure`] indexes its edges into a `HashMap` up front. That is the right +//! shape when the asserted set is already in memory and will be queried many +//! times, and the wrong shape when the substrate is a hydrated columnar table +//! you would rather not copy: building the index costs a pass over every edge +//! and holds a map proportional to them, before the first question is asked. +//! +//! This module is the other end. A [`Parents`] implementation answers *one* +//! question — who are this class's asserted superclasses — and [`LensClosure`] +//! computes entailment on top of it, touching only the classes a query actually +//! reaches. +//! +//! # No hashing, anywhere on this path +//! +//! Dedup is an ordered insert and membership is a binary search. Nothing here +//! hashes. That is not a micro-optimisation: the addresses arrive already +//! ordered from the bake, hashing throws that ordering away, and it has to be +//! rebuilt before two ancestries can be intersected. Keeping the order means +//! [`LensClosure::meet`] is one merge pass, and it means two runs produce +//! byte-identical output — the property a layer that emits *facts* has to have. +//! +//! # The lens never caches, and that is the law rather than an oversight +//! +//! A transitive closure is a deterministic function of the asserted edges: it +//! recomputes equal, every time. Under the workspace's zero-copy law that makes +//! it a **projection** — `output_rung == max(input_rungs)` — and a projection is +//! never stored, however much arithmetic produced it. So [`LensClosure`] holds +//! no memo table, no visited cache between calls, and no adjacency copy. It +//! holds a borrowed reference and a depth cap. +//! +//! The cost of that is real and is not hidden: repeated queries redo work. What +//! it buys is that nothing is resident between queries, which is the point when +//! the rows live in a hydrated table an order of magnitude larger than RAM. +//! +//! # What it does NOT change +//! +//! The answers. [`LensClosure::supers_of`] and [`LensClosure::entails`] return +//! exactly what [`Closure`]'s equivalents return over the same asserted set — +//! asserted in the test module, and measured against the real bake in +//! `examples/elk_lens_vs_closure.rs`. A lens that were *cheaper* and *different* +//! would be a different reasoner wearing this one's name. +//! +//! # Where the lens genuinely wins +//! +//! [`LensClosure::entails`] **early-exits** on the first hit. `Closure::entails` +//! is `sub == sup || self.supers_of(sub).contains(&sup)`, which materialises the +//! whole ancestor set before testing membership. For a yes-answer near the query +//! class, the lens stops there; for a no-answer both walk the same ground. + +use std::collections::VecDeque; + +use crate::{ClassAddr, DEFAULT_DEPTH_CAP}; + +/// A source of **asserted** superclasses for a class. +/// +/// One method, taking a visitor rather than returning a collection: an +/// implementation over borrowed rows can then decode links in place and hand +/// them out one at a time, with nothing allocated per call. Returning +/// `Vec` would put an allocation on the hot path of every hop and +/// quietly undo the reason this trait exists. +/// +/// **Asserted, not entailed.** An implementation reports the direct parents it +/// holds and performs no transitive reasoning of its own — that is +/// [`LensClosure`]'s job, and splitting it this way is what lets the same lens +/// serve a closure, a one-hop query, or a cycle check without any of them +/// re-deriving what the others did. +pub trait Parents { + /// Visit each asserted superclass of `c`, in any order. + /// + /// Called once per class per walk, so an implementation should avoid + /// allocating. Duplicates are permitted; [`LensClosure`] de-duplicates. + fn parents_of(&self, c: ClassAddr, visit: &mut dyn FnMut(ClassAddr)); +} + +/// Any closure of the right shape is a [`Parents`] source. +/// +/// This is what keeps the seam dependency-free in both directions: the crate +/// that owns the byte layout can hand over a closure capturing its own borrowed +/// view, and neither crate has to name the other's types. There is no adapter +/// struct to write and no `ogar-obo`/`ogar-elk` dependency edge in either +/// direction. +impl Parents for F +where + F: Fn(ClassAddr, &mut dyn FnMut(ClassAddr)), +{ + #[inline] + fn parents_of(&self, c: ClassAddr, visit: &mut dyn FnMut(ClassAddr)) { + self(c, visit); + } +} + +/// Entailment computed against a [`Parents`] source, materialising nothing. +/// +/// Borrows its source; holds no state beyond the depth cap. Constructing one is +/// free, so it is meant to be created per query rather than kept. +#[derive(Debug, Clone, Copy)] +pub struct LensClosure<'p, P: Parents + ?Sized> { + parents: &'p P, + depth_cap: usize, +} + +impl<'p, P: Parents + ?Sized> LensClosure<'p, P> { + /// Borrow a parent source at the default depth cap. + #[must_use] + pub const fn new(parents: &'p P) -> Self { + Self { + parents, + depth_cap: DEFAULT_DEPTH_CAP, + } + } + + /// Override the transitivity depth guard. + #[must_use] + pub const fn with_depth_cap(mut self, cap: usize) -> Self { + self.depth_cap = cap; + self + } + + /// The depth guard in force — so a caller can tell "not entailed" from + /// "the walk stopped", which are different answers. + #[must_use] + pub const fn depth_cap(&self) -> usize { + self.depth_cap + } + + /// **R1 + R2** — every superclass of `c`, transitively. + /// + /// Breadth-first, so the walk terminates at `depth_cap` hops rather than at + /// `depth_cap` nodes. Excludes `c` itself unless a cycle genuinely returns + /// to it — the signal [`Self::reaches_itself`] reads. + /// + /// The returned set is the caller's answer, not a cache: it is dropped when + /// the caller drops it, and asking twice walks twice. + #[must_use] + pub fn supers_of(&self, c: ClassAddr) -> Vec { + let mut seen: Vec = Vec::new(); + let mut queue = VecDeque::from([(c, 0usize)]); + while let Some((node, depth)) = queue.pop_front() { + if depth >= self.depth_cap { + continue; + } + self.parents.parents_of(node, &mut |p| { + // Ordered insert, not a hash probe. The result stays sorted by + // construction, which is what makes `meet` a merge pass — and + // it is deterministic run to run, which a hash set is not. + if let Err(at) = seen.binary_search(&p) { + seen.insert(at, p); + queue.push_back((p, depth + 1)); + } + }); + } + seen + } + + /// **Does `sub ⊑ sup` follow?** Reflexive per R1. + /// + /// Early-exits on the first hit instead of building the full ancestor set, + /// which is the one place the lens is strictly cheaper than the indexed + /// [`Closure`](crate::Closure) rather than merely leaner. + #[must_use] + pub fn entails(&self, sub: ClassAddr, sup: ClassAddr) -> bool { + if sub == sup { + return true; + } + let mut seen: Vec = Vec::new(); + let mut queue = VecDeque::from([(sub, 0usize)]); + while let Some((node, depth)) = queue.pop_front() { + if depth >= self.depth_cap { + continue; + } + let mut hit = false; + self.parents.parents_of(node, &mut |p| { + if p == sup { + hit = true; + } + if let Err(at) = seen.binary_search(&p) { + seen.insert(at, p); + queue.push_back((p, depth + 1)); + } + }); + if hit { + return true; + } + } + false + } + + /// Ancestors as a **sorted, de-duplicated** address list. + /// + /// The shape [`Self::supers_of`] should have had from the start. A hash set + /// discards the ordering the bake already provides — rows are emitted + /// ascending by `(classid, identity)` — and that ordering is exactly what + /// makes [`meet`](Self::meet) a single merge pass instead of a lookup + /// storm. Sorted output is also deterministic run to run, which a hash set + /// is not, and determinism is the property a fact-producing layer is + /// supposed to have. + #[must_use] + pub fn ancestors_sorted(&self, c: ClassAddr) -> Vec { + // already sorted by construction — supers_of does ordered inserts + self.supers_of(c) + } + + /// **Schnittpunkte** — the addresses reachable from *every* seed. + /// + /// This is the multi-hop cross-ontology lookup, and it is deliberately not + /// a matrix product: an x-hop question across ontologies asks where the + /// reachable sets MEET, and meeting is an intersection over sorted + /// addresses — one merge pass per seed, no vectors, no per-edge payload, + /// nothing serialized. + /// + /// Returns sorted addresses. An empty seed list yields nothing (there is no + /// meaningful "intersection of no sets" here — the identity would be every + /// address in existence, which is not an answer a caller can use). + /// + /// The seeds may sit in different ontologies; the result names wherever + /// their ancestries coincide, which is the fact the caller is after. + #[must_use] + pub fn meet(&self, seeds: &[ClassAddr]) -> Vec { + let Some((first, rest)) = seeds.split_first() else { + return Vec::new(); + }; + let mut acc = self.ancestors_sorted(*first); + for s in rest { + if acc.is_empty() { + break; + } + acc = intersect_sorted(&acc, &self.ancestors_sorted(*s)); + } + acc + } + + /// Whether `c` reaches itself — an equivalence cycle through `c`. + /// + /// Per-class rather than whole-graph: a lens has no membership roll to + /// enumerate, and inventing one would mean materialising the very index + /// this module exists to avoid. A caller that wants every cycle iterates + /// the classes it cares about; a caller that has the full roll can use + /// [`Closure::equivalence_cycles`](crate::Closure::equivalence_cycles). + #[must_use] + pub fn reaches_itself(&self, c: ClassAddr) -> bool { + self.supers_of(c).binary_search(&c).is_ok() + } +} + +/// Intersect two sorted, de-duplicated address lists in one pass. +/// +/// Linear in the inputs and allocation-free apart from the result, which is why +/// the sorted representation is worth keeping: the same job over hash sets is a +/// probe per element and returns in arbitrary order. +#[must_use] +pub fn intersect_sorted(a: &[ClassAddr], b: &[ClassAddr]) -> Vec { + let (mut i, mut j) = (0usize, 0usize); + let mut out = Vec::new(); + while i < a.len() && j < b.len() { + match a[i].cmp(&b[j]) { + core::cmp::Ordering::Less => i += 1, + core::cmp::Ordering::Greater => j += 1, + core::cmp::Ordering::Equal => { + out.push(a[i]); + i += 1; + j += 1; + } + } + } + out +} + +// ── S4b: the existential arm (R∃) ──────────────────────────────────────────── + +/// An existential role — the `r` in `A ⊑ ∃r.B`. +/// +/// A plain tag rather than a rich type, so this crate stays zero-dep and does +/// not need the producer's predicate enum. It is deliberately NOT a +/// [`Subsumption`](crate::Subsumption): a role is an existential restriction, +/// and the two compose by different rules. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Role(pub u8); + +/// A source of **asserted existential fillers** for a class under one role. +/// +/// Same visitor shape as [`Parents`], and for the same reason: an +/// implementation over borrowed rows hands out fillers one at a time with +/// nothing allocated per call. +pub trait Fillers { + /// Visit each asserted filler of `role` on `c`, in any order. + fn fillers_of(&self, c: ClassAddr, role: Role, visit: &mut dyn FnMut(ClassAddr)); +} + +impl Fillers for F +where + F: Fn(ClassAddr, Role, &mut dyn FnMut(ClassAddr)), +{ + #[inline] + fn fillers_of(&self, c: ClassAddr, role: Role, visit: &mut dyn FnMut(ClassAddr)) { + self(c, role, visit); + } +} + +/// **R∃** — `A ⊑ ∃r.B` and `B ⊑ C` give `A ⊑ ∃r.C`. +/// +/// Returns every filler of `role` on `subject`, each generalized upward through +/// `grounding_spine`. Sorted and de-duplicated; no hashing. +/// +/// # The two spines are NOT the same, and swapping them is the classic error +/// +/// `grounding_spine` is the spine a **filler** generalizes up. For the OBO core +/// that is `is_a ∪ part_of`: an anatomical filler's grounding is inherited +/// through mereology as well as taxonomy, which is the deductive form of +/// "inherit grounding". It is deliberately a DIFFERENT source from the one +/// [`LensClosure::entails`] uses, which must be `is_a` alone. +/// +/// The distinction is exactly the false-ancestor hazard, and it is easy to get +/// backwards: generalizing a **subject** through `part_of` is unsound +/// (`A part_of B`, `B ⊑ C` does NOT give `A ⊑ C`); generalizing a **filler** +/// through it is sound and is what this function does. Both sources are +/// therefore explicit parameters — there is no default that could silently be +/// the wrong one. +/// +/// Note this generalizes the filler only. A caller wanting "every phenotype of +/// this disease INCLUDING those asserted on its superclasses" walks the +/// subject's own subsumption ancestors first and unions the results — that is a +/// larger question, and keeping it out of here is what lets this match the +/// producer's own saturation count exactly. +#[must_use] +pub fn fillers_closed( + subject: ClassAddr, + role: Role, + fillers: &F, + grounding_spine: &S, + depth_cap: usize, +) -> Vec +where + S: Parents + ?Sized, + F: Fillers + ?Sized, +{ + let mut out: Vec = Vec::new(); + let spine = LensClosure::new(grounding_spine).with_depth_cap(depth_cap); + fillers.fillers_of(subject, role, &mut |o| { + if let Err(at) = out.binary_search(&o) { + out.insert(at, o); + } + for a in spine.supers_of(o) { + if let Err(at) = out.binary_search(&a) { + out.insert(at, a); + } + } + }); + out +} + +/// The **cross-ontology Schnittpunkt** — where several subjects' role-fillers +/// meet, after R∃ generalization. +/// +/// This is what `meet` over `is_a` alone cannot answer: subsumption never +/// leaves its own ontology, so two diseases only coincide once you cross by a +/// role (`has_phenotype` into HPO, `has_anatomy` into Uberon) and then climb +/// the filler's own spine. Intersection over sorted addresses, one merge pass +/// per seed. +/// +/// Empty seeds yield nothing — the identity of an intersection over no sets is +/// not an answer a caller can use. +#[must_use] +pub fn meet_via( + seeds: &[ClassAddr], + role: Role, + fillers: &F, + grounding_spine: &S, + depth_cap: usize, +) -> Vec +where + S: Parents + ?Sized, + F: Fillers + ?Sized, +{ + let Some((first, rest)) = seeds.split_first() else { + return Vec::new(); + }; + let mut acc = fillers_closed(*first, role, fillers, grounding_spine, depth_cap); + for s in rest { + if acc.is_empty() { + break; + } + acc = intersect_sorted( + &acc, + &fillers_closed(*s, role, fillers, grounding_spine, depth_cap), + ); + } + acc +} + +/// Keep only the **most specific** members of a sorted address set — those that +/// are not an ancestor of another member. +/// +/// # Why a raw meet is sound and useless +/// +/// Every filler generalizes up to its ontology's root, so ANY two subjects +/// share that root and [`meet_via`] returns it. Measured on the OBO core: 120 +/// of 120 sampled disease pairs "shared a phenotype", and in every case the +/// shared address was `HP:0000001` — "All". A result that fires on every input +/// carries exactly as much information as one that never fires. +/// +/// The informative answer to *what do these two subjects share* is the most +/// specific common filler, which is what this returns: drop `x` when some other +/// member `y` has `x` among its ancestors, because `y` already says everything +/// `x` does and says it more precisely. +/// +/// Cost is |set|² spine walks in the worst case, which is fine because a meet +/// over seeded facts is small. It is deliberately NOT folded into [`meet_via`]: +/// the raw meet is the sound object, and narrowing is a presentation choice a +/// caller makes. +#[must_use] +pub fn most_specific(set: &[ClassAddr], spine: &S, depth_cap: usize) -> Vec +where + S: Parents + ?Sized, +{ + let lens = LensClosure::new(spine).with_depth_cap(depth_cap); + let mut out = Vec::new(); + for (i, x) in set.iter().enumerate() { + let redundant = set + .iter() + .enumerate() + .any(|(j, y)| i != j && lens.supers_of(*y).binary_search(x).is_ok()); + if !redundant { + out.push(*x); + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{Closure, Subsumption}; + + const NS: u32 = 0x0301_0000; + + fn c(id: u32) -> ClassAddr { + ClassAddr::new(NS, id) + } + + /// A deliberately dumb source: a linear scan of asserted pairs. Slow on + /// purpose — it shares no code with `Closure`'s HashMap index, so agreement + /// between the two is evidence rather than tautology. + struct Pairs(Vec<(u32, u32)>); + + impl Parents for Pairs { + fn parents_of(&self, x: ClassAddr, visit: &mut dyn FnMut(ClassAddr)) { + for (a, b) in &self.0 { + if c(*a) == x { + visit(c(*b)); + } + } + } + } + + fn both(edges: &[(u32, u32)]) -> (Closure, Pairs) { + let cl = Closure::from_asserted(edges.iter().map(|(a, b)| Subsumption::new(c(*a), c(*b)))); + (cl, Pairs(edges.to_vec())) + } + + #[test] + fn the_lens_agrees_with_the_indexed_closure() { + let edges = [(1, 2), (2, 3), (3, 4), (5, 3), (6, 7)]; + let (cl, src) = both(&edges); + let lens = LensClosure::new(&src); + for i in 1..=8u32 { + let mut want: Vec<_> = cl.supers_of(c(i)).into_iter().collect(); + want.sort_unstable(); + assert_eq!(lens.supers_of(c(i)), want, "ancestor sets diverge at {i}"); + for j in 1..=8u32 { + assert_eq!( + lens.entails(c(i), c(j)), + cl.entails(c(i), c(j)), + "entailment diverges at {i} -> {j}" + ); + } + } + } + + /// Anti-vacuity for the test above: the fixture must contain both + /// entailed and NOT-entailed pairs, or agreement proves nothing. + #[test] + fn the_agreement_fixture_discriminates() { + let edges = [(1, 2), (2, 3), (3, 4), (5, 3), (6, 7)]; + let (_, src) = both(&edges); + let lens = LensClosure::new(&src); + assert!(lens.entails(c(1), c(4)), "derived through three hops"); + assert!( + !lens.entails(c(1), c(7)), + "must NOT entail a separate component" + ); + assert!(!lens.entails(c(4), c(1)), "and must not run the wrong way"); + let reached: usize = (1..=8u32).map(|i| lens.supers_of(c(i)).len()).sum(); + assert!( + reached > 0 && reached < 8 * 8, + "fixture is neither empty nor total" + ); + } + + #[test] + fn reflexivity_holds_with_no_edges_at_all() { + let src = Pairs(vec![]); + let lens = LensClosure::new(&src); + assert!(lens.entails(c(7), c(7))); + assert!(lens.supers_of(c(7)).is_empty()); + } + + #[test] + fn a_cycle_is_seen_and_a_clean_spine_is_not() { + let cyc = Pairs(vec![(1, 2), (2, 1)]); + let lens = LensClosure::new(&cyc); + assert!(lens.reaches_itself(c(1))); + assert!(lens.reaches_itself(c(2))); + + // can-stay-silent, over a NON-trivial acyclic graph + let clean = Pairs(vec![(1, 2), (2, 3), (1, 3), (4, 3)]); + let lens = LensClosure::new(&clean); + for i in 1..=4u32 { + assert!(!lens.reaches_itself(c(i)), "{i} is not on a cycle"); + } + } + + #[test] + fn the_depth_cap_binds_and_is_visible() { + let chain: Vec<(u32, u32)> = (1..10).map(|i| (i, i + 1)).collect(); + let src = Pairs(chain); + let shallow = LensClosure::new(&src).with_depth_cap(2); + assert_eq!(shallow.depth_cap(), 2); + assert!(!shallow.entails(c(1), c(10)), "beyond the cap"); + assert!(shallow.entails(c(1), c(3)), "within the cap"); + assert!( + LensClosure::new(&src).entails(c(1), c(10)), + "reachable at the default cap" + ); + } + + #[test] + fn intersect_is_a_single_merge_pass_and_can_return_nothing() { + let a: Vec = [1u32, 3, 5, 7].iter().map(|i| c(*i)).collect(); + let b: Vec = [3u32, 4, 5, 9].iter().map(|i| c(*i)).collect(); + assert_eq!(intersect_sorted(&a, &b), vec![c(3), c(5)]); + // anti-vacuity: disjoint inputs must yield empty, not everything + let d: Vec = [2u32, 6].iter().map(|i| c(*i)).collect(); + assert!(intersect_sorted(&a, &d).is_empty()); + assert!(intersect_sorted(&a, &[]).is_empty()); + } + + /// The Schnittpunkt: two classes in DIFFERENT namespaces whose ancestries + /// coincide at a shared address. This is the cross-ontology x-hop lookup, + /// and it is an intersection — no vectors, no matrix product. + #[test] + fn meet_finds_where_two_ontologies_coincide() { + let other = 0x0302_0000; + let shared = ClassAddr::new(NS, 100); + let f = move |x: ClassAddr, visit: &mut dyn FnMut(ClassAddr)| { + // MONDO:1 -> MONDO:100 ; HP:1 -> MONDO:100 ; MONDO:2 -> MONDO:200 + if x == ClassAddr::new(NS, 1) || x == ClassAddr::new(other, 1) { + visit(shared); + } + if x == ClassAddr::new(NS, 2) { + visit(ClassAddr::new(NS, 200)); + } + }; + let lens = LensClosure::new(&f); + assert_eq!( + lens.meet(&[ClassAddr::new(NS, 1), ClassAddr::new(other, 1)]), + vec![shared], + "the two ancestries meet at exactly one address" + ); + // can-stay-silent: seeds that share nothing produce no Schnittpunkt + assert!( + lens.meet(&[ClassAddr::new(NS, 1), ClassAddr::new(NS, 2)]) + .is_empty(), + "disjoint ancestries must not manufacture a meeting point" + ); + assert!(lens.meet(&[]).is_empty(), "no seeds is not 'everything'"); + } + + #[test] + fn ancestors_sorted_is_sorted_deduped_and_agrees_with_the_set() { + let edges = [(1, 2), (2, 3), (1, 3), (3, 4)]; + let (cl, src) = both(&edges); + let lens = LensClosure::new(&src); + let v = lens.ancestors_sorted(c(1)); + assert!(v.windows(2).all(|w| w[0] < w[1]), "sorted and deduped"); + let mut want = cl.supers_of(c(1)).into_iter().collect::>(); + want.sort_unstable(); + assert_eq!(v, want, "same content as the indexed closure"); + } + + /// A closure is a `Parents` source with no adapter — the property that + /// keeps the seam dependency-free in both directions. + #[test] + fn a_bare_closure_is_a_parent_source() { + let f = |x: ClassAddr, visit: &mut dyn FnMut(ClassAddr)| { + if x == c(1) { + visit(c(2)); + } + if x == c(2) { + visit(c(3)); + } + }; + let lens = LensClosure::new(&f); + assert!(lens.entails(c(1), c(3))); + assert!(!lens.entails(c(3), c(1))); + } + + /// `entails` must not walk further than it has to. The source counts its + /// own calls, so "early exit" is measured rather than asserted. + #[test] + fn entails_early_exits_instead_of_building_the_whole_set() { + use std::cell::Cell; + let calls = Cell::new(0usize); + let chain: Vec<(u32, u32)> = (1..40).map(|i| (i, i + 1)).collect(); + let counting = |x: ClassAddr, visit: &mut dyn FnMut(ClassAddr)| { + calls.set(calls.get() + 1); + for (a, b) in &chain { + if c(*a) == x { + visit(c(*b)); + } + } + }; + let lens = LensClosure::new(&counting); + + assert!(lens.entails(c(1), c(3))); + let near = calls.get(); + calls.set(0); + let _ = lens.supers_of(c(1)); + let full = calls.get(); + assert!( + near < full, + "early exit must touch fewer classes than the full walk ({near} vs {full})" + ); + } +} diff --git a/crates/ogar-elk/src/lib.rs b/crates/ogar-elk/src/lib.rs index 334a57a7..acc747e4 100644 --- a/crates/ogar-elk/src/lib.rs +++ b/crates/ogar-elk/src/lib.rs @@ -2,9 +2,10 @@ //! //! # What this is, and the two questions it answers //! -//! Third member of the factfinder family beside [`ogar-obo`] (harvest) and -//! [`ogar-ro`] (the predicate palette). Those two say what is *asserted*; this -//! one says what *follows*. It answers exactly two questions: +//! Third member of the factfinder family beside [`ogar-obo`] (harvest and bake) +//! and [`ogar-ro`] (the predicate palette). Those two say what is *asserted*; +//! this one observes the baked result and says what *follows*. It answers +//! exactly two questions: //! //! 1. **Does `A ⊑ B` follow** from the asserted edges? ([`Closure::entails`]) //! 2. **Is adding a set of axioms sound** — does the merged closure introduce @@ -18,29 +19,34 @@ //! [`ogar-obo`]: https://docs.rs/ogar-obo //! [`ogar-ro`]: https://docs.rs/ogar-ro //! -//! # Where this runs: BEFORE the bake, never after +//! # Where this runs: AFTER the bake, as observation //! -//! **The joins are pre-bake.** Reconciling independently authored sources — -//! deciding which assertions corroborate, which enrich, and which contradict — -//! is exactly the work that must finish before anything is baked, because the -//! bake is what freezes the answer into positions. So this crate is a **stage**, -//! not a layer: it runs once, upstream, and then it is done. +//! **The joins are pre-bake; this is not a join.** Reconciling independently +//! authored sources is upstream work that must finish before anything is baked, +//! because the bake is what freezes the answer into positions. What this crate +//! does is the opposite end: it **observes the baked substrate** and reports +//! what follows from it. The bake is its input, not its output. +//! +//! Concretely, its input is the `is_a` links carried in a baked row's own edge +//! lanes — `ogar_obo::edges::subsumptions` is the adapter, and it exists so a +//! reasoner reads addresses out of positions rather than re-deriving them. //! //! Three consequences follow, and they are the whole design: //! -//! 1. **Types are legal here.** [`ClassAddr`] and [`Subsumption`] exist because -//! a join needs a key and a directed edge. They are pre-bake scaffolding. -//! **Nothing this crate defines survives the bake** — afterwards there are -//! only classes, and a class is resolved by position, not by a type declared -//! here. If one of these types ever appears in a post-bake read path, that is -//! the leak, and it is this doc that says so. +//! 1. **These are readings, not schema.** [`ClassAddr`] is the `(classid, +//! identity)` pair a baked key already carries, decoded from its position — +//! the same shape `ogar_obo::edges::Link` reads out of a lane. It declares +//! nothing about a class and outlives nothing; a class is still resolved by +//! position. [`Subsumption`] is a direction, and it is a distinct type only +//! so a caller cannot hand this crate a `part_of` edge by accident. //! 2. **Nothing here is in the hot path**, so nothing here may pretend to be. //! There is no serialization surface — not behind a feature, not optionally. -//! A join validator that could serialize its verdict would invite someone to -//! ship the verdict instead of the bake. -//! 3. **No file, no CURIE, no label.** Input is already-joined addressed edges. +//! An observer that could serialize its verdict would invite someone to ship +//! the verdict as if it were substrate. +//! 3. **No file, no CURIE, no label.** Input is addressed edges read from baked +//! rows — which carry no CURIE and no label to reach for in the first place. //! Reaching back for the source document would put parsing inside the -//! validator and re-introduce the coupling the bake exists to remove. +//! observer and re-introduce the coupling the bake exists to remove. //! //! # The fragment, stated precisely //! @@ -83,6 +89,10 @@ #![forbid(unsafe_code)] +pub mod lens; + +pub use lens::{Fillers, LensClosure, Parents, Role, fillers_closed, meet_via, most_specific}; + use std::collections::{HashMap, HashSet, VecDeque}; /// The **join key** the pre-bake reconciliation uses: `(classid, identity)`. diff --git a/crates/ogar-obo/Cargo.toml b/crates/ogar-obo/Cargo.toml index 2414cf82..390da108 100644 --- a/crates/ogar-obo/Cargo.toml +++ b/crates/ogar-obo/Cargo.toml @@ -11,9 +11,28 @@ description = "OBO-core reference bake — MONDO (disease) · HPO (phenotype) · [features] default = [] serde = ["dep:serde"] +# RDF front-end. Off by default so the crate keeps its "deps: ogar-vocab only" +# lean profile; the three ox* crates are pure Rust (no C, no build script) and +# are pinned to the SAME versions lance-graph-ontology pins, so a workspace +# that builds both resolves one copy of oxrdf. +rdf = ["dep:oxrdf", "dep:oxrdfxml", "dep:oxttl"] [dependencies] ogar-vocab = { path = "../ogar-vocab" } serde = { workspace = true, optional = true } +oxrdf = { version = "0.3", optional = true } +oxrdfxml = { version = "0.2", optional = true } +oxttl = { version = "0.2", optional = true } [dev-dependencies] + +# These drive the RDF front-end, so they cannot build without it. Declared +# rather than #[cfg]-guarded so `cargo clippy --all-targets` on the default +# feature set skips them instead of failing. +[[example]] +name = "probe_spine" +required-features = ["rdf"] + +[[example]] +name = "bake_spine" +required-features = ["rdf"] diff --git a/crates/ogar-obo/docs/EDGE-LANES.md b/crates/ogar-obo/docs/EDGE-LANES.md new file mode 100644 index 00000000..7329b6da --- /dev/null +++ b/crates/ogar-obo/docs/EDGE-LANES.md @@ -0,0 +1,196 @@ +# Edge lanes — the `col_idx` half of the CSR + +> Status: **shipped**, measured on the full OBO core. +> Code: `crates/ogar-obo/src/edges.rs` · producer `bake()` · reader `EdgeLanes`. +> Probes: `examples/probe_edge_capacity.rs`, `examples/verify_edges.rs`. + +## The gap this closes + +A baked row is `key(16) | edges(16) | value(480)`. The `edges` block is a +per-predicate **degree histogram**: one byte per `Predicate` discriminant, +saturating at 255. It says *how many* targets a node has per predicate. + +It does not say *which*. In CSR terms the bake persisted `row_ptr` and dropped +`col_idx`. The consequence is sharp and was worth stating plainly before any +code was written: **a row could describe its own local shape and still not be +traversed.** Every traversal had to fall back to the out-of-line SPO triple +table, which is a different artifact with a different lifetime. + +This module puts the targets in the row. The degree histogram thereby stops +being decorative and becomes load-bearing — it is the index that tells a reader +which predicate each successive target belongs to. + +## The lane form + +A lane is the one form the substrate has: `classid(4) + 12 content bytes`, and +**the classid says how to read its own 12 bytes.** An edge lane reads its +content as the G2 `4×u24` carving — four literal links — and carries **the +target ontology's own row classid**. + +```text +lane: | classid: u32 | u24 | u24 | u24 | u24 | + ^ target ontology ^ four CURIE numerics +``` + +So a link is exactly the `(classid, identity)` pair a *key* carries. The same +binary search that resolves a key resolves a link; nothing new is invented to +point at a node. `edges::subject()` and `edges::Link` are deliberately the same +shape, because a subject and a target are the same kind of thing. + +### Why `4×u24` and not the `6×(u8:u8)` rails + +The L1 rails are the cheaper form and the eventual destination — but a rail ref +is **basin-local**, capped at 256 targets, and explicitly never a global +pointer. The bake has no basin: HEEL/HIP/TWIG and leaf are **zero on all 68,797 +rows** (the cascade is dormant, RESERVE-DON'T-RECLAIM), while identities run +past 8.9 million. With no basin to be local to, a one-byte ref has nothing to +mean. + +So the G2 carving is spent exactly where a rail cannot reach, and only there. +When an HHTL basin mint wakes routing, the rails become the cheaper in-basin +form; the offsets are already reserved and the migration costs no re-bake. + +This is also why the *identity* does **not** use G2: it rides the V3 +`family:identity` rail (`family = num >> 16`, `identity = num & 0xFFFF`), which +carries the same 24 bits as an `X:Y` pair with 8 bits of headroom — the thing a +flat u24 cannot do, because it has no axis. Sort order is unchanged, since +`family` holds the high bits, so ordering by `(classid, family, identity)` is +still ordering by `(classid, num)` and binary-search-by-key survives. + +## Layout + +The 480-byte value slab is 30 lanes. + +| lanes | use | +|---|---| +| 0–5 | reserved for further value tenants (RESERVE, DON'T RECLAIM) | +| 6 | `EntityType` tenant — a `u16` namespace ordinal at slab offset 96 | +| 7–29 | **edge targets** — 23 lanes × 4 links = **92 slots** | + +`EDGE_LANE_SLAB_OFFSET = 112`, `EDGE_LANE_COUNT = 23`, `EDGE_SLOT_COUNT = 92`. +A `const _: () = assert!(…)` pins that the edge lanes run to the end of the row +without overlapping a value tenant, so a future tenant carve cannot silently +collide with them. + +## The reader's contract + +Targets are emitted ascending by `(predicate, target namespace, numeric)`, and +**a lane never mixes target ontologies** — its classid names exactly one, so a +namespace change starts a new lane even mid-predicate. That fragmentation is +deliberate: it is what keeps a lane's classid from lying. + +Unused slots in a lane are `0`. That is unambiguous, and it is a *measured* +fact, not an assumption: `probe_edge_capacity` checks assumption **A1** — CURIE +numeric `0` never occurs — across the real sources, and it does not. + +A reader walks lanes in order and consumes `degree(p)` links for each predicate +`p` in ascending order, treating a `0` slot as "this lane is spent, advance". +That is `EdgeLanes`, which borrows the row and decodes a `u24` with a shift — +no allocation, no gathered window, nothing owned that the lane already holds. + +```rust +let lanes = EdgeLanes::new(&row.0); +lanes.degree(Predicate::IsA); // row_ptr: one byte, zero value decode +lanes.links_of(Predicate::IsA); // col_idx: borrowed iterator +``` + +The predicate is **not stored on the lane**. Recovering it from the histogram is +what makes the two halves one structure rather than two parallel encodings that +can disagree. + +## Measurements + +All from `probe_edge_capacity` and `verify_edges` on the full OBO core +(68,797 rows), run before the layout constant was chosen. + +| quantity | value | +|---|---| +| subjects with ≥1 edge | 68,789 | +| total triples | 186,029 | +| max out-degree (one row) | 62 | +| **max lane demand (one row)** | **16** (against a 23-lane budget) | +| rows exceeding 16 lanes | 0 | +| A1 — CURIE numeric `0` occurrences | 0 | +| A2 — numerics over 24 bits | 0 | +| links packed / dropped / rows overflowed | 186,029 / **0** / **0** | + +Lane demand, not out-degree, is the binding constraint — the `(predicate, +namespace)` grouping fragments lanes, so a 62-target row needs 16 lanes rather +than `ceil(62/4) = 16`… which here coincides, but does not in general, and the +probe measures the real thing rather than the convenient proxy. + +### No silent truncation + +`pack_edges` reports overflow in `PackStats` and writes nothing past the +budget; `bake` accumulates `links_dropped` / `rows_overflowed` so a driver can +**refuse to ship** an artifact that is missing edges. A packer that quietly +truncated would produce a graph that looks complete and is not. + +## Verification + +Three independent paths agree, which is the point — a single path agreeing with +itself would prove nothing. + +1. **Lane decode vs the triple table.** `verify_edges` reads links back out of + the row bytes and compares them as a multiset against the SPO triples the + bake built from the parsed source. Different code, same input. + + | ontology | rows | links read | `is_a` degree total | + |---|---:|---:|---:| + | RO | 4 | 3 | 2 | + | UBERON | 14,975 | 71,844 | 27,170 | + | HP | 19,836 | 50,284 | 31,284 | + | PATO | 1,887 | 3,389 | 2,909 | + | MONDO | 32,095 | 60,509 | 46,448 | + + All MATCH. The `is_a` totals also equal the degree bytes already present in + the previously-baked slabs, so the packing agrees with the artifact it + extends — that was the falsifier fixed *before* any code was written. + +2. **Closure vs saturation.** Reading `is_a` out of the lanes and closing it + (see `ogar-elk`) yields **738,651** transitive ancestors and **0** + equivalence cycles. `reason::saturate` computes 738,651 subsumption pairs + and 0 cycles from the out-of-line triple table. Two reasoners, two + representations, identical to the unit. + +3. **By hand.** RO is four rows and was checked against the source directly: + + ```text + RO:0002532 --Other--> RO:0002533 (relationship: RO:0002524 RO:0002533) + RO:0002533 --is_a---> RO:0002532 + RO:0002534 --is_a---> RO:0002532 + RO:0002577 --(none) (only parent BFO:0000040 — outside the core) + ``` + + `RO:0002577` carrying *zero* links is the interesting case: its degree byte + is 0, not a dangling 1, so the histogram and the lanes agree about an edge + that was correctly never emitted. + +## Relation to `ogar-ro` + +`ogar_obo::Predicate` and the `ogar-ro` crate are **different things** and +should not be conflated: + +- `ogar_obo::Predicate` (discriminants 1–7) is the byte palette **carried on + baked edges** — it indexes the degree histogram and orders the lanes. It is + the THINK arm: what the baked graph asserts. +- `ogar-ro` is a callable **vocabulary** (`RELATION_BODY_CONCEPT_ID = 0x0306`) + that mints RO/BFO predicates as `FnIndex` bytes so a template body can + *assert* an edge — `Call = (predicate : subject, object)`, a statement, not + an expression. It is the DO arm, and its operands resolve against a + basin-local `RELATION_TARGET_CODEBOOK`, not against CURIE numerics. + +They meet at the ontology, not in the bytes. + +## Open + +- **The rails are unused until a basin exists.** The migration path is named + above and costs no re-bake, but it has no scheduled trigger. +- **`LaneShape` / `CascadeShape` duplication (W-RO-5)** is an unresolved + cross-repo question recorded in `ogar-ro`'s crate docs, deliberately not + decided by either side. +- **Degree saturation** at 255 is untested against real data because the + measured maximum is 62. If a future source pushes a single predicate past + 255, the histogram silently clamps while the lanes would still hold the + targets — the two halves would disagree. A guard belongs here before any + source with that shape is baked. diff --git a/crates/ogar-obo/examples/bake_obo.rs b/crates/ogar-obo/examples/bake_obo.rs index 1411c7f8..d6d36716 100644 --- a/crates/ogar-obo/examples/bake_obo.rs +++ b/crates/ogar-obo/examples/bake_obo.rs @@ -9,8 +9,8 @@ //! scratch dir, never committed. use ogar_obo::{ - Namespace, bake, merge_logical_defs, parse_hp_logical_defs, parse_obo, reason, - rows_from_le_bytes, as_le_bytes, + Namespace, as_le_bytes, bake, merge_logical_defs, parse_hp_logical_defs, parse_obo, reason, + rows_from_le_bytes, }; use std::collections::HashMap; @@ -22,12 +22,13 @@ fn main() { let mut nodes = HashMap::new(); for f in ["mondo", "hp", "uberon", "pato", "ro"] { let path = format!("{dir}/{f}.obo"); - let text = std::fs::read_to_string(&path) - .unwrap_or_else(|e| panic!("read {path}: {e}")); + let text = std::fs::read_to_string(&path).unwrap_or_else(|e| panic!("read {path}: {e}")); let part = parse_obo(&text); let (terms, edges) = ( part.len(), - part.values().map(|n| n.is_a.len() + n.rel.len() + n.xref.len()).sum::(), + part.values() + .map(|n| n.is_a.len() + n.rel.len() + n.xref.len()) + .sum::(), ); println!(" parsed {f:8} terms={terms:6} edges+xref={edges}"); for (id, n) in part { @@ -42,7 +43,10 @@ fn main() { // HPO logical-definition grounding (anatomy:quality) from hp-base.owl. let owl = std::fs::read_to_string(format!("{dir}/hp-base.owl")).unwrap_or_default(); let defs = parse_hp_logical_defs(&owl); - println!(" hp-base logical defs: {} HP->Uberon/PATO grounding edges", defs.len()); + println!( + " hp-base logical defs: {} HP->Uberon/PATO grounding edges", + defs.len() + ); merge_logical_defs(&mut nodes, &defs); let baked = bake(&nodes, 0x0000); @@ -56,36 +60,83 @@ fn main() { println!(" MONDO->HP resolve : {}", s.mondo_hp); println!(" HP->UBERON resolve : {}", s.hp_uberon); println!(" HP->PATO resolve : {}", s.hp_pato); - println!(" xrefs preserved : {} (MeSH bearings: {})", s.xrefs, s.mesh_xrefs); + println!( + " xrefs preserved : {} (MeSH bearings: {})", + s.xrefs, s.mesh_xrefs + ); + // The ship gate. These were computed but never printed, so a driver run + // could look clean while the artifact was missing edges — exactly the + // silent shape gate A1 was about. Print them, and refuse rather than warn. + println!(" links packed (col_idx) : {}", s.links_packed); + println!(" links DROPPED : {}", s.links_dropped); + println!(" rows overflowed : {}", s.rows_overflowed); + assert_eq!( + s.links_dropped, 0, + "refusing to write an artifact that is missing {} edge(s)", + s.links_dropped + ); + assert_eq!( + s.rows_overflowed, 0, + "refusing to write an artifact with {} truncated row(s)", + s.rows_overflowed + ); + assert_eq!( + s.links_packed, s.triples, + "every SPO triple must also be in-row: {} packed vs {} triples", + s.links_packed, s.triples + ); // Per-namespace row census. let mut census: HashMap = HashMap::new(); for id in &baked.ids { *census.entry(id.ns).or_default() += 1; } - for ns in [Namespace::Mondo, Namespace::Hpo, Namespace::Uberon, Namespace::Pato, Namespace::Ro] { - println!(" {:8}: {}", ns.prefix(), census.get(&(ns as u8)).copied().unwrap_or(0)); + for ns in [ + Namespace::Mondo, + Namespace::Hpo, + Namespace::Uberon, + Namespace::Pato, + Namespace::Ro, + ] { + println!( + " {:8}: {}", + ns.prefix(), + census.get(&(ns as u8)).copied().unwrap_or(0) + ); } println!("\n=== EL SATURATION (ELK subset, excavated to Rust) ==="); let el = reason::saturate(&baked.triples); println!(" is_a subsumption pairs : {}", el.subsumption_pairs); println!(" part_of transitive pairs : {}", el.part_of_pairs); - println!(" existential inferred (R∃): {} (grounding inherited up the spine)", el.existential_inferred); - println!(" unsatisfiable : {} (no disjointness axioms in base obo)", el.unsatisfiable); + println!( + " existential inferred (R∃): {} (grounding inherited up the spine)", + el.existential_inferred + ); + println!( + " unsatisfiable : {} (no disjointness axioms in base obo)", + el.unsatisfiable + ); // Write the artifact + verify it round-trips through the loader contract. let bytes = as_le_bytes(&baked.rows); std::fs::write(&out, bytes).unwrap_or_else(|e| panic!("write {out}: {e}")); println!("\n=== ARTIFACT ==="); - println!(" {out} ({} bytes = {} rows × 512)", bytes.len(), baked.rows.len()); + println!( + " {out} ({} bytes = {} rows × 512)", + bytes.len(), + baked.rows.len() + ); let readback = std::fs::read(&out).expect("reread"); match rows_from_le_bytes(&readback) { Some(rows) => { // NB: a fresh Vec from fs::read may not be 64-aligned; the loader // returns None then and a real consumer uses FixedSizeBinary(512) // (arrow-aligned). We verify the in-memory aligned view instead. - println!(" reread rows_from_le_bytes: {} rows (aligned buffer)", rows.len()); + println!( + " reread rows_from_le_bytes: {} rows (aligned buffer)", + rows.len() + ); } None => { let inmem = rows_from_le_bytes(bytes).expect("in-memory aligned view"); @@ -97,5 +148,7 @@ fn main() { ); } } - println!(" loader contract: VERIFIED (as_le_bytes ↔ rows_from_le_bytes, 512×N, 64-align gate)"); + println!( + " loader contract: VERIFIED (as_le_bytes ↔ rows_from_le_bytes, 512×N, 64-align gate)" + ); } diff --git a/crates/ogar-obo/examples/bake_spine.rs b/crates/ogar-obo/examples/bake_spine.rs new file mode 100644 index 00000000..729f3e01 --- /dev/null +++ b/crates/ogar-obo/examples/bake_spine.rs @@ -0,0 +1,160 @@ +//! Bake the meta-study spine — the leg gate A1 was blocking. +//! +//! ```text +//! cargo run -p ogar-obo --features rdf --example bake_spine -- [out.soa] +//! ``` +//! +//! Reads one RDF document per namespace (`.`), extracts through +//! `ogar_obo::rdf` (assertions retained, identity from the CURIE numeric), and +//! bakes through `bake_with` against +//! [`ogar_obo::registry::META_STUDY_SPINE`] — no code path per ontology, only +//! table rows. +//! +//! Three of these namespaces carry a real term at numeric `0`. Before the A1 +//! `+1` bias those links encoded to the lane pad and truncated their whole +//! lane silently, so this bake could not be trusted. The run asserts the ship +//! gate (`links_dropped == 0`, `rows_overflowed == 0`, +//! `links_packed == triples`) and refuses to write otherwise. + +use std::collections::HashMap; +use std::path::Path; + +use ogar_obo::rdf::{self, OBO_RESTRICTION_PREDICATES}; +use ogar_obo::registry::META_STUDY_SPINE; +use ogar_obo::{OboNode, Predicate, TermId, as_le_bytes, bake_with, rows_from_le_bytes}; + +fn main() { + let mut a = std::env::args().skip(1); + let dir = a.next().unwrap_or_else(|| "/tmp/onto".to_string()); + let out = a.next().unwrap_or_else(|| format!("{dir}/spine.soa")); + let reg = &META_STUDY_SPINE; + + let mut nodes: HashMap = HashMap::new(); + let mut missing: Vec<&str> = Vec::new(); + for spec in reg.specs() { + let Some((path, bytes)) = read_named(Path::new(&dir), spec.prefix) else { + missing.push(spec.prefix); + continue; + }; + let fmt = rdf::detect_format( + path.extension().and_then(|e| e.to_str()), + &bytes[..bytes.len().min(256)], + ); + let part = rdf::parse_rdf(&bytes, fmt, reg, OBO_RESTRICTION_PREDICATES) + .unwrap_or_else(|e| panic!("{}: {e}", path.display())); + println!(" parsed {:6} classes={}", spec.prefix, part.len()); + for (id, n) in part { + let e = nodes.entry(id).or_default(); + e.is_a.extend(n.is_a); + e.rel.extend(n.rel); + e.xref.extend(n.xref); + e.obsolete |= n.obsolete; + } + } + assert!( + missing.is_empty(), + "refusing to bake a partial spine — not read: {missing:?}" + ); + + // The A1 evidence, measured on the real input rather than assumed. + let zeros: Vec<&str> = reg + .specs() + .iter() + .enumerate() + .filter(|(i, _)| { + nodes.contains_key(&TermId { + ns: *i as u8, + num: 0, + }) + }) + .map(|(_, s)| s.prefix) + .collect(); + println!("\n namespaces with a real term at numeric 0: {zeros:?}"); + let zero_in_edges: usize = nodes + .values() + .map(|n| { + n.is_a.iter().filter(|t| t.num == 0).count() + + n.rel.iter().filter(|(_, t)| t.num == 0).count() + }) + .sum(); + println!( + " edges POINTING AT a numeric-0 term : {zero_in_edges} \ + (each one truncated its lane before A1)" + ); + + let baked = bake_with(&nodes, 0x0000, reg); + let s = &baked.stats; + println!("\n=== BAKE ==="); + println!(" rows : {}", s.nodes); + println!(" obsolete : {}", s.obsolete); + println!(" triples : {}", s.triples); + println!(" links packed : {}", s.links_packed); + println!(" links DROPPED : {}", s.links_dropped); + println!(" rows overflowed : {}", s.rows_overflowed); + println!(" is_a cycles : {}", s.is_a_cycles); + println!(" dangling : {}", s.dangling); + assert_eq!(s.links_dropped, 0, "artifact is missing edges"); + assert_eq!(s.rows_overflowed, 0, "artifact has truncated rows"); + assert_eq!(s.links_packed, s.triples, "not every triple is in-row"); + + // Per-namespace census in table (= row) order. + let mut census: HashMap = HashMap::new(); + for id in &baked.ids { + *census.entry(id.ns).or_default() += 1; + } + for (i, spec) in reg.specs().iter().enumerate() { + println!( + " {:6}: {:5} rows classid 0x{:08X}", + spec.prefix, + census.get(&(i as u8)).copied().unwrap_or(0), + reg.render_classid(i as u8, 0x0000).unwrap() + ); + } + + // Read every numeric-0 link back out of the baked lanes — the direct + // proof that the bake is now traversable through those terms. + let mut recovered = 0usize; + for row in &baked.rows { + for (_, l) in ogar_obo::edges::EdgeLanes::new(&row.0).links() { + if l.num == 0 { + recovered += 1; + } + } + } + println!("\n numeric-0 links READ BACK from lanes : {recovered} of {zero_in_edges}"); + assert_eq!( + recovered, zero_in_edges, + "every numeric-0 edge must survive the round trip — this is gate A1" + ); + + let bytes = as_le_bytes(&baked.rows); + std::fs::write(&out, bytes).unwrap_or_else(|e| panic!("write {out}: {e}")); + let inmem = rows_from_le_bytes(bytes).expect("in-memory aligned view"); + assert_eq!(inmem.len(), baked.rows.len()); + println!( + "\n=== ARTIFACT ===\n {out} ({} bytes = {} rows x 512)\n loader contract: VERIFIED", + bytes.len(), + baked.rows.len() + ); + + let other = baked + .triples + .iter() + .filter(|t| t.p == Predicate::Other) + .count(); + println!( + " typed edges on Predicate::Other: {other} of {} (no CURIE row matched)", + s.triples + ); +} + +fn read_named(dir: &Path, prefix: &str) -> Option<(std::path::PathBuf, Vec)> { + for e in std::fs::read_dir(dir).ok()?.flatten() { + let p = e.path(); + if p.file_stem().and_then(|s| s.to_str()) == Some(prefix) { + let b = std::fs::read(&p).ok()?; + return Some((p, b)); + } + } + None +} diff --git a/crates/ogar-obo/examples/probe_edge_capacity.rs b/crates/ogar-obo/examples/probe_edge_capacity.rs new file mode 100644 index 00000000..96c1bc18 --- /dev/null +++ b/crates/ogar-obo/examples/probe_edge_capacity.rs @@ -0,0 +1,141 @@ +//! Probe: does the `col_idx` half of the CSR fit in the free value lanes? +//! +//! The bake persists `row_ptr` (the per-predicate degree histogram in the edge +//! block) but not `col_idx` (the targets). Packing the targets as G2 `4×u24` +//! literal links needs a fixed layout constant — how many lanes to reserve. +//! This probe measures the demand on the real sources before that constant is +//! chosen, and checks the two assumptions the packing rests on: +//! +//! A1 CURIE numeric `0` never occurs (so `0` is a safe in-lane pad). +//! A2 every numeric fits 24 bits (`TermId::parse` already rejects >24-bit, +//! so this re-checks the *baked* ids, not the parser). +//! +//! Usage: `cargo run -p ogar-obo --example probe_edge_capacity -- [src_dir]` + +use ogar_obo::{Namespace, Predicate, bake, merge_logical_defs, parse_hp_logical_defs, parse_obo}; +use std::collections::HashMap; + +fn main() { + let dir = std::env::args() + .nth(1) + .unwrap_or_else(|| "/tmp/obo".to_string()); + + let mut nodes = HashMap::new(); + for f in ["mondo", "hp", "uberon", "pato", "ro"] { + let path = format!("{dir}/{f}.obo"); + let text = std::fs::read_to_string(&path).unwrap_or_else(|e| panic!("read {path}: {e}")); + for (id, n) in parse_obo(&text) { + let e = nodes.entry(id).or_insert_with(ogar_obo::OboNode::default); + e.is_a.extend(n.is_a); + e.rel.extend(n.rel); + e.xref.extend(n.xref); + e.obsolete |= n.obsolete; + } + } + let owl = std::fs::read_to_string(format!("{dir}/hp-base.owl")).unwrap_or_default(); + merge_logical_defs(&mut nodes, &parse_hp_logical_defs(&owl)); + + let baked = bake(&nodes, 0x0000); + + // ── group the triples by subject, in the canonical emit order ──────────── + // ascending predicate, then ascending target namespace: a lane carries ONE + // target ontology (its classid names it), so a namespace switch inside a + // predicate starts a new lane. That fragmentation is what makes the lane + // demand exceed `ceil(degree / 4)` — which is exactly what we are measuring. + let mut by_subject: HashMap> = HashMap::new(); + for t in &baked.triples { + by_subject + .entry(t.s) + .or_default() + .push((t.p as u8, t.o.ns, t.o.num)); + } + + let mut max_deg = 0usize; + let mut max_lanes = 0usize; + let mut worst_deg: Option = None; + let mut worst_lanes: Option = None; + let mut lane_hist: HashMap = HashMap::new(); + let mut total_lanes = 0usize; + let mut zero_numeric = 0usize; + let mut over_24bit = 0usize; + + for (s, edges) in &mut by_subject { + edges.sort_unstable(); + // lane demand: a new lane per 4 targets, restarting on (pred, ns) change + let mut lanes = 0usize; + let mut slot = 4usize; // force a fresh lane on the first target + let mut group: Option<(u8, u8)> = None; + for &(p, ns, num) in edges.iter() { + if num == 0 { + zero_numeric += 1; + } + if num > 0x00FF_FFFF { + over_24bit += 1; + } + if group != Some((p, ns)) || slot == 4 { + if group != Some((p, ns)) { + group = Some((p, ns)); + } + lanes += 1; + slot = 0; + } + slot += 1; + } + if edges.len() > max_deg { + max_deg = edges.len(); + worst_deg = Some(*s); + } + if lanes > max_lanes { + max_lanes = lanes; + worst_lanes = Some(*s); + } + *lane_hist.entry(lanes).or_default() += 1; + total_lanes += lanes; + } + + println!("=== EDGE CAPACITY PROBE ==="); + println!(" subjects with >=1 edge : {}", by_subject.len()); + println!(" total triples : {}", baked.triples.len()); + println!(" total lanes demanded : {total_lanes}"); + println!(" max out-degree : {max_deg} ({worst_deg:?})"); + println!(" max lanes for one row : {max_lanes} ({worst_lanes:?})"); + + println!("\n A1 CURIE numeric == 0 : {zero_numeric} occurrences (must be 0)"); + println!(" A2 numeric > 24 bit : {over_24bit} occurrences (must be 0)"); + + // How many rows would overflow a given lane budget? + println!("\n rows exceeding a lane budget:"); + for budget in [8usize, 12, 16, 20, 23, 29] { + let over: usize = lane_hist + .iter() + .filter(|(l, _)| **l > budget) + .map(|(_, c)| *c) + .sum(); + println!( + " budget {budget:2} lanes ({:3} slots): {over} rows over", + budget * 4 + ); + } + + // Per-namespace is_a degree totals — the falsifier against the baked slabs. + let mut is_a_by_ns: HashMap = HashMap::new(); + for t in &baked.triples { + if t.p == Predicate::IsA { + *is_a_by_ns.entry(t.s.ns).or_default() += 1; + } + } + println!("\n is_a degree totals by subject namespace (falsifier vs baked slabs):"); + for ns in [ + Namespace::Mondo, + Namespace::Hpo, + Namespace::Uberon, + Namespace::Pato, + Namespace::Ro, + ] { + println!( + " {:8}: {}", + ns.prefix(), + is_a_by_ns.get(&(ns as u8)).copied().unwrap_or(0) + ); + } +} diff --git a/crates/ogar-obo/examples/probe_spine.rs b/crates/ogar-obo/examples/probe_spine.rs new file mode 100644 index 00000000..dd2d2aae --- /dev/null +++ b/crates/ogar-obo/examples/probe_spine.rs @@ -0,0 +1,197 @@ +//! Measure what the RDF front-end actually harvests from the meta-study spine. +//! +//! This is the falsifier for [`ogar_obo::rdf`]: it runs the *shipped* code path +//! over real ontology documents and reports numbers, rather than asserting the +//! path works. It also measures the numeric-`0` pad collision that gates +//! [`ogar_obo::registry::META_STUDY_SPINE`] — so the gate is a measurement, not +//! a claim. +//! +//! ```text +//! cargo run -p ogar-obo --features rdf --example probe_spine -- +//! ``` +//! +//! `` holds one document per namespace, named `.` (the +//! extension picks the serialization via `rdf::detect_format`). A missing file +//! is reported, never silently skipped — a spine that quietly measured 4 of 8 +//! namespaces would read as a complete result. + +use std::collections::BTreeMap; +use std::path::Path; + +use ogar_obo::rdf::{self, OBO_RESTRICTION_PREDICATES, RdfFormat}; +use ogar_obo::registry::{META_STUDY_SPINE, NsRegistry}; +use ogar_obo::{OboNode, Predicate, TermId}; + +fn main() { + let dir = std::env::args().nth(1).unwrap_or_else(|| { + eprintln!("usage: probe_spine "); + std::process::exit(2); + }); + let dir = Path::new(&dir); + let reg = &META_STUDY_SPINE; + + // Harvest every namespace into ONE map: the spine is a single graph whose + // members import each other, and the cross-ontology edge count below is + // only meaningful across the union. + let mut union: std::collections::HashMap = std::collections::HashMap::new(); + let mut missing: Vec<&str> = Vec::new(); + let mut per_file: Vec<(String, usize)> = Vec::new(); + + for spec in reg.specs() { + let Some((path, bytes)) = read_named(dir, spec.prefix) else { + missing.push(spec.prefix); + continue; + }; + let fmt = rdf::detect_format( + path.extension().and_then(|e| e.to_str()), + &bytes[..bytes.len().min(256)], + ); + match rdf::parse_rdf(&bytes, fmt, reg, OBO_RESTRICTION_PREDICATES) { + Ok(nodes) => { + per_file.push((format!("{} ({:?})", path.display(), fmt), nodes.len())); + for (id, node) in nodes { + let slot = union.entry(id).or_default(); + slot.is_a.extend(node.is_a); + slot.rel.extend(node.rel); + slot.xref.extend(node.xref); + slot.obsolete |= node.obsolete; + } + } + Err(e) => { + eprintln!("!! {}: {e}", path.display()); + missing.push(spec.prefix); + } + } + } + + for (f, n) in &per_file { + println!("read {f}: {n} classes"); + } + if !missing.is_empty() { + println!("\n!! NOT MEASURED: {missing:?} — the totals below exclude them"); + } + + // Per-namespace census, keyed by ordinal so the report follows table order. + let mut own: BTreeMap = BTreeMap::new(); + for (id, node) in &union { + let c = own.entry(id.ns).or_default(); + c.classes += 1; + c.obsolete += usize::from(node.obsolete); + c.is_a += node.is_a.len(); + c.rel += node.rel.len(); + c.max_num = c.max_num.max(id.num); + if id.num == 0 { + c.zero_id += 1; + } + c.max_is_a = c.max_is_a.max(node.is_a.len()); + for parent in &node.is_a { + if parent.ns != id.ns { + *c.cross.entry(parent.ns).or_default() += 1; + } + } + for (p, o) in &node.rel { + if *p == Predicate::Other { + c.rel_other += 1; + } + if o.ns != id.ns { + *c.cross.entry(o.ns).or_default() += 1; + } + } + } + + println!( + "\n{:<7} {:>4} {:>8} {:>7} {:>7} {:>6} {:>9} {:>7} {:>6}", + "ns", "ord", "classes", "is_a", "maxIsA", "rel", "max num", "id==0", "obs" + ); + let (mut t_c, mut t_i, mut t_r, mut t_z) = (0, 0, 0, 0); + for (ord, c) in &own { + println!( + "{:<7} {:>4} {:>8} {:>7} {:>7} {:>6} {:>9} {:>7} {:>6}", + name(reg, *ord), + ord, + c.classes, + c.is_a, + c.max_is_a, + c.rel, + c.max_num, + c.zero_id, + c.obsolete + ); + t_c += c.classes; + t_i += c.is_a; + t_r += c.rel; + t_z += c.zero_id; + } + println!( + "{:<7} {:>4} {:>8} {:>7} {:>7} {:>6}", + "TOTAL", "", t_c, t_i, "", t_r + ); + + println!("\ncross-namespace edges (subject ns -> object ns : count)"); + for (ord, c) in &own { + for (other, n) in &c.cross { + println!(" {} -> {} : {n}", name(reg, *ord), name(reg, *other)); + } + } + + let rel_other: usize = own.values().map(|c| c.rel_other).sum(); + println!( + "\ntyped edges landing on Predicate::Other: {rel_other} of {t_r} \ + (no CURIE row in OBO_RESTRICTION_PREDICATES — reported, never guessed)" + ); + + // The gate, measured. + println!("\n── numeric-0 pad collision (gates META_STUDY_SPINE) ──"); + if t_z == 0 { + println!(" none — no term in this spine carries numeric 0"); + } else { + for (ord, c) in &own { + if c.zero_id > 0 { + println!( + " {} has a real term at numeric 0 — an edge into it is \ + indistinguishable from the lane pad", + name(reg, *ord) + ); + } + } + println!( + " => {t_z} affected namespace(s). Baking them is unsound until the \ + edges.rs +1 bias lands (operator call: it moves every shipped digest)." + ); + } +} + +#[derive(Default)] +struct Census { + classes: usize, + obsolete: usize, + is_a: usize, + max_is_a: usize, + rel: usize, + rel_other: usize, + max_num: u32, + zero_id: usize, + cross: BTreeMap, +} + +fn name(reg: &NsRegistry, ord: u8) -> &'static str { + reg.spec(ord).map_or("?", |s| s.prefix) +} + +/// Find `/.` — the serialization is decided by +/// `detect_format`, not by the caller guessing the extension. +fn read_named(dir: &Path, prefix: &str) -> Option<(std::path::PathBuf, Vec)> { + let entries = std::fs::read_dir(dir).ok()?; + for e in entries.flatten() { + let p = e.path(); + if p.file_stem().and_then(|s| s.to_str()) == Some(prefix) { + let bytes = std::fs::read(&p).ok()?; + return Some((p, bytes)); + } + } + None +} + +// Keep the unused-import lint honest when the format enum is only used via +// `detect_format`'s return value. +const _: fn() -> RdfFormat = || RdfFormat::Turtle; diff --git a/crates/ogar-obo/examples/split_namespaces.rs b/crates/ogar-obo/examples/split_namespaces.rs new file mode 100644 index 00000000..9e36e74c --- /dev/null +++ b/crates/ogar-obo/examples/split_namespaces.rs @@ -0,0 +1,45 @@ +//! Split a baked union into per-namespace slabs, exactly as the S3 layout holds them. +use ogar_obo::{ + Row512, as_le_bytes, bake_with, merge_logical_defs, parse_hp_logical_defs, parse_obo, + registry::OBO_CORE, +}; +use std::collections::HashMap; +fn main() { + let dir = std::env::args().nth(1).unwrap_or("/tmp/obo".into()); + let out = std::env::args().nth(2).unwrap_or("/tmp/slabs".into()); + std::fs::create_dir_all(&out).unwrap(); + let mut nodes = HashMap::new(); + for f in ["mondo", "hp", "uberon", "pato", "ro"] { + for (id, n) in parse_obo(&std::fs::read_to_string(format!("{dir}/{f}.obo")).unwrap()) { + let e: &mut ogar_obo::OboNode = nodes.entry(id).or_default(); + e.is_a.extend(n.is_a); + e.rel.extend(n.rel); + e.xref.extend(n.xref); + e.obsolete |= n.obsolete; + } + } + let owl = std::fs::read_to_string(format!("{dir}/hp-base.owl")).unwrap_or_default(); + merge_logical_defs(&mut nodes, &parse_hp_logical_defs(&owl)); + let baked = bake_with(&nodes, 0x0000, &OBO_CORE); + assert_eq!(baked.stats.links_dropped, 0); + assert_eq!(baked.stats.rows_overflowed, 0); + let mut per: HashMap<&str, Vec> = HashMap::new(); + for (i, id) in baked.ids.iter().enumerate() { + let name = match id.ns { + 0 => "mondo", + 1 => "hpo", + 2 => "uberon", + 3 => "pato", + _ => "ro", + }; + per.entry(name).or_default().push(baked.rows[i]); + } + let mut names: Vec<_> = per.keys().copied().collect(); + names.sort_unstable(); + for n in names { + let rows = &per[n]; + let p = format!("{out}/obo.{n}.soa"); + std::fs::write(&p, as_le_bytes(rows)).unwrap(); + println!("{p} {} rows {} bytes", rows.len(), rows.len() * 512); + } +} diff --git a/crates/ogar-obo/examples/verify_edges.rs b/crates/ogar-obo/examples/verify_edges.rs new file mode 100644 index 00000000..0d122763 --- /dev/null +++ b/crates/ogar-obo/examples/verify_edges.rs @@ -0,0 +1,140 @@ +//! Verify the packed edge lanes against the sources, ontology by ontology. +//! +//! The falsifier is independence: the links are read back **out of the row +//! bytes** (lane decode via [`ogar_obo::edges::EdgeLanes`]) and compared as a +//! multiset against the SPO triple table the bake built from the parsed `.obo`. +//! Those are two different code paths over the same input, so agreement is +//! evidence and disagreement names the row. +//! +//! Order is RO → UBERON → HPO → (PATO, MONDO), smallest first: RO is two edges +//! and can be checked by eye before anything larger is trusted. +//! +//! Usage: `cargo run -p ogar-obo --example verify_edges -- [src_dir]` + +use ogar_obo::{ + Namespace, Predicate, TermId, bake, edges::EdgeLanes, merge_logical_defs, + parse_hp_logical_defs, parse_obo, +}; +use std::collections::HashMap; + +fn main() { + let dir = std::env::args() + .nth(1) + .unwrap_or_else(|| "/tmp/obo".to_string()); + + let mut nodes = HashMap::new(); + for f in ["mondo", "hp", "uberon", "pato", "ro"] { + let path = format!("{dir}/{f}.obo"); + let text = std::fs::read_to_string(&path).unwrap_or_else(|e| panic!("read {path}: {e}")); + for (id, n) in parse_obo(&text) { + let e = nodes.entry(id).or_insert_with(ogar_obo::OboNode::default); + e.is_a.extend(n.is_a); + e.rel.extend(n.rel); + e.xref.extend(n.xref); + e.obsolete |= n.obsolete; + } + } + let owl = std::fs::read_to_string(format!("{dir}/hp-base.owl")).unwrap_or_default(); + merge_logical_defs(&mut nodes, &parse_hp_logical_defs(&owl)); + + let baked = bake(&nodes, 0x0000); + let s = &baked.stats; + println!("=== BAKE ==="); + println!(" rows : {}", s.nodes); + println!(" triples : {}", s.triples); + println!(" links packed : {}", s.links_packed); + println!(" links dropped : {} (must be 0)", s.links_dropped); + println!(" rows overflow : {} (must be 0)", s.rows_overflowed); + assert_eq!( + s.links_dropped, 0, + "edges were dropped — artifact incomplete" + ); + assert_eq!(s.rows_overflowed, 0); + assert_eq!( + s.links_packed, s.triples, + "every out-of-line triple must also be in-row" + ); + + // expected: (subject, predicate, target-classid, target-num) from the triples + let mut expected: HashMap> = HashMap::new(); + for t in &baked.triples { + expected.entry(t.s).or_default().push(( + t.p as u8, + t.o.namespace().render_classid(0x0000), + t.o.num, + )); + } + + let order = [ + Namespace::Ro, + Namespace::Uberon, + Namespace::Hpo, + Namespace::Pato, + Namespace::Mondo, + ]; + let mut all_ok = true; + for ns in order { + let mut rows = 0usize; + let mut links = 0usize; + let mut is_a = 0usize; + let mut bad: Vec = Vec::new(); + for (i, id) in baked.ids.iter().enumerate() { + if id.ns != ns as u8 { + continue; + } + rows += 1; + let lanes = EdgeLanes::new(&baked.rows[i].0); + let mut got: Vec<(u8, u32, u32)> = lanes + .links() + .map(|(p, l)| (p as u8, l.classid, l.num)) + .collect(); + links += got.len(); + is_a += lanes.degree(Predicate::IsA) as usize; + let mut want = expected.get(id).cloned().unwrap_or_default(); + got.sort_unstable(); + want.sort_unstable(); + if got != want { + bad.push(*id); + } + } + let ok = bad.is_empty(); + all_ok &= ok; + println!( + "\n=== {:8} rows={rows} links_read={links} is_a_degree_total={is_a} — {}", + ns.prefix(), + if ok { "MATCH" } else { "MISMATCH" } + ); + if !ok { + for id in bad.iter().take(5) { + let i = baked.ids.iter().position(|x| x == id).unwrap(); + println!( + " {}:{:07} got={:?} want={:?}", + ns.prefix(), + id.num, + EdgeLanes::new(&baked.rows[i].0) + .links() + .map(|(p, l)| (p, l.num)) + .collect::>(), + expected.get(id) + ); + } + println!(" ... {} mismatching rows total", bad.len()); + } + // RO is small enough to print in full — the by-eye check. + if ns == Namespace::Ro { + for (i, id) in baked.ids.iter().enumerate() { + if id.ns != ns as u8 { + continue; + } + let l: Vec<_> = EdgeLanes::new(&baked.rows[i].0) + .links() + .map(|(p, l)| format!("{p:?}->0x{:08x}:{}", l.classid, l.num)) + .collect(); + println!(" RO:{:07} -> [{}]", id.num, l.join(", ")); + } + } + } + + assert!(all_ok, "lane decode disagrees with the triple table"); + println!("\nALL LANES VERIFIED against the SPO triple table."); +} diff --git a/crates/ogar-obo/src/crosswalk.rs b/crates/ogar-obo/src/crosswalk.rs index 57e7c612..b905daad 100644 --- a/crates/ogar-obo/src/crosswalk.rs +++ b/crates/ogar-obo/src/crosswalk.rs @@ -103,16 +103,17 @@ impl Crosswalk { // roll up: drop a trailing digit past the 3-char WHO stem, else the // sub-category dot, else give up. if let Some(pos) = c.rfind(|ch: char| ch.is_ascii_digit()) - && (c.len() > 3 || (c.contains('.') && pos > c.find('.').unwrap())) { - c.truncate(pos); - if c.ends_with('.') { - c.pop(); - } - if c.is_empty() { - return None; - } - continue; + && (c.len() > 3 || (c.contains('.') && pos > c.find('.').unwrap())) + { + c.truncate(pos); + if c.ends_with('.') { + c.pop(); } + if c.is_empty() { + return None; + } + continue; + } return None; } } @@ -153,16 +154,31 @@ mod tests { let mut nodes: HashMap = HashMap::new(); // MONDO:5148 (T2DM) xrefs ICD-10 E11 + MeSH D003924 nodes.insert( - TermId { ns: Namespace::Mondo as u8, num: 5148 }, + TermId { + ns: Namespace::Mondo as u8, + num: 5148, + }, node_with_xrefs(vec![ - Xref { source: XrefSource::Icd, id: "E11".into() }, - Xref { source: XrefSource::Mesh, id: "D003924".into() }, + Xref { + source: XrefSource::Icd, + id: "E11".into(), + }, + Xref { + source: XrefSource::Mesh, + id: "D003924".into(), + }, ]), ); // UBERON:945 (stomach) xrefs FMA:7148 nodes.insert( - TermId { ns: Namespace::Uberon as u8, num: 945 }, - node_with_xrefs(vec![Xref { source: XrefSource::Other("FMA".into()), id: "7148".into() }]), + TermId { + ns: Namespace::Uberon as u8, + num: 945, + }, + node_with_xrefs(vec![Xref { + source: XrefSource::Other("FMA".into()), + id: "7148".into(), + }]), ); let baked = bake(&nodes, 0x0000); let cw = Crosswalk::from_bake(&baked); @@ -187,8 +203,14 @@ mod tests { // roll up to it (measured: MONDO carries no ICD-10-GM). let mut nodes: HashMap = HashMap::new(); nodes.insert( - TermId { ns: Namespace::Mondo as u8, num: 5148 }, - node_with_xrefs(vec![Xref { source: XrefSource::Icd, id: "E11".into() }]), + TermId { + ns: Namespace::Mondo as u8, + num: 5148, + }, + node_with_xrefs(vec![Xref { + source: XrefSource::Icd, + id: "E11".into(), + }]), ); let cw = Crosswalk::from_bake(&bake(&nodes, 0x0000)); // direct GM code fails, rollup succeeds — no separate GM table diff --git a/crates/ogar-obo/src/edges.rs b/crates/ogar-obo/src/edges.rs new file mode 100644 index 00000000..6a7ca30e --- /dev/null +++ b/crates/ogar-obo/src/edges.rs @@ -0,0 +1,723 @@ +//! The `col_idx` half of the CSR — edge targets packed into value lanes. +//! +//! # Why this module exists +//! +//! The bake's edge block at `[16,32)` is a per-predicate **degree histogram**: +//! it says *how many* targets a node has per predicate, but not *which*. In CSR +//! terms the bake persisted `row_ptr` and dropped `col_idx`, so the on-disk +//! artifact could describe a node's local shape but could not be traversed. +//! This module completes it: the targets ride the free value lanes, and the +//! degree histogram becomes load-bearing instead of decorative. +//! +//! # The lane form +//! +//! A lane is the one form the substrate has — `classid(4) + 12 content bytes`, +//! and the classid says how to read its own 12 bytes. An edge lane reads its +//! content as the **G2 `4×u24`** carving (`legacy_outliers::LegacyOutlier:: +//! WideTriple`), i.e. four literal links, and its `classid` is **the target +//! ontology's own row classid**. So a link is exactly the `(classid, identity)` +//! pair a *key* carries — the same binary search that resolves a key resolves a +//! link. Nothing new is invented to point at a node. +//! +//! ```text +//! lane: | classid: u32 | u24 | u24 | u24 | u24 | +//! ^ target ontology ^ four CURIE numerics +//! ``` +//! +//! # Layout +//! +//! The 480-byte value slab is 30 lanes. Lane 6 is the `EntityType` tenant +//! ([`crate::ENTITY_TYPE_SLAB_OFFSET`]); lanes 0..6 stay reserved for further +//! value tenants (RESERVE, DON'T RECLAIM). Edges occupy lanes 7..30 — +//! [`EDGE_LANE_COUNT`] lanes = [`EDGE_SLOT_COUNT`] link slots. +//! +//! Measured demand on the OBO core (`probe_edge_capacity`): 186 029 triples +//! over 68 789 subjects, max out-degree **62**, max lane demand for a single +//! row **16**. No row overflows the 23-lane budget, and the packer never +//! truncates silently — an overflow is reported ([`PackStats::dropped`]) so a +//! driver can fail rather than quietly lose edges. +//! +//! # Emit order (the reader's contract) +//! +//! Targets are emitted ascending by `(predicate, target namespace, numeric)`, +//! and **a lane never mixes target ontologies** — its classid names exactly +//! one, so a namespace change starts a new lane even mid-predicate. Unused +//! slots in a lane are all-zero bytes, and a reader treats a zero slot as +//! "this lane is spent, advance". +//! +//! # The `+1` numeric bias (gate A1, applied 2026-08-08) +//! +//! An all-zero slot is the pad, so the slot encoding must have no *other* +//! meaning for zero. The original contract achieved that by assertion — +//! *"no OBO CURIE numeric is 0"* — which held for the five baked namespaces +//! and **is false in general**. `probe_spine` measured a real term at numeric +//! `0` in three further namespaces, one of them that namespace's own root. An +//! edge into such a term was indistinguishable from a pad, so a reader's walk +//! stopped early and *silently* — the worst failure shape available, because +//! the artifact still validates and the traversal just returns less. +//! +//! The fix removes the assumption instead of documenting around it: the +//! numeric is stored **biased by one** ([`encode_numeric`] / [`decode_numeric`]). +//! Stored `0` now means pad and nothing else, by construction rather than by +//! a property of the input. +//! +//! Cost: one value off the top of the range. [`MAX_LINK_NUMERIC`] is +//! `2²⁴ − 2 = 16_777_214`, against a measured maximum of `9_999_994` across +//! every namespace probed. A numeric above that is **reported as dropped**, +//! never truncated (see [`PackStats::dropped`]). +//! +//! This changed every byte of every edge lane, so it re-baked all five core +//! slabs and moved their digests. That was the price of the gate and it was +//! paid deliberately, not absorbed. +//! +//! A reader walks lanes in order and consumes `degree(p)` targets for each +//! predicate `p` in ascending order. That is [`EdgeLanes`], which borrows the +//! row and decodes a u24 with a shift — no allocation, no gathered window. + +use crate::registry::NsRegistry; +use crate::{EDGES_OFFSET, NODE_ROW_STRIDE, Namespace, Predicate, TermId, VALUE_OFFSET}; + +/// Bytes per lane — the one form (`classid(4) + content(12)`). +pub const LANE_BYTES: usize = 16; +/// Links per lane under the G2 `4×u24` carving. +pub const LINKS_PER_LANE: usize = 4; +/// First edge lane's offset **within the 480-byte value slab** (lane 7). +pub const EDGE_LANE_SLAB_OFFSET: usize = 112; +/// Number of lanes reserved for edge targets (lanes 7..30). +pub const EDGE_LANE_COUNT: usize = 23; +/// Total link slots per row. +pub const EDGE_SLOT_COUNT: usize = EDGE_LANE_COUNT * LINKS_PER_LANE; + +/// The largest CURIE numeric an edge lane can carry. +/// +/// One below the u24 ceiling, because the stored form is biased by one so that +/// stored `0` can mean *pad* unconditionally (see the module docs, gate A1). +/// Measured maximum across every namespace probed: `9_999_994`. +pub const MAX_LINK_NUMERIC: u32 = 0x00FF_FFFF - 1; + +/// CURIE numeric → its stored form. Total on `0..=MAX_LINK_NUMERIC`. +#[inline] +#[must_use] +pub const fn encode_numeric(num: u32) -> u32 { + num + 1 +} + +/// Stored form → CURIE numeric. `stored == 0` is the pad and has no numeric, +/// so callers must test for it *before* decoding; [`EdgeLanes::links`] does. +#[inline] +#[must_use] +pub const fn decode_numeric(stored: u32) -> u32 { + stored - 1 +} + +const _: () = assert!(EDGE_LANE_SLAB_OFFSET > crate::ENTITY_TYPE_SLAB_OFFSET); +const _: () = assert!( + VALUE_OFFSET + EDGE_LANE_SLAB_OFFSET + EDGE_LANE_COUNT * LANE_BYTES == NODE_ROW_STRIDE, + "edge lanes must run to the end of the row without overlapping a value tenant" +); + +/// Read a little-endian `u24` (as a `u32`, high byte zero). +#[inline] +const fn u24_le(a: u8, b: u8, c: u8) -> u32 { + (a as u32) | ((b as u32) << 8) | ((c as u32) << 16) +} + +/// Write a little-endian `u24`. Debug-asserts the 24-bit fit — no silent +/// truncation, the same guard the key's u24 groups use. +#[inline] +fn put_u24_le(dst: &mut [u8], v: u32) { + debug_assert!(v <= 0x00FF_FFFF, "edge target numeric must fit 24 bits"); + dst[0] = (v & 0xFF) as u8; + dst[1] = ((v >> 8) & 0xFF) as u8; + dst[2] = ((v >> 16) & 0xFF) as u8; +} + +/// One resolved link: the `(classid, identity)` pair, exactly as a key carries +/// it. `classid` names the target ontology; `num` is the CURIE numeric. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub struct Link { + /// the target row's classid (`Namespace::render_classid`) + pub classid: u32, + /// the target CURIE numeric (24-bit) + pub num: u32, +} + +/// What a pack pass did — reported so a driver can refuse to ship a truncation. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct PackStats { + /// links written into lanes + pub written: usize, + /// lanes consumed + pub lanes: usize, + /// links that did NOT fit the lane budget (must be 0 to ship) + pub dropped: usize, + /// rows that dropped at least one link + pub rows_overflowed: usize, +} + +/// Pack one node's targets into its row's edge lanes. +/// +/// `targets` must already be the node's full out-edge set; this function sorts +/// it into the canonical `(predicate, namespace, numeric)` emit order itself, +/// so callers cannot get the reader's contract wrong by accident. +/// +/// Returns the per-row stats. A link is counted in [`PackStats::dropped`] and +/// **not** written — never silently truncated — when either the lane budget is +/// exhausted or the numeric exceeds [`MAX_LINK_NUMERIC`]. Both mean the same +/// thing to a driver: the artifact is missing an edge, so it must not ship. +pub fn pack_edges( + row: &mut [u8; NODE_ROW_STRIDE], + targets: &mut [(Predicate, TermId)], + app_prefix: u16, + reg: &NsRegistry, +) -> PackStats { + let mut st = PackStats::default(); + targets.sort_unstable_by_key(|(p, t)| (*p as u8, t.ns, t.num)); + + let base = VALUE_OFFSET + EDGE_LANE_SLAB_OFFSET; + let mut lane = 0usize; + let mut slot = LINKS_PER_LANE; // force a fresh lane on the first target + let mut group: Option<(u8, u8)> = None; + + for (i, (p, t)) in targets.iter().enumerate() { + if t.num > MAX_LINK_NUMERIC { + // Unrepresentable under the +1 bias — the one numeric the biased + // u24 cannot hold. Reported, not truncated, and not allowed to + // consume a slot: a driver refuses to ship on `dropped != 0`. + st.dropped += 1; + st.rows_overflowed = 1; + continue; + } + let g = (*p as u8, t.ns); + if group != Some(g) || slot == LINKS_PER_LANE { + group = Some(g); + lane += 1; + slot = 0; + if lane > EDGE_LANE_COUNT { + // budget exhausted — count this one and every one after it, + // write nothing more. + st.dropped += targets.len() - i; + st.rows_overflowed = 1; + st.lanes = EDGE_LANE_COUNT; + return st; + } + let off = base + (lane - 1) * LANE_BYTES; + let Some(classid) = reg.render_classid(t.ns, app_prefix) else { + // A target whose namespace is not in this registry has no + // classid, so the lane could not name it. Report, never guess. + st.dropped += 1; + st.rows_overflowed = 1; + lane -= 1; + slot = LINKS_PER_LANE; + group = None; + continue; + }; + row[off..off + 4].copy_from_slice(&classid.to_le_bytes()); + } + let off = base + (lane - 1) * LANE_BYTES + 4 + slot * 3; + put_u24_le(&mut row[off..off + 3], encode_numeric(t.num)); + slot += 1; + st.written += 1; + } + st.lanes = lane; + st +} + +/// A borrowed view over one row's edge lanes — the traversal peek. +/// +/// Holds no copy of anything: `degree` reads the edge block in place and +/// [`links`] decodes a `u24` with a shift as it walks. This is the `col_idx` +/// side of a CSR row; combined with [`EdgeLanes::degree`] (`row_ptr`) it is a +/// complete adjacency row, which is what makes a semiring traversal — rather +/// than pointer chasing — the natural way to consume it. +#[derive(Debug, Clone, Copy)] +pub struct EdgeLanes<'a> { + row: &'a [u8; NODE_ROW_STRIDE], +} + +impl<'a> EdgeLanes<'a> { + /// Borrow a row's edge lanes. + #[must_use] + pub const fn new(row: &'a [u8; NODE_ROW_STRIDE]) -> Self { + EdgeLanes { row } + } + + /// This node's out-degree for one predicate — the `row_ptr` entry, read + /// straight from the edge block with no value decode at all. + #[must_use] + pub const fn degree(&self, p: Predicate) -> u8 { + self.row[EDGES_OFFSET + p as usize] + } + + /// Total out-degree across every predicate. + #[must_use] + pub fn total_degree(&self) -> usize { + (1..8).map(|i| self.row[EDGES_OFFSET + i] as usize).sum() + } + + /// Every link in emit order, paired with its predicate. + /// + /// Walks lanes in order, consuming `degree(p)` links per predicate in + /// ascending predicate order; a `0` slot means the lane is spent (no OBO + /// CURIE numeric is 0) and the walk advances to the next lane. + pub fn links(self) -> impl Iterator + 'a { + LinkIter { + row: self.row, + lane: 0, + slot: 0, + pred: 0, + remaining: 0, + } + } + + /// Only the links carried by one predicate — the traversal hot path + /// (`is_a` for the subsumption spine, `part_of` for the anatomy backbone). + pub fn links_of(self, p: Predicate) -> impl Iterator + 'a { + self.links().filter(move |(q, _)| *q == p).map(|(_, l)| l) + } +} + +struct LinkIter<'a> { + row: &'a [u8; NODE_ROW_STRIDE], + lane: usize, + slot: usize, + pred: u8, + remaining: u8, +} + +impl Iterator for LinkIter<'_> { + type Item = (Predicate, Link); + + fn next(&mut self) -> Option<(Predicate, Link)> { + let base = VALUE_OFFSET + EDGE_LANE_SLAB_OFFSET; + loop { + // advance to the next predicate that has any degree left + while self.remaining == 0 { + self.pred += 1; + if self.pred > 7 { + return None; + } + self.remaining = self.row[EDGES_OFFSET + self.pred as usize]; + } + if self.lane >= EDGE_LANE_COUNT { + return None; + } + if self.slot >= LINKS_PER_LANE { + self.lane += 1; + self.slot = 0; + continue; + } + let off = base + self.lane * LANE_BYTES; + let n = u24_le( + self.row[off + 4 + self.slot * 3], + self.row[off + 5 + self.slot * 3], + self.row[off + 6 + self.slot * 3], + ); + if n == 0 { + // Pad — the lane is spent, next lane. Unconditional under the + // +1 bias: no real target encodes to 0, whatever its numeric. + self.lane += 1; + self.slot = 0; + continue; + } + let classid = u32::from_le_bytes([ + self.row[off], + self.row[off + 1], + self.row[off + 2], + self.row[off + 3], + ]); + self.slot += 1; + self.remaining -= 1; + let p = PREDICATES[self.pred as usize]; + return Some(( + p, + Link { + classid, + num: decode_numeric(n), + }, + )); + } + } +} + +/// Predicate by discriminant; index 0 is reserved and never yielded. +const PREDICATES: [Predicate; 8] = [ + Predicate::IsA, // slot 0 reserved — never reached (degree byte 0 unused) + Predicate::IsA, + Predicate::PartOf, + Predicate::HasPhenotype, + Predicate::HasLocation, + Predicate::HasAnatomy, + Predicate::HasQuality, + Predicate::Other, +]; + +/// The classid a link into `ns` carries, for callers building expectations. +#[must_use] +pub const fn target_classid(ns: Namespace, app_prefix: u16) -> u32 { + ns.render_classid(app_prefix) +} + +/// This row's own address as a [`Link`] — the same `(classid, identity)` shape +/// a target carries, because a subject and a target are the same kind of thing. +#[must_use] +pub const fn subject(row: &[u8; NODE_ROW_STRIDE]) -> Link { + let (classid, num) = crate::unpack_key(row); + Link { classid, num } +} + +/// The **subsumption projection** over baked rows: `(subclass, superclass)` +/// pairs, `is_a` only. +/// +/// This is the adapter a post-bake EL reasoner consumes. It reads the baked +/// artifact and nothing else — no file, no CURIE, no label — and the `is_a` +/// filter is the point rather than a convenience: walking subsumption together +/// with `part_of` derives FALSE ancestors (`A part_of B` and `B ⊑ C` does not +/// give `A ⊑ C`), so a reasoner without role composition must never be handed +/// the mixed set. Filtering here means a caller cannot make that mistake by +/// forgetting to. +/// +/// Borrows throughout: the pairs are decoded from the rows as the iterator +/// walks, nothing is gathered. +pub fn subsumptions(rows: &[crate::Row512]) -> impl Iterator + '_ { + rows.iter().flat_map(|row| { + let s = subject(&row.0); + EdgeLanes::new(&row.0) + .links_of(Predicate::IsA) + .map(move |p| (s, p)) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn t(ns: Namespace, num: u32) -> TermId { + TermId { ns: ns as u8, num } + } + + fn row_with_degrees(degs: &[(Predicate, u8)]) -> [u8; NODE_ROW_STRIDE] { + let mut row = [0u8; NODE_ROW_STRIDE]; + for (p, d) in degs { + row[EDGES_OFFSET + *p as usize] = *d; + } + row + } + + #[test] + fn the_budget_covers_the_measured_worst_row() { + // probe_edge_capacity on the real OBO core: max out-degree 62, max lane + // demand 16. Both must fit, with the lane demand the binding one. + const { assert!(16 <= EDGE_LANE_COUNT) }; + const { assert!(62 <= EDGE_SLOT_COUNT) }; + } + + #[test] + fn a_single_link_round_trips_with_its_target_classid() { + let mut row = row_with_degrees(&[(Predicate::IsA, 1)]); + let mut tg = vec![(Predicate::IsA, t(Namespace::Mondo, 5015))]; + let st = pack_edges(&mut row, &mut tg, 0x0000, &crate::registry::OBO_CORE); + assert_eq!(st.written, 1); + assert_eq!(st.dropped, 0); + + let lanes = EdgeLanes::new(&row); + assert_eq!(lanes.degree(Predicate::IsA), 1); + let got: Vec<_> = lanes.links().collect(); + assert_eq!( + got, + vec![( + Predicate::IsA, + Link { + classid: 0x0301_0000, + num: 5015 + } + )] + ); + } + + #[test] + fn a_namespace_change_starts_a_new_lane_so_a_classid_never_lies() { + // one predicate, two target ontologies: the lane classid names exactly + // one ontology, so the second target cannot share the first's lane. + let mut row = row_with_degrees(&[(Predicate::Other, 2)]); + let mut tg = vec![ + (Predicate::Other, t(Namespace::Mondo, 1)), + (Predicate::Other, t(Namespace::Hpo, 2)), + ]; + let st = pack_edges(&mut row, &mut tg, 0x0000, &crate::registry::OBO_CORE); + assert_eq!(st.written, 2); + assert_eq!(st.lanes, 2, "a namespace change must start a new lane"); + + let got: Vec<_> = EdgeLanes::new(&row).links().map(|(_, l)| l).collect(); + assert_eq!( + got, + vec![ + Link { + classid: 0x0301_0000, + num: 1 + }, + Link { + classid: 0x0302_0000, + num: 2 + } + ] + ); + } + + #[test] + fn a_full_lane_spills_into_the_next_one() { + let mut row = row_with_degrees(&[(Predicate::IsA, 5)]); + let mut tg: Vec<_> = (1..=5) + .map(|n| (Predicate::IsA, t(Namespace::Uberon, n))) + .collect(); + let st = pack_edges(&mut row, &mut tg, 0x0000, &crate::registry::OBO_CORE); + assert_eq!(st.written, 5); + assert_eq!(st.lanes, 2); + let got: Vec<_> = EdgeLanes::new(&row) + .links_of(Predicate::IsA) + .map(|l| l.num) + .collect(); + assert_eq!(got, vec![1, 2, 3, 4, 5]); + } + + #[test] + fn predicates_are_recovered_from_the_degree_histogram() { + // the lane carries no predicate byte — the reader recovers it from + // row_ptr. Two predicates, one shared target ontology. + let mut row = row_with_degrees(&[(Predicate::IsA, 2), (Predicate::PartOf, 1)]); + let mut tg = vec![ + (Predicate::PartOf, t(Namespace::Uberon, 300)), + (Predicate::IsA, t(Namespace::Uberon, 100)), + (Predicate::IsA, t(Namespace::Uberon, 200)), + ]; + pack_edges(&mut row, &mut tg, 0x0000, &crate::registry::OBO_CORE); + let got: Vec<_> = EdgeLanes::new(&row) + .links() + .map(|(p, l)| (p, l.num)) + .collect(); + assert_eq!( + got, + vec![ + (Predicate::IsA, 100), + (Predicate::IsA, 200), + (Predicate::PartOf, 300), + ] + ); + } + + #[test] + fn an_overflow_is_reported_and_never_silently_truncated() { + let mut row = row_with_degrees(&[(Predicate::IsA, 255)]); + // one target ontology, more links than the budget holds + let over = EDGE_SLOT_COUNT + 7; + let mut tg: Vec<_> = (1..=over as u32) + .map(|n| (Predicate::IsA, t(Namespace::Hpo, n))) + .collect(); + let st = pack_edges(&mut row, &mut tg, 0x0000, &crate::registry::OBO_CORE); + assert_eq!(st.rows_overflowed, 1); + assert_eq!(st.dropped, over - EDGE_SLOT_COUNT); + assert_eq!(st.written, EDGE_SLOT_COUNT); + } + + #[test] + fn the_reader_can_fail() { + // anti-vacuity: the walk is sensitive to the histogram, not decorative. + // Same lane bytes, a degree of 1 instead of 3 → exactly one link read. + let mut row = row_with_degrees(&[(Predicate::IsA, 3)]); + let mut tg: Vec<_> = (1..=3) + .map(|n| (Predicate::IsA, t(Namespace::Pato, n))) + .collect(); + pack_edges(&mut row, &mut tg, 0x0000, &crate::registry::OBO_CORE); + assert_eq!(EdgeLanes::new(&row).links().count(), 3); + + row[EDGES_OFFSET + Predicate::IsA as usize] = 1; + assert_eq!( + EdgeLanes::new(&row).links().count(), + 1, + "row_ptr is load-bearing: a wrong degree must change the walk" + ); + } + + #[test] + fn the_subsumption_projection_drops_every_non_is_a_edge() { + // A row with both an is_a and a part_of parent. Handing the mixed set + // to a reasoner without role composition derives false ancestors, so + // the projection must yield ONLY the is_a edge. + let mut row = row_with_degrees(&[(Predicate::IsA, 1), (Predicate::PartOf, 1)]); + row[0..16].copy_from_slice(&crate::pack_key(0x0303_0000, 2244)); + let mut tg = vec![ + (Predicate::IsA, t(Namespace::Uberon, 10)), + (Predicate::PartOf, t(Namespace::Uberon, 20)), + ]; + pack_edges(&mut row, &mut tg, 0x0000, &crate::registry::OBO_CORE); + + // both edges are readable... + assert_eq!(EdgeLanes::new(&row).links().count(), 2); + // ...but only the subsumption is projected, and the subject is the row. + let rows = [crate::Row512(row)]; + let got: Vec<_> = subsumptions(&rows).collect(); + assert_eq!( + got, + vec![( + Link { + classid: 0x0303_0000, + num: 2244 + }, + Link { + classid: 0x0303_0000, + num: 10 + } + )], + "part_of must not reach a reasoner that lacks role composition" + ); + } + + /// Gate A1, stated as the bug it fixes. + /// + /// A link to numeric `0` — a real term in three measured namespaces, one + /// of them a namespace root — must round-trip AND must not end the walk. + /// Under the pre-bias encoding this test fails on both counts: the slot + /// bytes are all-zero, so the reader reads a pad, drops the link, and + /// **silently truncates the rest of the lane** — the second target is lost + /// too. That is why the assertion below checks the *following* link, not + /// just the zero one. + #[test] + fn a_link_to_numeric_zero_survives_and_does_not_end_the_walk() { + let mut row = row_with_degrees(&[(Predicate::IsA, 2)]); + let mut tg = vec![ + (Predicate::IsA, t(Namespace::Hpo, 0)), + (Predicate::IsA, t(Namespace::Hpo, 118)), + ]; + let st = pack_edges(&mut row, &mut tg, 0x0000, &crate::registry::OBO_CORE); + assert_eq!(st.written, 2); + assert_eq!(st.dropped, 0); + + let got: Vec<_> = EdgeLanes::new(&row) + .links_of(Predicate::IsA) + .map(|l| l.num) + .collect(); + assert_eq!( + got, + vec![0, 118], + "numeric 0 is a real target, and the link after it must survive too" + ); + + // The stored bytes must NOT be zero — that is the whole mechanism. + let base = VALUE_OFFSET + EDGE_LANE_SLAB_OFFSET; + assert_ne!( + u24_le(row[base + 4], row[base + 5], row[base + 6]), + 0, + "a real target must never encode to the pad value" + ); + } + + /// The pad still works — the can-it-stay-silent half. An unused slot must + /// still terminate the lane, or the bias would have traded one silent + /// failure for another. + #[test] + fn an_unused_slot_still_reads_as_pad_and_advances_the_lane() { + // Two target ontologies force two lanes; the first lane keeps three + // empty slots after its single link. + let mut row = row_with_degrees(&[(Predicate::IsA, 2)]); + let mut tg = vec![ + (Predicate::IsA, t(Namespace::Mondo, 5015)), + (Predicate::IsA, t(Namespace::Uberon, 955)), + ]; + pack_edges(&mut row, &mut tg, 0x0000, &crate::registry::OBO_CORE); + let got: Vec<_> = EdgeLanes::new(&row) + .links() + .map(|(_, l)| (l.classid, l.num)) + .collect(); + assert_eq!( + got, + vec![(0x0301_0000, 5015), (0x0303_0000, 955)], + "the three empty slots after the first link must advance the lane, \ + not yield phantom links" + ); + } + + #[test] + fn the_bias_round_trips_across_the_whole_representable_range() { + for n in [ + 0u32, + 1, + 2, + 118, + 65_535, + 65_536, + 700_092, + 9_999_994, + MAX_LINK_NUMERIC, + ] { + assert_eq!(decode_numeric(encode_numeric(n)), n, "{n} must round-trip"); + assert_ne!(encode_numeric(n), 0, "{n} must not collide with the pad"); + assert!( + encode_numeric(n) <= 0x00FF_FFFF, + "{n} must still fit the u24 slot after biasing" + ); + } + } + + /// The one numeric the biased u24 cannot hold is reported, not truncated. + /// + /// `TermId::parse` already rejects `> 0x00FF_FFFF`, so this is reachable + /// only by constructing a `TermId` directly — but a guard that cannot be + /// shown to fire is the defect one level up. + #[test] + fn an_unrepresentable_numeric_is_dropped_rather_than_wrapped() { + let mut row = row_with_degrees(&[(Predicate::IsA, 2)]); + let mut tg = vec![ + ( + Predicate::IsA, + TermId { + ns: Namespace::Hpo as u8, + num: 0x00FF_FFFF, + }, + ), + (Predicate::IsA, t(Namespace::Hpo, 7)), + ]; + let st = pack_edges(&mut row, &mut tg, 0x0000, &crate::registry::OBO_CORE); + assert_eq!(st.dropped, 1, "the unrepresentable link must be reported"); + assert_eq!(st.rows_overflowed, 1, "and must block the ship"); + assert_eq!(st.written, 1, "the representable link still lands"); + // It must not have wrapped to the pad, nor to some other term. + let got: Vec<_> = EdgeLanes::new(&row) + .links_of(Predicate::IsA) + .map(|l| l.num) + .collect(); + assert_eq!(got, vec![7], "no phantom target from a wrapped numeric"); + } + + #[test] + fn a_row_key_round_trips_through_the_family_identity_rail() { + // an oversize CURIE (> u16) must survive: the numeric rides the rail, + // high half in `family`. A truncating decode would return 4556 here. + let mut row = [0u8; NODE_ROW_STRIDE]; + row[0..16].copy_from_slice(&crate::pack_key(0x0301_0000, 700_092)); + assert_eq!( + subject(&row), + Link { + classid: 0x0301_0000, + num: 700_092 + } + ); + } + + #[test] + fn an_edge_lane_never_overwrites_the_entity_type_tenant() { + let mut row = [0u8; NODE_ROW_STRIDE]; + let et = VALUE_OFFSET + crate::ENTITY_TYPE_SLAB_OFFSET; + row[et..et + 2].copy_from_slice(&7u16.to_le_bytes()); + row[EDGES_OFFSET + Predicate::IsA as usize] = EDGE_SLOT_COUNT as u8; + let mut tg: Vec<_> = (1..=EDGE_SLOT_COUNT as u32) + .map(|n| (Predicate::IsA, t(Namespace::Ro, n))) + .collect(); + let st = pack_edges(&mut row, &mut tg, 0x0000, &crate::registry::OBO_CORE); + assert_eq!(st.dropped, 0); + assert_eq!( + u16::from_le_bytes([row[et], row[et + 1]]), + 7, + "a full edge pack must leave the EntityType tenant intact" + ); + } +} diff --git a/crates/ogar-obo/src/lib.rs b/crates/ogar-obo/src/lib.rs index a5f54450..f1fd1e28 100644 --- a/crates/ogar-obo/src/lib.rs +++ b/crates/ogar-obo/src/lib.rs @@ -41,11 +41,16 @@ #![deny(missing_docs)] pub mod crosswalk; +pub mod edges; +#[cfg(feature = "rdf")] +pub mod rdf; pub mod reason; +pub mod registry; /// RF2 snapshot reader — the one parser for SNOMED release-format files /// (active filter, snapshot uniqueness, column integrity). Consumers never /// hand-roll this. pub mod rf2; +pub mod spine; /// Row stride of the canonical SoA node — `key(16) + edges(16) + value(480)`. /// Mirrors `lance_graph_contract::canonical_node::NODE_ROW_STRIDE`. @@ -175,10 +180,7 @@ impl TermId { if num > 0x00FF_FFFF { return None; } - Some(TermId { - ns: ns as u8, - num, - }) + Some(TermId { ns: ns as u8, num }) } /// This term's namespace. @@ -287,7 +289,7 @@ impl Row512 { /// and family stay `0` in this first bake — the zero-fallback ladder means /// `identity` alone discriminates until an HHTL basin mint wakes routing /// (RESERVE, DON'T RECLAIM: fixed offsets, zero layout change later). -fn pack_key(classid: u32, identity: u32) -> [u8; 16] { +pub(crate) fn pack_key(classid: u32, identity: u32) -> [u8; 16] { let mut k = [0u8; 16]; k[0..4].copy_from_slice(&classid.to_le_bytes()); // 4..10 HEEL/HIP/TWIG = 0 (dormant cascade) @@ -321,13 +323,37 @@ fn pack_key(classid: u32, identity: u32) -> [u8; 16] { k } -/// Pack one node into its 512-byte row. Edges are placed **out-of-line** as -/// SPO triples (see [`bake`]); the in-row `edges` block records the *degree* -/// per predicate (a one-byte histogram, saturating at 255) so a reader has the -/// node's local shape without the triple table, and the `EntityType` value -/// tenant records the namespace. The one-byte basin-adjacency use of the edge -/// block is deferred to the HHTL-clustering pass (RESERVE, DON'T RECLAIM). -fn pack_row(classid: u32, id: &TermId, node: &OboNode) -> Row512 { +/// Read a row's key back as `(classid, CURIE numeric)` — the inverse of +/// `pack_key`, recombining the `family:identity` rail into the 24-bit numeric. +/// +/// This is the *whole* prerender property in one call: a reader learns which +/// class a row is and which instance, with **zero value decode**. +#[must_use] +pub const fn unpack_key(row: &[u8; NODE_ROW_STRIDE]) -> (u32, u32) { + let classid = u32::from_le_bytes([row[0], row[1], row[2], row[3]]); + let family = u16::from_le_bytes([row[12], row[13]]) as u32; + let identity = u16::from_le_bytes([row[14], row[15]]) as u32; + (classid, (family << 16) | identity) +} + +/// Pack one node into its 512-byte row. +/// +/// The in-row `edges` block records the *degree* per predicate (a one-byte +/// histogram, saturating at 255) — the CSR `row_ptr`. The **targets** (`col_idx`) +/// ride the free value lanes as G2 `4×u24` literal links; see [`edges`] for the +/// lane form and the reader's contract. The `EntityType` value tenant records +/// the namespace. The same edges are also emitted out-of-line as SPO triples by +/// [`bake`] — the in-row copy is what makes a row traversable on its own. +/// +/// Returns the row plus the pack stats, so a driver can refuse to ship a +/// truncation rather than lose edges silently. +fn pack_row( + classid: u32, + id: &TermId, + node: &OboNode, + app_prefix: u16, + reg: ®istry::NsRegistry, +) -> (Row512, edges::PackStats) { let mut row = Row512::zeroed(); row.0[0..16].copy_from_slice(&pack_key(classid, id.num)); // edges block [16,32): a per-predicate degree histogram (index by Predicate @@ -343,7 +369,13 @@ fn pack_row(classid: u32, id: &TermId, node: &OboNode) -> Row512 { // value slab: EntityType tenant (u16 namespace) at its carve offset. let et = VALUE_OFFSET + ENTITY_TYPE_SLAB_OFFSET; row.0[et..et + 2].copy_from_slice(&(id.ns as u16).to_le_bytes()); - row + // value slab: the edge targets — the col_idx half of the CSR. + let mut targets: Vec<(Predicate, TermId)> = + Vec::with_capacity(node.is_a.len() + node.rel.len()); + targets.extend(node.is_a.iter().map(|t| (Predicate::IsA, *t))); + targets.extend(node.rel.iter().copied()); + let st = edges::pack_edges(&mut row.0, &mut targets, app_prefix, reg); + (row, st) } // ── OBO `.obo` parser ───────────────────────────────────────────────────── @@ -418,16 +450,18 @@ pub fn parse_obo(text: &str) -> std::collections::HashMap { // `relationship: ! label` — the target is the LAST // whitespace token before any `!`. if let Some(sid) = cur - && let Some(t) = last_curie(rest) { - let p = classify(sid.namespace(), t.namespace()); - nodes.entry(sid).or_default().rel.push((p, t)); - } + && let Some(t) = last_curie(rest) + { + let p = classify(sid.namespace(), t.namespace()); + nodes.entry(sid).or_default().rel.push((p, t)); + } } else if let Some(rest) = line.strip_prefix("intersection_of: ") { if let Some(sid) = cur - && let Some(t) = last_curie(rest) { - let p = classify(sid.namespace(), t.namespace()); - nodes.entry(sid).or_default().rel.push((p, t)); - } + && let Some(t) = last_curie(rest) + { + let p = classify(sid.namespace(), t.namespace()); + nodes.entry(sid).or_default().rel.push((p, t)); + } } else if let Some(rest) = line.strip_prefix("xref: ") { // `xref: : ! label` — the projection-join / guideline // bearing. Kept verbatim; NEVER truncated (MeSH → Leitlinie spider). @@ -435,12 +469,14 @@ pub fn parse_obo(text: &str) -> std::collections::HashMap { let tok = rest.split('!').next().unwrap_or(rest).trim(); let tok = tok.split_whitespace().next().unwrap_or(tok); if let Some((src, id)) = tok.split_once(':') - && !src.is_empty() && !id.is_empty() { - nodes.entry(sid).or_default().xref.push(Xref { - source: XrefSource::from_prefix(src), - id: id.to_string(), - }); - } + && !src.is_empty() + && !id.is_empty() + { + nodes.entry(sid).or_default().xref.push(Xref { + source: XrefSource::from_prefix(src), + id: id.to_string(), + }); + } } } } @@ -450,14 +486,22 @@ pub fn parse_obo(text: &str) -> std::collections::HashMap { /// First whitespace token of a line (before any `!` comment) — the target of /// an `is_a:` line. fn first_curie(s: &str) -> &str { - s.split('!').next().unwrap_or(s).split_whitespace().next().unwrap_or("").trim() + s.split('!') + .next() + .unwrap_or(s) + .split_whitespace() + .next() + .unwrap_or("") + .trim() } /// Last CURIE-shaped token before any `!` — the object of a `relationship:` / /// `intersection_of:` line (the predicate is the earlier token). fn last_curie(s: &str) -> Option { let head = s.split('!').next().unwrap_or(s); - head.split_whitespace().rfind(|t| t.contains(':')).and_then(TermId::parse) + head.split_whitespace() + .rfind(|t| t.contains(':')) + .and_then(TermId::parse) } // ── bake: nodes+edges → 512-byte rows + SPO triples + stats ──────────────── @@ -474,7 +518,10 @@ pub struct BakeStats { pub triples: usize, /// `is_a` cycles found (SCC size > 1) — must be 0 on a clean OBO core pub is_a_cycles: usize, - /// edges whose object namespace is core but the object node is absent + /// edges dropped because their object gets no ROW — absent from the + /// sources, or present but obsolete and therefore not tiled. Counting + /// only the first was the bug: an edge into an obsolete term reported + /// clean here and dangling to any reader of the artifact. pub dangling: usize, /// MONDO→HP has_phenotype edges resolving to a loaded HP node pub mondo_hp: usize, @@ -486,6 +533,19 @@ pub struct BakeStats { pub xrefs: usize, /// MeSH xrefs specifically — the guideline-spider bearings pub mesh_xrefs: usize, + /// edge targets written into value lanes — the CSR `col_idx`. Equals + /// [`BakeStats::triples`] on a clean bake (every edge is also in-row). + pub links_packed: usize, + /// edge targets that did NOT fit the lane budget. **Must be 0 to ship** — + /// a non-zero value means the artifact is missing edges. + pub links_dropped: usize, + /// rows that overflowed the lane budget (must be 0 to ship) + pub rows_overflowed: usize, + /// duplicate edges collapsed at bake time — the same `(predicate, object)` + /// asserted more than once for one subject, which a multi-source union + /// produces routinely. Reported rather than silent: it is the difference + /// between the degree histogram and the distinct edge count. + pub duplicates: usize, } /// The bake result: the 512-byte SoA rows (loader-facing) + the SPO triple @@ -510,9 +570,23 @@ pub struct Bake { /// logical-def edges folded into the nodes' `rel`) into [`Bake`]. `app_prefix` /// is the lo-u16 render skin (`0x0000` = the canonical reference skin). #[must_use] -pub fn bake( +pub fn bake(nodes: &std::collections::HashMap, app_prefix: u16) -> Bake { + bake_with(nodes, app_prefix, ®istry::OBO_CORE) +} + +/// [`bake`] against an explicit namespace table. +/// +/// This is the general form; `bake` is this against +/// [`registry::OBO_CORE`]. A row's classid and every edge lane's classid come +/// from `reg`, so a bake over a different namespace set needs no code change — +/// only a different table. A target whose namespace is absent from `reg` has +/// no classid to name it, so its link is reported in +/// [`BakeStats::links_dropped`] rather than guessed at. +#[must_use] +pub fn bake_with( nodes: &std::collections::HashMap, app_prefix: u16, + reg: ®istry::NsRegistry, ) -> Bake { let mut ids: Vec = nodes .iter() @@ -529,10 +603,56 @@ pub fn bake( ..Default::default() }; + // The set of ids that will actually GET a row. `dangling` used to mean + // "object absent from the node map", but an obsolete term IS in the map + // and deliberately gets no row — so an edge into one pointed at a row that + // does not exist, and the stat said 0. Measured on the RDF spine: 55 such + // edges, reported as dangling by a reader and as clean by the bake. + // + // An edge into an obsolete term is itself obsolete, so it is dropped + // rather than emitted, and counted. Digest-neutral for the `.obo` core + // (no edge there targets an untiled row); it is the RDF path, where + // `owl:deprecated` classes are still referenced, that this corrects. + let tiled: std::collections::HashSet = ids.iter().copied().collect(); + for id in &ids { let node = &nodes[id]; - let classid = id.namespace().render_classid(app_prefix); - rows.push(pack_row(classid, id, node)); + let node = { + let mut n = node.clone(); + let before = n.is_a.len() + n.rel.len(); + n.is_a.retain(|t| tiled.contains(t)); + n.rel.retain(|(_, t)| tiled.contains(t)); + stats.dangling += before - (n.is_a.len() + n.rel.len()); + // Deduplicate. A driver that unions several sources — `bake_obo` + // merges five `.obo` files with `extend` — concatenates the parent + // lists of a term that appears in more than one, so the same + // `is_a` lands twice. Measured on the shipped core: 15,128 + // duplicate slots across 10,589 rows, 14 % of the is_a links + // written. + // + // Idempotent semantically (`A is_a B` twice is `A is_a B`), so it + // produces no wrong answer — but it spends lane budget and makes + // the degree histogram over-report a row's out-degree, which is + // what `resident_out_degree` reads. `rdf::parse_rdf` already + // dedups its own output; doing it here covers every driver instead + // of every driver having to remember. + let pre = n.is_a.len() + n.rel.len(); + n.is_a.sort_unstable(); + n.is_a.dedup(); + n.rel.sort_unstable(); + n.rel.dedup(); + stats.duplicates += pre - (n.is_a.len() + n.rel.len()); + n + }; + let node = &node; + let Some(classid) = reg.render_classid(id.ns, app_prefix) else { + continue; + }; + let (row, pk) = pack_row(classid, id, node, app_prefix, reg); + rows.push(row); + stats.links_packed += pk.written; + stats.links_dropped += pk.dropped; + stats.rows_overflowed += pk.rows_overflowed; // Preserve every external cross-reference — the projection-join / // guideline-spider bearings. NEVER truncated. for x in &node.xref { @@ -547,9 +667,10 @@ pub fn bake( p: Predicate::IsA, o: *parent, }); - if !nodes.contains_key(parent) { - stats.dangling += 1; - } + debug_assert!( + nodes.contains_key(parent), + "untiled targets already filtered" + ); } for (p, o) in &node.rel { triples.push(Triple { @@ -557,15 +678,11 @@ pub fn bake( p: *p, o: *o, }); - if !nodes.contains_key(o) { - stats.dangling += 1; - } else { - match p { - Predicate::HasPhenotype => stats.mondo_hp += 1, - Predicate::HasAnatomy => stats.hp_uberon += 1, - Predicate::HasQuality => stats.hp_pato += 1, - _ => {} - } + match p { + Predicate::HasPhenotype => stats.mondo_hp += 1, + Predicate::HasAnatomy => stats.hp_uberon += 1, + Predicate::HasQuality => stats.hp_pato += 1, + _ => {} } } } @@ -659,22 +776,35 @@ pub fn parse_hp_logical_defs(owl: &str) -> Vec<(TermId, Predicate, TermId)> { in_eq += 1; } if in_eq > 0 - && let Some(sid) = cur { - for uid in find_obo_ids(line, "UBERON_") { - out.push(( - TermId { ns: Namespace::Hpo as u8, num: sid }, - Predicate::HasAnatomy, - TermId { ns: Namespace::Uberon as u8, num: uid }, - )); - } - for pid in find_obo_ids(line, "PATO_") { - out.push(( - TermId { ns: Namespace::Hpo as u8, num: sid }, - Predicate::HasQuality, - TermId { ns: Namespace::Pato as u8, num: pid }, - )); - } + && let Some(sid) = cur + { + for uid in find_obo_ids(line, "UBERON_") { + out.push(( + TermId { + ns: Namespace::Hpo as u8, + num: sid, + }, + Predicate::HasAnatomy, + TermId { + ns: Namespace::Uberon as u8, + num: uid, + }, + )); + } + for pid in find_obo_ids(line, "PATO_") { + out.push(( + TermId { + ns: Namespace::Hpo as u8, + num: sid, + }, + Predicate::HasQuality, + TermId { + ns: Namespace::Pato as u8, + num: pid, + }, + )); } + } if line.contains("") { in_eq = (in_eq - 1).max(0); } @@ -689,7 +819,10 @@ fn find_hp_about(line: &str) -> Option { let i = line.find("owl:Class rdf:about=")?; let rest = &line[i..]; let j = rest.find("HP_")?; - let digits: String = rest[j + 3..].chars().take_while(char::is_ascii_digit).collect(); + let digits: String = rest[j + 3..] + .chars() + .take_while(char::is_ascii_digit) + .collect(); digits.parse().ok() } @@ -701,9 +834,10 @@ fn find_obo_ids(line: &str, prefix: &str) -> Vec { let after = &hay[i + prefix.len()..]; let digits: String = after.chars().take_while(char::is_ascii_digit).collect(); if let Ok(n) = digits.parse::() - && n <= 0x00FF_FFFF { - ids.push(n); - } + && n <= 0x00FF_FFFF + { + ids.push(n); + } hay = &after[digits.len()..]; } ids @@ -746,15 +880,26 @@ mod tests { fn v3_tail_carries_oversize_curie_numerics_on_the_family_identity_rail() { // Above u16 — the case a bare `identity` cannot hold. let big = 700_092u32; - assert!(big > u32::from(u16::MAX), "fixture must exceed the u16 the rail exists for"); + assert!( + big > u32::from(u16::MAX), + "fixture must exceed the u16 the rail exists for" + ); let k = pack_key(Namespace::Mondo.render_classid(0x0000), big); // Byte positions, per new_v2. - assert_eq!(&k[10..12], &[0, 0], "leaf stays dormant (RESERVE, DON'T RECLAIM)"); + assert_eq!( + &k[10..12], + &[0, 0], + "leaf stays dormant (RESERVE, DON'T RECLAIM)" + ); let family = u16::from_le_bytes([k[12], k[13]]); let identity = u16::from_le_bytes([k[14], k[15]]); assert_eq!(u32::from(family), big >> 16, "family holds the high half"); - assert_eq!(u32::from(identity), big & 0xFFFF, "identity holds the low half"); + assert_eq!( + u32::from(identity), + big & 0xFFFF, + "identity holds the low half" + ); // Lossless through the public reader. let mut row = Row512::zeroed(); @@ -766,13 +911,19 @@ mod tests { // The V1 read of the SAME bytes must NOT agree — proof the tail really // moved, not that both layouts happen to coincide on this fixture. let v1_read = u32::from_le_bytes([row.0[13], row.0[14], row.0[15], 0]); - assert_ne!(v1_read, big, "a V1 u24 read of a V3 row must be observably wrong"); + assert_ne!( + v1_read, big, + "a V1 u24 read of a V3 row must be observably wrong" + ); // Ordering is preserved: family is the HIGH half, so (family, identity) // sorts as the numeric does. This is what keeps binary-search-by-key // valid over the sorted bake. let lo = pack_key(Namespace::Mondo.render_classid(0x0000), 5_148); - assert!(lo[12..16] < k[12..16], "tail bytes order as the numeric orders"); + assert!( + lo[12..16] < k[12..16], + "tail bytes order as the numeric orders" + ); } #[test] @@ -802,7 +953,11 @@ is_a: MONDO:0005015 ! diabetes mellitus\n"; let t = n(Namespace::Mondo, 5148); let node = &nodes[&t]; assert_eq!(node.xref.len(), 3, "all three xrefs kept"); - assert!(node.xref.iter().any(|x| x.source == XrefSource::Mesh && x.id == "D003924")); + assert!( + node.xref + .iter() + .any(|x| x.source == XrefSource::Mesh && x.id == "D003924") + ); let bake = bake(&nodes, 0x0000); assert_eq!(bake.stats.mesh_xrefs, 1, "MeSH bearing counted"); assert_eq!(bake.stats.xrefs, 3); diff --git a/crates/ogar-obo/src/rdf.rs b/crates/ogar-obo/src/rdf.rs new file mode 100644 index 00000000..6eff4e7e --- /dev/null +++ b/crates/ogar-obo/src/rdf.rs @@ -0,0 +1,640 @@ +//! # `rdf` — retain the assertions, key them by the CURIE numeric. +//! +//! The workspace already parses RDF: `lance-graph-ontology`'s `OwlHydrator` +//! reads Turtle and RDF/XML through `oxttl` / `oxrdfxml`. But it interns every +//! named IRI to a **sequential counter** and then *drops the triple* — its +//! `OntologySlot` is `{ entity_count, iri_to_id }` and nothing else. Two +//! consequences make it unusable as a bake front-end: +//! +//! 1. **No assertions survive.** There is no `subClassOf` anywhere after +//! hydration; `ContextBundle::edge_types` is a whitelist of predicate IRIs +//! a traversal *may* follow, not stored edges. +//! 2. **The identity is order-dependent.** An intern counter depends on file +//! order, so the same ontology hydrated from a different file list yields +//! different ids. A bake keyed on it would not be reproducible. +//! +//! This module is the opposite of both: it **retains** the edges and takes the +//! identity from the term itself — `…/obo/HP_0000118` → `HP:0000118` → the +//! 24-bit numeric `118`, which is exactly the identity [`crate::pack_key`] +//! writes on the `family:identity` rail. Bake the same ontology from any file +//! order and the rows are byte-identical. +//! +//! It is NOT a second parser: `oxttl` / `oxrdfxml` do the reading, pinned to +//! the same versions `lance-graph-ontology` pins. +//! +//! ## The extraction map IS the [`NsRegistry`] +//! +//! Every OBO PURL is `_`. One rule decodes all of them, +//! so a new ontology needs a **table row**, never a code path. That is the +//! whole per-ontology config: prefix + concept id. +//! +//! ## Predicates come from `owl:onProperty`, not from a namespace-pair guess +//! +//! [`crate::parse_obo`] classifies an edge by *which two ontologies it joins* +//! ([`crate::classify`]) — a deliberate choice, because the `.obo` +//! `relationship:` label string is unreliable. RDF does not have that problem: +//! a restriction names its property explicitly, so this path reads +//! [`OBO_RESTRICTION_PREDICATES`] and falls through to +//! [`Predicate::Other`](crate::Predicate::Other) rather than guessing. +//! +//! **This means an RDF bake and a `.obo` bake of the same ontology may assign +//! different predicates to the same edge.** That is a real difference between +//! two sources, not a bug in either; the shipped five-namespace bake was +//! produced from `.obo` and is not re-baked by this module. +//! +//! ## Documented limits (not silent gaps) +//! +//! - Only **direct** restrictions are harvested: `X subClassOf [ onProperty P ; +//! someValuesFrom Y ]` and the same shape under `owl:equivalentClass`. +//! Restrictions nested inside an `owl:intersectionOf` RDF **collection** are +//! NOT walked, so logical definitions expressed that way yield no edge here. +//! - `owl:equivalentClass` to a **named** class is skipped. Equivalence is not +//! subsumption, and recording it as `is_a` would invent a parent. +//! - A term becomes a node only if it is asserted `a owl:Class`. Being merely +//! *mentioned* as some other term's parent does not create a row — the same +//! rule `.obo` follows with its `id:` stanza line. + +use std::collections::HashMap; + +use oxrdf::{NamedOrBlankNode, Term}; +use oxrdfxml::RdfXmlParser; +use oxttl::TurtleParser; + +use crate::registry::NsRegistry; +use crate::{OboNode, Predicate, TermId, Xref, XrefSource}; + +/// The OBO PURL base every OBO Foundry term IRI is built on. +pub const OBO_PURL_BASE: &str = "http://purl.obolibrary.org/obo/"; + +const RDF_TYPE: &str = "http://www.w3.org/1999/02/22-rdf-syntax-ns#type"; +const OWL_CLASS: &str = "http://www.w3.org/2002/07/owl#Class"; +const RDFS_SUBCLASS_OF: &str = "http://www.w3.org/2000/01/rdf-schema#subClassOf"; +const OWL_ON_PROPERTY: &str = "http://www.w3.org/2002/07/owl#onProperty"; +const OWL_SOME_VALUES_FROM: &str = "http://www.w3.org/2002/07/owl#someValuesFrom"; +const OWL_EQUIVALENT_CLASS: &str = "http://www.w3.org/2002/07/owl#equivalentClass"; +const OWL_DEPRECATED: &str = "http://www.w3.org/2002/07/owl#deprecated"; +const OBO_IN_OWL_HAS_DB_XREF: &str = "http://www.geneontology.org/formats/oboInOwl#hasDbXref"; + +/// The RDF serializations this module reads. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RdfFormat { + /// Turtle / N-Triples family (`oxttl`). + Turtle, + /// RDF/XML (`oxrdfxml`). + RdfXml, +} + +/// Choose a serialization from the file extension, falling back to sniffing +/// the head of the document. +/// +/// `.owl` is genuinely ambiguous — it ships in both forms in the wild — so an +/// unknown or `.owl` extension sniffs for an XML preamble or an `` +/// opening tag and defaults to Turtle otherwise. Same rule +/// `lance-graph-ontology` uses, so the two agree on any given file. +#[must_use] +pub fn detect_format(extension: Option<&str>, head: &[u8]) -> RdfFormat { + match extension.map(str::to_ascii_lowercase).as_deref() { + Some("rdf" | "rdfxml" | "owx") => RdfFormat::RdfXml, + Some("ttl" | "nt" | "n3" | "trig") => RdfFormat::Turtle, + _ => { + let head = &head[..head.len().min(256)]; + let start = head + .iter() + .position(|b| !b.is_ascii_whitespace() && *b != 0xEF && *b != 0xBB && *b != 0xBF) + .unwrap_or(0); + let trimmed = &head[start..]; + if trimmed.starts_with(b") -> std::fmt::Result { + write!(f, "RDF parse error: {}", self.message) + } +} + +impl std::error::Error for RdfErr {} + +/// Split an OBO PURL into its `(prefix, 24-bit numeric)` parts. +/// +/// `None` when the IRI is not an OBO PURL, has no `_` separator, or the +/// numeric is malformed or exceeds 24 bits — the same width contract +/// [`TermId::parse`] enforces. +#[must_use] +pub fn purl_to_curie(iri: &str) -> Option<(&str, u32)> { + let local = iri.strip_prefix(OBO_PURL_BASE)?; + let (prefix, digits) = local.rsplit_once('_')?; + if prefix.is_empty() { + return None; + } + let num: u32 = digits.parse().ok()?; + if num > 0x00FF_FFFF { + return None; + } + Some((prefix, num)) +} + +/// Resolve an OBO PURL to a [`TermId`] under a registry, or `None` if the +/// prefix is outside the table (imports from ontologies this bake does not +/// tile are skipped, exactly as [`crate::parse_obo`] skips them). +#[must_use] +pub fn term_of(iri: &str, reg: &NsRegistry) -> Option { + let (prefix, num) = purl_to_curie(iri)?; + Some(TermId { + ns: reg.ordinal_of(prefix)?, + num, + }) +} + +/// Map a restriction's property IRI to an RO predicate through a table. +#[must_use] +pub fn predicate_of(property_iri: &str, table: &[PredSpec]) -> Predicate { + let Some((prefix, num)) = purl_to_curie(property_iri) else { + return Predicate::Other; + }; + table + .iter() + .find(|s| { + s.curie + .split_once(':') + .is_some_and(|(p, n)| p == prefix && n.parse::().ok() == Some(num)) + }) + .map_or(Predicate::Other, |s| s.predicate) +} + +/// A partially-seen `owl:Restriction` blank node. +#[derive(Default)] +struct Restriction { + property: Option, + filler: Option, +} + +/// Everything collected in the single streaming pass, before blank nodes are +/// resolved against the restrictions they refer to. +#[derive(Default)] +struct Harvest { + nodes: HashMap, + restrictions: HashMap, + blank_edges: Vec<(TermId, String)>, +} + +impl Harvest { + fn absorb( + &mut self, + subject: &NamedOrBlankNode, + predicate: &str, + object: &Term, + reg: &NsRegistry, + ) { + match subject { + NamedOrBlankNode::NamedNode(s) => { + let Some(sid) = term_of(s.as_str(), reg) else { + return; + }; + match (predicate, object) { + // `a owl:Class` is what creates a row — mere mention does not. + (RDF_TYPE, Term::NamedNode(o)) if o.as_str() == OWL_CLASS => { + self.nodes.entry(sid).or_default(); + } + (RDFS_SUBCLASS_OF, Term::NamedNode(o)) => { + if let Some(parent) = term_of(o.as_str(), reg) { + self.nodes.entry(sid).or_default().is_a.push(parent); + } + } + // A restriction hangs off a blank node; resolve it later. + (RDFS_SUBCLASS_OF | OWL_EQUIVALENT_CLASS, Term::BlankNode(b)) => { + self.blank_edges.push((sid, b.as_str().to_string())); + } + (OWL_DEPRECATED, Term::Literal(l)) if l.value() == "true" => { + self.nodes.entry(sid).or_default().obsolete = true; + } + (OBO_IN_OWL_HAS_DB_XREF, Term::Literal(l)) => { + if let Some((src, id)) = l.value().split_once(':') + && !src.is_empty() + && !id.is_empty() + { + self.nodes.entry(sid).or_default().xref.push(Xref { + source: XrefSource::from_prefix(src), + id: id.to_string(), + }); + } + } + _ => {} + } + } + NamedOrBlankNode::BlankNode(b) => { + let slot = self.restrictions.entry(b.as_str().to_string()).or_default(); + match (predicate, object) { + (OWL_ON_PROPERTY, Term::NamedNode(o)) => { + slot.property = Some(o.as_str().to_string()); + } + (OWL_SOME_VALUES_FROM, Term::NamedNode(o)) => { + slot.filler = Some(o.as_str().to_string()); + } + _ => {} + } + } + } + } + + /// Join the blank-node edges to their restrictions. A blank node with no + /// `onProperty`/`someValuesFrom` pair — an intersection list, a cardinality + /// restriction, an anonymous union — yields nothing, per the documented + /// limits. + fn resolve(mut self, reg: &NsRegistry, table: &[PredSpec]) -> HashMap { + for (sid, blank) in std::mem::take(&mut self.blank_edges) { + let Some(r) = self.restrictions.get(&blank) else { + continue; + }; + let (Some(prop), Some(filler)) = (r.property.as_deref(), r.filler.as_deref()) else { + continue; + }; + let Some(object) = term_of(filler, reg) else { + continue; + }; + let p = predicate_of(prop, table); + self.nodes.entry(sid).or_default().rel.push((p, object)); + } + // Deterministic edge order regardless of triple arrival order — the + // bake packs `is_a` then `rel` into value lanes positionally, so an + // unstable order here would produce a different (still correct) row. + for node in self.nodes.values_mut() { + node.is_a.sort_unstable(); + node.is_a.dedup(); + node.rel.sort_unstable(); + node.rel.dedup(); + } + self.nodes + } +} + +/// Parse an RDF document into the `(id → node)` map [`crate::bake`] consumes. +/// +/// Streams the triples once, then joins the blank-node restrictions. Terms +/// whose prefix is outside `reg` are skipped, so an ontology's imports do not +/// leak rows into a bake that was not asked to tile them. +/// +/// # Errors +/// +/// Returns [`RdfErr`] if the underlying `oxttl` / `oxrdfxml` parser rejects +/// the document. +pub fn parse_rdf( + bytes: &[u8], + format: RdfFormat, + reg: &NsRegistry, + table: &[PredSpec], +) -> Result, RdfErr> { + let mut h = Harvest::default(); + match format { + RdfFormat::Turtle => { + for t in TurtleParser::new().for_slice(bytes) { + let t = t.map_err(|e| RdfErr { + message: e.to_string(), + })?; + h.absorb(&t.subject, t.predicate.as_str(), &t.object, reg); + } + } + RdfFormat::RdfXml => { + for t in RdfXmlParser::new().for_slice(bytes) { + let t = t.map_err(|e| RdfErr { + message: e.to_string(), + })?; + h.absorb(&t.subject, t.predicate.as_str(), &t.object, reg); + } + } + } + Ok(h.resolve(reg, table)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::registry::{NsRegistry, NsSpec}; + + const SPINE: NsRegistry = NsRegistry::new(&[ + NsSpec { + prefix: "BFO", + concept_id: 0x0310, + }, + NsSpec { + prefix: "UBERON", + concept_id: 0x0303, + }, + ]); + + /// Turtle with: two classes, a named subClassOf, a blank-node part_of + /// restriction, a deprecation, an xref, an out-of-registry parent, and a + /// blank node that is NOT a well-formed restriction. + const TTL: &str = r#" +@prefix rdf: . +@prefix rdfs: . +@prefix owl: . +@prefix obo: . +@prefix oio: . + +obo:UBERON_0000955 a owl:Class ; + rdfs:subClassOf obo:UBERON_0000062 ; + rdfs:subClassOf obo:CHEBI_0000001 ; + rdfs:subClassOf [ a owl:Restriction ; + owl:onProperty obo:BFO_0000050 ; + owl:someValuesFrom obo:UBERON_0000033 ] ; + oio:hasDbXref "MeSH:D001921" . + +obo:UBERON_0000062 a owl:Class . +obo:UBERON_0000033 a owl:Class . + +obo:UBERON_0000099 a owl:Class ; + owl:deprecated "true" . + +obo:UBERON_0000077 a owl:Class ; + rdfs:subClassOf [ a owl:Class ; + owl:unionOf ( obo:UBERON_0000062 obo:UBERON_0000033 ) ] . +"#; + + fn u(num: u32) -> TermId { + TermId { ns: 1, num } + } + + #[test] + fn purl_decoding_accepts_obo_iris_and_rejects_everything_else() { + assert_eq!( + purl_to_curie("http://purl.obolibrary.org/obo/HP_0000118"), + Some(("HP", 118)) + ); + // The LAST underscore separates — some OBO locals carry one. + assert_eq!( + purl_to_curie("http://purl.obolibrary.org/obo/NCBITaxon_Union_0000001"), + Some(("NCBITaxon_Union", 1)) + ); + // Rejects: wrong base, no separator, non-numeric, over 24 bits. + assert_eq!(purl_to_curie("http://example.org/HP_0000118"), None); + assert_eq!( + purl_to_curie("http://purl.obolibrary.org/obo/HP0000118"), + None + ); + assert_eq!(purl_to_curie("http://purl.obolibrary.org/obo/HP_abc"), None); + assert_eq!( + purl_to_curie("http://purl.obolibrary.org/obo/HP_99999999"), + None, + "the 24-bit width contract must hold here as it does in TermId::parse" + ); + assert_eq!(purl_to_curie("http://purl.obolibrary.org/obo/_1"), None); + } + + #[test] + fn the_registry_is_the_extraction_map() { + // UBERON is in SPINE, HP is not. Same IRI shape, different outcome — + // so the table is genuinely consulted rather than a prefix hardcoded. + assert_eq!( + term_of("http://purl.obolibrary.org/obo/UBERON_0000955", &SPINE), + Some(u(955)) + ); + assert_eq!( + term_of("http://purl.obolibrary.org/obo/HP_0000118", &SPINE), + None + ); + // …and the ordinal is table-relative: UBERON is row 1 here, row 2 in + // the shipped core. + assert_eq!( + term_of("http://purl.obolibrary.org/obo/UBERON_0000955", &SPINE) + .unwrap() + .ns, + 1 + ); + assert_eq!( + term_of( + "http://purl.obolibrary.org/obo/UBERON_0000955", + &crate::registry::OBO_CORE + ) + .unwrap() + .ns, + 2 + ); + } + + #[test] + fn restriction_properties_map_through_the_table_and_fall_through_otherwise() { + assert_eq!( + predicate_of( + "http://purl.obolibrary.org/obo/BFO_0000050", + OBO_RESTRICTION_PREDICATES + ), + Predicate::PartOf + ); + assert_eq!( + predicate_of( + "http://purl.obolibrary.org/obo/RO_0002200", + OBO_RESTRICTION_PREDICATES + ), + Predicate::HasPhenotype + ); + // An OBO property with no row → Other, never a guess. + assert_eq!( + predicate_of( + "http://purl.obolibrary.org/obo/RO_0002131", + OBO_RESTRICTION_PREDICATES + ), + Predicate::Other + ); + // A non-OBO property IRI → Other. + assert_eq!( + predicate_of( + "http://www.w3.org/2000/01/rdf-schema#seeAlso", + OBO_RESTRICTION_PREDICATES + ), + Predicate::Other + ); + // An EMPTY table must classify everything as Other — proof the table + // is read rather than the mapping being baked into the function. + assert_eq!( + predicate_of("http://purl.obolibrary.org/obo/BFO_0000050", &[]), + Predicate::Other + ); + } + + #[test] + fn turtle_harvest_keeps_edges_and_drops_what_it_documents_dropping() { + let nodes = parse_rdf( + TTL.as_bytes(), + RdfFormat::Turtle, + &SPINE, + OBO_RESTRICTION_PREDICATES, + ) + .expect("turtle parses"); + + // Four `a owl:Class` assertions → four rows. UBERON_0000033 and + // _0000062 are declared classes AND referenced; CHEBI is referenced + // but outside the registry, so it must NOT create a row. + assert_eq!(nodes.len(), 5, "one row per `a owl:Class`, none for CHEBI"); + + let brain = &nodes[&u(955)]; + // The named parent is kept; the out-of-registry one is skipped. + assert_eq!(brain.is_a, vec![u(62)], "CHEBI parent must be skipped"); + // The blank-node restriction resolved to a typed edge. + assert_eq!(brain.rel, vec![(Predicate::PartOf, u(33))]); + // The xref survived. + assert_eq!(brain.xref.len(), 1); + assert_eq!(brain.xref[0].source, XrefSource::Mesh); + assert_eq!(brain.xref[0].id, "D001921"); + + assert!(nodes[&u(99)].obsolete, "owl:deprecated true is read"); + assert!(!brain.obsolete); + + // The documented limit, asserted rather than hoped: a blank node that + // is a unionOf — not an onProperty/someValuesFrom restriction — + // contributes NO edge. + assert!( + nodes[&u(77)].rel.is_empty() && nodes[&u(77)].is_a.is_empty(), + "a non-restriction blank node must yield nothing, not a wrong edge" + ); + } + + /// The property this module exists for: identity comes from the term, so + /// the harvest cannot depend on document order. + /// + /// **What would make it fail:** an intern-counter identity, which is what + /// `lance-graph-ontology`'s `OwlHydrator` assigns. Reversing the stanzas + /// would then renumber every term. + #[test] + fn identity_is_the_curie_numeric_so_document_order_cannot_change_it() { + let forward = parse_rdf( + TTL.as_bytes(), + RdfFormat::Turtle, + &SPINE, + OBO_RESTRICTION_PREDICATES, + ) + .unwrap(); + + // Re-emit the same graph with the class stanzas in reverse order. + let mut stanzas: Vec<&str> = TTL.split("\n\n").collect(); + let header = stanzas.remove(0); + stanzas.reverse(); + let reordered = format!("{header}\n\n{}", stanzas.join("\n\n")); + let backward = parse_rdf( + reordered.as_bytes(), + RdfFormat::Turtle, + &SPINE, + OBO_RESTRICTION_PREDICATES, + ) + .unwrap(); + + assert_eq!(forward.len(), backward.len()); + for (id, node) in &forward { + let other = backward.get(id).expect("same ids under any order"); + assert_eq!(node.is_a, other.is_a, "{id:?} parents must match"); + assert_eq!(node.rel, other.rel, "{id:?} typed edges must match"); + assert_eq!(node.obsolete, other.obsolete); + } + // The reorder must have actually changed the bytes, or this proves + // nothing about order-independence. + assert_ne!(reordered.as_str(), TTL); + } + + #[test] + fn rdfxml_reads_the_same_graph_as_turtle() { + const XML: &str = r#" + + + + + + + + + + + +"#; + let nodes = parse_rdf( + XML.as_bytes(), + RdfFormat::RdfXml, + &SPINE, + OBO_RESTRICTION_PREDICATES, + ) + .expect("rdf/xml parses"); + let brain = &nodes[&u(955)]; + assert_eq!(brain.is_a, vec![u(62)]); + assert_eq!(brain.rel, vec![(Predicate::PartOf, u(33))]); + } + + #[test] + fn a_malformed_document_errors_rather_than_returning_an_empty_graph() { + let e = parse_rdf( + b"this is not turtle @@@", + RdfFormat::Turtle, + &SPINE, + OBO_RESTRICTION_PREDICATES, + ); + assert!( + e.is_err(), + "a parse failure must not look like an empty ontology" + ); + } + + #[test] + fn format_detection_prefers_the_extension_then_sniffs() { + assert_eq!(detect_format(Some("ttl"), b">, -) -> HashMap> { +fn ancestor_closure(adj: &HashMap>) -> HashMap> { // Kahn order over the parent graph, then DP bottom-up. // Build reverse (parent -> children) for indegree over child->parent edges. let mut all: Vec = Vec::new(); @@ -309,9 +307,15 @@ mod tests { fn t(sns: Namespace, s: u32, p: Predicate, ons: Namespace, o: u32) -> Triple { Triple { - s: TermId { ns: sns as u8, num: s }, + s: TermId { + ns: sns as u8, + num: s, + }, p, - o: TermId { ns: ons as u8, num: o }, + o: TermId { + ns: ons as u8, + num: o, + }, } } @@ -353,9 +357,27 @@ mod tests { // UBERON:200 is_a UBERON:300 (which is a kind of ...) // ⟹ HP:9 is grounded to UBERON:{200,300} too (2 inferred-beyond-asserted). let tr = vec![ - t(Namespace::Hpo, 9, Predicate::HasAnatomy, Namespace::Uberon, 100), - t(Namespace::Uberon, 100, Predicate::PartOf, Namespace::Uberon, 200), - t(Namespace::Uberon, 200, Predicate::IsA, Namespace::Uberon, 300), + t( + Namespace::Hpo, + 9, + Predicate::HasAnatomy, + Namespace::Uberon, + 100, + ), + t( + Namespace::Uberon, + 100, + Predicate::PartOf, + Namespace::Uberon, + 200, + ), + t( + Namespace::Uberon, + 200, + Predicate::IsA, + Namespace::Uberon, + 300, + ), ]; let s = saturate(&tr); // R∃ fires on BOTH existentials: HP:9→{200,300} (2) AND the part_of diff --git a/crates/ogar-obo/src/registry.rs b/crates/ogar-obo/src/registry.rs new file mode 100644 index 00000000..7bbd12ba --- /dev/null +++ b/crates/ogar-obo/src/registry.rs @@ -0,0 +1,448 @@ +//! # `registry` — the namespace table as **data**, not as five `match` arms. +//! +//! The shipped bake reaches its namespaces through the [`crate::Namespace`] enum, +//! which spreads one namespace's facts across four `match` bodies and an +//! index array ([`crate::Namespace::prefix`], [`crate::Namespace::from_prefix`], +//! [`crate::Namespace::concept_id`], [`crate::Namespace::namespace`]). Adding a sixth +//! namespace means editing all five sites with hand-written literals — the +//! shape that makes a bake un-extendable without a code change per ontology. +//! +//! This module carries the same facts as **one table of rows**. A driver that +//! wants to bake a *different* set of namespaces passes a different +//! [`NsRegistry`]; it does not touch this crate's types. +//! +//! ## The ordinal is the sort key — table ORDER is load-bearing +//! +//! A [`TermId`]'s `ns` field is the row's **index in the registry**, and +//! [`crate::bake`] sorts ids by `(ns, num)`. So the table's order determines +//! the row order of the baked artifact, and therefore its digest. Reordering +//! [`OBO_CORE`] silently re-bakes every downstream slab. +//! +//! [`obo_core_matches_the_shipped_enum`](self) pins the mirror: every row's +//! index and concept id must equal the enum's discriminant and +//! [`crate::Namespace::concept_id`]. That test fails on any reorder or renumber — +//! it is the drift guard that makes a second copy of these facts safe to hold. +//! +//! ## What is NOT here +//! +//! [`crate::classify`] maps a `(subject_ns, object_ns)` pair to an RO +//! [`Predicate`](crate::Predicate). That is *semantics between two named +//! ontologies*, not a per-namespace attribute, so it stays on the enum. A +//! namespace outside the core classifies as +//! [`Predicate::Other`](crate::Predicate::Other) — correct for the upper-level +//! spine ontologies, whose edges are `is_a` skeletons with no cross-ontology +//! predicate to name. + +use crate::TermId; + +/// One namespace row — the config unit. +/// +/// `prefix` is the OBO CURIE prefix as it appears left of the colon +/// (`"MONDO"`, `"HP"`, …). `concept_id` is the canonical hi-u16 that +/// [`crate::Namespace::render_classid`] shifts into the classid. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct NsSpec { + /// The OBO CURIE prefix, matched case-sensitively (OBO prefixes are + /// upper-case by convention and the shipped five are matched exactly). + pub prefix: &'static str, + /// The canonical hi-u16 concept id (`0xDDCC`, domain `DD` · slot `CC`). + pub concept_id: u16, +} + +/// An ordered namespace table. The row **index** is the [`TermId::ns`] +/// ordinal; see the module docs on why order is load-bearing. +#[derive(Debug, Clone, Copy)] +pub struct NsRegistry { + specs: &'static [NsSpec], +} + +impl NsRegistry { + /// Build a registry over a static table. `specs.len()` must be `<= 256` + /// because the ordinal is stored in [`TermId::ns`] (`u8`). + #[must_use] + pub const fn new(specs: &'static [NsSpec]) -> Self { + assert!(specs.len() <= 256, "ordinal must fit TermId::ns (u8)"); + Self { specs } + } + + /// Number of namespaces in the table. + #[must_use] + pub const fn len(&self) -> usize { + self.specs.len() + } + + /// Whether the table is empty. + #[must_use] + pub const fn is_empty(&self) -> bool { + self.specs.is_empty() + } + + /// Every row, in ordinal order. + #[must_use] + pub const fn specs(&self) -> &'static [NsSpec] { + self.specs + } + + /// The ordinal of a CURIE prefix, or `None` if the prefix is not in this + /// table. Linear scan — the tables are tens of rows, and this runs once + /// per CURIE during a parse, never in a query path. + #[must_use] + pub fn ordinal_of(&self, prefix: &str) -> Option { + self.specs + .iter() + .position(|s| s.prefix == prefix) + .map(|i| i as u8) + } + + /// The row at an ordinal. + #[must_use] + pub fn spec(&self, ordinal: u8) -> Option<&'static NsSpec> { + self.specs.get(ordinal as usize) + } + + /// The concept id at an ordinal. + #[must_use] + pub fn concept_id(&self, ordinal: u8) -> Option { + self.spec(ordinal).map(|s| s.concept_id) + } + + /// The full V3 render classid — canon-high `(concept as u32) << 16 | + /// app_prefix`, identical to [`crate::Namespace::render_classid`]. + #[must_use] + pub fn render_classid(&self, ordinal: u8, app_prefix: u16) -> Option { + self.concept_id(ordinal) + .map(|c| (u32::from(c) << 16) | u32::from(app_prefix)) + } + + /// Parse a CURIE against **this** table. Same contract as + /// [`TermId::parse`] (which is this, against [`OBO_CORE`]): `None` if the + /// prefix is absent from the table, the numeric is malformed, or the + /// numeric exceeds 24 bits. + #[must_use] + pub fn parse(&self, curie: &str) -> Option { + let (p, n) = curie.split_once(':')?; + let ns = self.ordinal_of(p)?; + let num: u32 = n.trim().parse().ok()?; + if num > 0x00FF_FFFF { + return None; + } + Some(TermId { ns, num }) + } +} + +/// The **shipped** five-namespace core, in the enum's discriminant order. +/// +/// This table is a second copy of facts the [`crate::Namespace`] enum also holds. +/// It is safe to hold only because +/// [`obo_core_matches_the_shipped_enum`](self) fails the build's test suite +/// the moment the two disagree. +pub const OBO_CORE: NsRegistry = NsRegistry::new(&[ + NsSpec { + prefix: "MONDO", + concept_id: 0x0301, + }, + NsSpec { + prefix: "HP", + concept_id: 0x0302, + }, + NsSpec { + prefix: "UBERON", + concept_id: 0x0303, + }, + NsSpec { + prefix: "PATO", + concept_id: 0x0304, + }, + NsSpec { + prefix: "RO", + concept_id: 0x0305, + }, +]); + +/// The upper-level **meta-study spine** — the `is_a` skeleton a study-level +/// bake grounds on. Same `0x03` Ontology domain as [`OBO_CORE`], under the +/// same local authority its concept-id docs describe. +/// +/// ## Why this starts at `0x0340` — and why it took three attempts +/// +/// Minted at `0x0306..=0x030D`, which collided with +/// `ogar_ro::RELATION_BODY_CONCEPT_ID` (`0x0306`). Moved to +/// `0x0310..=0x0317`, which collided with **four** consumer allocations — +/// OPS `0x0311`, a product lane `0x0313`, the UMLS CUI convergence lane +/// `0x0315`, Orphanet `0x0317`. +/// +/// The second attempt failed for a reason worth naming: the `0x03` domain's +/// occupancy was enumerated **in this repo only**. A private consumer holds an +/// odd-stride run — `0x0307, 09, 0B, 0D, 0F, 11, 13, 15, 17, 19, 1B, 1D` — +/// that no test here can see, and it grows into odd slots, so any nearby block +/// collides eventually rather than immediately. Picking a "clean" block by eye +/// is the failure mode; there is no vantage point in this crate from which the +/// block is verifiably clean. +/// +/// `0x0340..=0x0347` sits clear of both the core band and that run's growth, +/// but **the spacing is a mitigation, not the guard.** The guard has to live +/// where both sides are visible — i.e. in the consumer. The band test below +/// only proves this table is self-consistent and clear of the core; it +/// cannot and does not prove the consumer is clear, and reading it as if it +/// did is what produced the second collision. +/// +/// ## Order is a convention, not a topological sort +/// +/// The rows run roughly upper→applied, by which direction the cross-namespace +/// edges dominate. That is **not** a dependency order in the strict sense: the +/// import graph measured by `examples/probe_spine.rs` is cyclic — e.g. `OBI` +/// and `IAO` reference each other (394 one way, 10 the other), as do `OBI` and +/// `COB` (236 / 3) and `OBI` and `OBCS` (58 / 70). +/// +/// Nothing depends on the order being acyclic: [`crate::bake`] takes the +/// *union* of all namespaces and sorts rows by `(ns, num)`, so a cyclic import +/// graph bakes fine. What the order does fix is the **row order**, and through +/// it the artifact digest — so reordering this table re-bakes everything, +/// exactly as for [`OBO_CORE`]. +/// +/// # ⚠ Gate: this table is NOT bakeable until the numeric-`0` pad collision is resolved +/// +/// [`crate::edges`] packs edge targets into value lanes and uses the numeric +/// `0` as the **pad** meaning *lane spent, advance*. Three of these namespaces +/// contain a real term whose CURIE numeric is `0` — in at least one case the +/// namespace's own root. An edge pointing at such a term is indistinguishable +/// from a pad, so a reader's walk stops early and **silently**. +/// +/// The fix is local to [`crate::edges`] — bias the stored numeric by `+1` +/// (`store = num + 1`, `read = stored - 1`), which costs one value off the top +/// of the range (2²⁴−2 addressable, against a measured maximum well below it). +/// It is not applied here because it moves **every** shipped slab digest and +/// forces a re-bake of all five core tables; that is an operator call, not a +/// side effect of adding a table. +/// +/// Until then: the rows below are config a driver may *read*, and +/// `examples/probe_spine.rs` measures which namespaces actually trip the +/// collision — but a bake over this table would be unsound for those. +pub const META_STUDY_SPINE: NsRegistry = NsRegistry::new(&[ + NsSpec { + prefix: "BFO", + concept_id: 0x0340, + }, + NsSpec { + prefix: "COB", + concept_id: 0x0341, + }, + NsSpec { + prefix: "IAO", + concept_id: 0x0342, + }, + NsSpec { + prefix: "OBI", + concept_id: 0x0343, + }, + NsSpec { + prefix: "OBCS", + concept_id: 0x0344, + }, + NsSpec { + prefix: "SEPIO", + concept_id: 0x0345, + }, + NsSpec { + prefix: "ECO", + concept_id: 0x0346, + }, + NsSpec { + prefix: "FBbi", + concept_id: 0x0347, + }, +]); + +#[cfg(test)] +mod tests { + use super::*; + use crate::Namespace; + + /// The drift guard that makes the duplicate table safe. + /// + /// **What would make it fail:** reordering [`OBO_CORE`], renumbering a + /// concept id, or renaming a prefix. Each of those silently changes the + /// baked row order or the classid of every row in a namespace — i.e. + /// moves the shipped digests — and this test is what refuses it. + #[test] + fn obo_core_matches_the_shipped_enum() { + let enum_order = [ + Namespace::Mondo, + Namespace::Hpo, + Namespace::Uberon, + Namespace::Pato, + Namespace::Ro, + ]; + assert_eq!( + OBO_CORE.len(), + enum_order.len(), + "registry and enum must carry the same namespace count" + ); + for (i, ns) in enum_order.iter().enumerate() { + let ord = i as u8; + assert_eq!( + OBO_CORE.ordinal_of(ns.prefix()), + Some(ord), + "{} must sit at the enum's discriminant — row order is the sort key", + ns.prefix() + ); + assert_eq!( + OBO_CORE.concept_id(ord), + Some(ns.concept_id()), + "{} concept id must mirror the enum", + ns.prefix() + ); + assert_eq!( + OBO_CORE.render_classid(ord, 0x00AB), + Some(ns.render_classid(0x00AB)), + "{} classid must mirror the enum under a non-zero app prefix", + ns.prefix() + ); + } + } + + /// The registry parse must agree with the shipped [`TermId::parse`] on + /// every accept AND every reject — otherwise routing a bake through the + /// table would change which terms are tiled. + #[test] + fn registry_parse_agrees_with_shipped_parse() { + let accept = [ + "MONDO:0007739", + "HP:0000118", + "UBERON:0000955", + "PATO:0000001", + "RO:0002200", + "MONDO:0700092", // above u16 — the rail case + ]; + for c in accept { + assert_eq!( + OBO_CORE.parse(c), + TermId::parse(c), + "{c} must parse identically through the table" + ); + assert!(OBO_CORE.parse(c).is_some(), "{c} must actually parse"); + } + let reject = [ + "CHEBI:12345", // prefix outside the table + "MONDO:99999999", // > 24 bit + "MONDO:not-a-num", // malformed numeric + "nocolon", // no separator + "mondo:0007739", // case differs — OBO prefixes are upper-case + ]; + for c in reject { + assert_eq!( + OBO_CORE.parse(c), + TermId::parse(c), + "{c} must reject identically through the table" + ); + assert!(OBO_CORE.parse(c).is_none(), "{c} must actually reject"); + } + } + + /// A registry over a *different* table resolves namespaces the core does + /// not — the whole point. This is the can-it-fire half: if the table were + /// ignored and the core consulted instead, `BFO:0000002` would not parse + /// and the classid would not be the extended one. + #[test] + fn a_different_table_resolves_namespaces_the_core_cannot() { + const EXTENDED: NsRegistry = NsRegistry::new(&[ + NsSpec { + prefix: "BFO", + concept_id: 0x0340, + }, + NsSpec { + prefix: "MONDO", + concept_id: 0x0301, + }, + ]); + let bfo = EXTENDED.parse("BFO:0000002").expect("BFO parses here"); + assert_eq!(bfo.ns, 0, "BFO is row 0 of this table"); + assert_eq!(bfo.num, 2); + assert_eq!(EXTENDED.render_classid(0, 0x0000), Some(0x0340_0000)); + assert!( + TermId::parse("BFO:0000002").is_none(), + "the shipped core must NOT resolve BFO — otherwise this proves nothing" + ); + + // Ordinals are table-relative: MONDO is row 1 here, row 0 in the core. + // A TermId is therefore only meaningful WITH its registry. + assert_eq!(EXTENDED.parse("MONDO:0007739").map(|t| t.ns), Some(1)); + assert_eq!(TermId::parse("MONDO:0007739").map(|t| t.ns), Some(0)); + } + + /// The guard the 2026-08-08 audit had to be run by hand to find. + /// + /// `META_STUDY_SPINE` was minted at `0x0306..=0x030D`, and `0x0306` is + /// `ogar_ro::RELATION_BODY_CONCEPT_ID` — live, and already consumed + /// downstream. Two concepts at one address is not a near-miss: a baked BFO + /// row and an `ogar-ro` relation-body row become the same classid. + /// + /// This crate cannot see `ogar-ro` (that crate deps *this* one, so the + /// dependency only runs one way), so the band is asserted instead of the + /// constant: everything below `SPINE_BAND_START` belongs to the core + /// allocations, and the spine must stay above it. `ogar-ro` carries the + /// mirror guard against its own real constant. + /// + /// **What would make it fail:** minting a spine row inside the reserved + /// band, or giving two tables the same id. + #[test] + fn spine_does_not_collide_with_the_core_or_the_reserved_band() { + /// First id of the spine block. Everything below belongs to the core + /// band or to consumer allocations this crate cannot see. + const SPINE_BAND_START: u16 = 0x0340; + + for s in META_STUDY_SPINE.specs() { + assert!( + s.concept_id >= SPINE_BAND_START, + "{} at {:#06X} sits below the spine block — that range holds the \ + core namespaces, ogar-ro (0x0306), and consumer allocations \ + this crate cannot see", + s.prefix, + s.concept_id + ); + } + for c in OBO_CORE.specs() { + assert!( + c.concept_id < SPINE_BAND_START, + "{} at {:#06X} escaped the reserved core band", + c.prefix, + c.concept_id + ); + } + + // No id twice, across BOTH tables — the general form of the bug. + let mut ids: Vec<(u16, &str)> = OBO_CORE + .specs() + .iter() + .chain(META_STUDY_SPINE.specs()) + .map(|s| (s.concept_id, s.prefix)) + .collect(); + ids.sort_unstable(); + for w in ids.windows(2) { + assert_ne!( + w[0].0, w[1].0, + "{} and {} both claim {:#06X}", + w[0].1, w[1].1, w[0].0 + ); + } + + // Every id stays inside the 0x03 Ontology domain — a spine that ran + // off the end of the block would silently land in another domain. + for s in OBO_CORE.specs().iter().chain(META_STUDY_SPINE.specs()) { + assert_eq!( + s.concept_id >> 8, + 0x03, + "{} at {:#06X} left the 0x03 Ontology domain", + s.prefix, + s.concept_id + ); + } + } + + #[test] + fn unknown_ordinal_resolves_to_none_rather_than_panicking() { + assert_eq!(OBO_CORE.spec(9), None); + assert_eq!(OBO_CORE.concept_id(9), None); + assert_eq!(OBO_CORE.render_classid(9, 0), None); + } +} diff --git a/crates/ogar-obo/src/spine.rs b/crates/ogar-obo/src/spine.rs new file mode 100644 index 00000000..fa3378a7 --- /dev/null +++ b/crates/ogar-obo/src/spine.rs @@ -0,0 +1,375 @@ +//! `SpineLens` — a borrowed view that answers "who are this class's parents?" +//! straight out of baked rows, with no adjacency index built. +//! +//! # Why this exists rather than an adjacency copy +//! +//! The obvious way to traverse a baked graph is to read every edge once into an +//! adjacency structure and walk that. Measured, for this substrate, that is the +//! expensive option: `lance-graph`'s `GrBMatrix` carries a `Vec` of +//! 16 384-bit values — ~2 KiB **per edge** — so materialising the 107 813 `is_a` +//! edges as a matrix costs on the order of 200 MB to hold an adjacency the rows +//! already contain. That is the opposite of the deployment's whole shape (a +//! large hydrated table, a small resident set), and it is also what the +//! workspace's zero-copy law forbids on principle rather than on cost. +//! +//! So the lens does not build anything. It borrows the rows and answers each +//! question by locating a row and decoding its lanes in place. +//! +//! # The two steps, and what each costs +//! +//! 1. **Resolve** — `(classid, num)` → row index, by binary search. Sound +//! because the bake emits rows in ascending `(classid, num)`: `family` holds +//! the high half of the numeric, so ordering by `(classid, family, identity)` +//! *is* ordering by `(classid, num)`. Verified over the shipped bake — all +//! 68 797 rows strictly ascending, zero inversions. ~17 probes at that size. +//! 2. **Walk** — decode that row's `is_a` links from its edge lanes +//! ([`crate::edges::EdgeLanes`]), a shift per link, nothing allocated. +//! +//! Against an indexed closure the trade is explicit: a `HashMap` gives O(1) +//! hops after paying a full pass over every edge and holding a map proportional +//! to them; the lens pays O(log n) per hop and holds nothing. Which wins depends +//! on how many questions are asked against how much memory is available, and +//! `examples/elk_lens_vs_closure.rs` measures it rather than asserting it. +//! +//! # It borrows, and it does not cache +//! +//! One field, a shared slice, in the house style of the workspace's other +//! lenses. No memo table: a transitive closure recomputes equal, which under +//! the zero-copy law makes it a projection, and a projection is never stored +//! however much work produced it. Asking twice walks twice, deliberately. + +use crate::{NODE_ROW_STRIDE, Predicate, Row512, edges::EdgeLanes, unpack_key}; + +/// A borrowed view over baked rows that resolves addresses and reads parents. +/// +/// Holds a shared slice and nothing else. Constructing one is free; it is meant +/// to be made where it is used rather than kept. +#[derive(Clone, Copy)] +pub struct SpineLens<'a> { + rows: &'a [Row512], +} + +impl core::fmt::Debug for SpineLens<'_> { + /// Prints the row COUNT, never the rows. A lens over a hydrated table can + /// borrow tens of megabytes; a derived `Debug` would render all of it. + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("SpineLens") + .field("rows", &self.rows.len()) + .finish() + } +} + +impl<'a> SpineLens<'a> { + /// Borrow a slice of baked rows. + /// + /// The rows **must** be in the bake's own order — ascending `(classid, + /// num)` — because [`Self::resolve`] binary-searches them. Handing over an + /// unsorted slice does not panic; it silently fails to find rows, which is + /// why [`Self::is_sorted`] exists and why the example asserts it before use. + #[must_use] + pub const fn new(rows: &'a [Row512]) -> Self { + Self { rows } + } + + /// Number of rows in view. + #[must_use] + pub const fn len(&self) -> usize { + self.rows.len() + } + + /// Whether the view is empty. + #[must_use] + pub const fn is_empty(&self) -> bool { + self.rows.is_empty() + } + + /// The `(classid, num)` address of row `i`, or `None` if out of range. + #[must_use] + pub fn address_at(&self, i: usize) -> Option<(u32, u32)> { + self.rows.get(i).map(|r| unpack_key(&r.0)) + } + + /// Check the precondition [`Self::resolve`] relies on: strictly ascending + /// `(classid, num)`. + /// + /// O(n) and therefore not called per query — it is for a caller to assert + /// once at the boundary where rows arrive, so an unsorted slice fails + /// loudly there instead of returning empty answers everywhere else. + #[must_use] + pub fn is_sorted(&self) -> bool { + self.rows + .windows(2) + .all(|w| unpack_key(&w[0].0) < unpack_key(&w[1].0)) + } + + /// Locate the row for an address by binary search. + /// + /// `None` means no such row is in view — which is a real and expected + /// answer, not an error: an edge may point at a term outside the loaded + /// set, and the walk simply stops there rather than inventing a node. + #[must_use] + pub fn resolve(&self, classid: u32, num: u32) -> Option { + self.rows + .binary_search_by(|r| unpack_key(&r.0).cmp(&(classid, num))) + .ok() + } + + /// This row's out-degree for one predicate — the CSR `row_ptr` entry, read + /// from the edge block with **no value decode at all**. + #[must_use] + pub fn degree_at(&self, i: usize, p: Predicate) -> u8 { + self.rows + .get(i) + .map_or(0, |r| EdgeLanes::new(&r.0).degree(p)) + } + + /// Visit every link of one predicate for the row at `i`, decoded in place. + pub fn links_at(&self, i: usize, p: Predicate, visit: &mut dyn FnMut(u32, u32)) { + let Some(row) = self.rows.get(i) else { return }; + for l in EdgeLanes::new(&row.0).links_of(p) { + visit(l.classid, l.num); + } + } + + /// Visit the **asserted superclasses** of an address: resolve, then walk + /// that row's `is_a` links. + /// + /// `is_a` only, and that is the safety rather than a convenience — walking + /// subsumption together with `part_of` derives false ancestors (`A part_of + /// B` and `B ⊑ C` does not give `A ⊑ C`), and a reasoner without role + /// composition must never be handed the mixed set. + /// + /// An address with no row in view yields nothing, silently: an edge into an + /// unloaded ontology is a boundary, not a fault. + pub fn parents_of(&self, classid: u32, num: u32, visit: &mut dyn FnMut(u32, u32)) { + if let Some(i) = self.resolve(classid, num) { + self.links_at(i, Predicate::IsA, visit); + } + } + + /// The **grounding spine** of an address: `is_a` ∪ `part_of` parents. + /// + /// This is the spine an existential FILLER generalizes up (R∃), and it is + /// deliberately NOT [`Self::parents_of`]. An anatomical filler's grounding + /// is inherited through mereology as well as taxonomy — the deductive form + /// of "inherit grounding up the anatomy spine". + /// + /// **Never hand this to a subsumption reasoner.** Generalizing a *subject* + /// through `part_of` is the false-ancestor error (`A part_of B`, `B ⊑ C` + /// does not give `A ⊑ C`); generalizing a *filler* through it is sound. + /// [`Self::parents_of`] stays `is_a`-only for exactly that reason, and the + /// two are separate methods so a caller must choose on purpose. + pub fn grounding_parents_of(&self, classid: u32, num: u32, visit: &mut dyn FnMut(u32, u32)) { + if let Some(i) = self.resolve(classid, num) { + self.links_at(i, Predicate::IsA, visit); + self.links_at(i, Predicate::PartOf, visit); + } + } + + /// The asserted existential fillers of one role on an address. + /// + /// A thin typed read — the R∃ generalization is the reasoner's job, not + /// this one's. An address with no row in view yields nothing, silently. + pub fn fillers_of( + &self, + classid: u32, + num: u32, + role: Predicate, + visit: &mut dyn FnMut(u32, u32), + ) { + if let Some(i) = self.resolve(classid, num) { + self.links_at(i, role, visit); + } + } +} + +/// Reinterpret a byte buffer as rows, if it is correctly sized and aligned. +/// +/// Returns `None` when the length is not a whole number of rows or the buffer +/// is not 64-aligned — the same gate `rows_from_le_bytes` applies, restated +/// here so a lens caller has one call rather than two. +#[must_use] +pub fn lens_from_le_bytes(bytes: &[u8]) -> Option> { + if !bytes.len().is_multiple_of(NODE_ROW_STRIDE) { + return None; + } + crate::rows_from_le_bytes(bytes).map(SpineLens::new) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{Namespace, TermId, edges::pack_edges}; + + const MONDO: u32 = 0x0301_0000; + + fn row(num: u32, parents: &[u32]) -> Row512 { + let mut r = Row512::zeroed(); + r.0[0..16].copy_from_slice(&crate::pack_key(MONDO, num)); + r.0[crate::EDGES_OFFSET + Predicate::IsA as usize] = parents.len() as u8; + let mut tg: Vec<_> = parents + .iter() + .map(|p| { + ( + Predicate::IsA, + TermId { + ns: Namespace::Mondo as u8, + num: *p, + }, + ) + }) + .collect(); + pack_edges(&mut r.0, &mut tg, 0x0000, &crate::registry::OBO_CORE); + r + } + + /// 1 -> 2 -> 3, plus an isolated 9. + fn fixture() -> Vec { + vec![row(1, &[2]), row(2, &[3]), row(3, &[]), row(9, &[])] + } + + #[test] + fn the_grounding_spine_includes_part_of_and_the_subsumption_one_does_not() { + // one row with BOTH an is_a and a part_of parent + let mut r = Row512::zeroed(); + r.0[0..16].copy_from_slice(&crate::pack_key(MONDO, 1)); + r.0[crate::EDGES_OFFSET + Predicate::IsA as usize] = 1; + r.0[crate::EDGES_OFFSET + Predicate::PartOf as usize] = 1; + let mut tg = vec![ + ( + Predicate::IsA, + TermId { + ns: Namespace::Mondo as u8, + num: 2, + }, + ), + ( + Predicate::PartOf, + TermId { + ns: Namespace::Mondo as u8, + num: 3, + }, + ), + ]; + pack_edges(&mut r.0, &mut tg, 0x0000, &crate::registry::OBO_CORE); + let rows = vec![r]; + let lens = SpineLens::new(&rows); + + let mut subsumption = Vec::new(); + lens.parents_of(MONDO, 1, &mut |_, n| subsumption.push(n)); + assert_eq!( + subsumption, + vec![2], + "part_of must NOT reach a subsumption walk" + ); + + let mut grounding = Vec::new(); + lens.grounding_parents_of(MONDO, 1, &mut |_, n| grounding.push(n)); + assert_eq!( + grounding, + vec![2, 3], + "the grounding spine is is_a UNION part_of" + ); + } + + #[test] + fn fillers_are_read_per_role_and_can_be_empty() { + let mut r = Row512::zeroed(); + r.0[0..16].copy_from_slice(&crate::pack_key(MONDO, 1)); + r.0[crate::EDGES_OFFSET + Predicate::HasPhenotype as usize] = 1; + let mut tg = vec![( + Predicate::HasPhenotype, + TermId { + ns: Namespace::Hpo as u8, + num: 42, + }, + )]; + pack_edges(&mut r.0, &mut tg, 0x0000, &crate::registry::OBO_CORE); + let rows = vec![r]; + let lens = SpineLens::new(&rows); + + let mut got = Vec::new(); + lens.fillers_of(MONDO, 1, Predicate::HasPhenotype, &mut |c, n| { + got.push((c, n)) + }); + assert_eq!(got, vec![(0x0302_0000, 42)], "crosses into HPO"); + + // can-stay-silent: a role with no asserted filler yields nothing + got.clear(); + lens.fillers_of(MONDO, 1, Predicate::HasAnatomy, &mut |c, n| { + got.push((c, n)) + }); + assert!( + got.is_empty(), + "an unasserted role must not manufacture a filler" + ); + } + + #[test] + fn resolve_finds_every_row_and_misses_absent_ones() { + let rows = fixture(); + let lens = SpineLens::new(&rows); + assert!(lens.is_sorted()); + for (i, n) in [1u32, 2, 3, 9].iter().enumerate() { + assert_eq!(lens.resolve(MONDO, *n), Some(i)); + } + // anti-vacuity: resolve must be able to say no + assert_eq!(lens.resolve(MONDO, 4), None, "absent numeric"); + assert_eq!( + lens.resolve(0x0302_0000, 1), + None, + "right numeric, wrong class" + ); + } + + #[test] + fn parents_come_from_the_lanes_and_stop_at_the_boundary() { + let rows = fixture(); + let lens = SpineLens::new(&rows); + let mut got = Vec::new(); + lens.parents_of(MONDO, 1, &mut |c, n| got.push((c, n))); + assert_eq!(got, vec![(MONDO, 2)]); + + got.clear(); + lens.parents_of(MONDO, 3, &mut |c, n| got.push((c, n))); + assert!(got.is_empty(), "a root has no parents"); + + got.clear(); + lens.parents_of(MONDO, 404, &mut |c, n| got.push((c, n))); + assert!( + got.is_empty(), + "an unresolvable address yields nothing, silently" + ); + } + + #[test] + fn degree_reads_without_decoding_any_value() { + let rows = fixture(); + let lens = SpineLens::new(&rows); + assert_eq!(lens.degree_at(0, Predicate::IsA), 1); + assert_eq!(lens.degree_at(2, Predicate::IsA), 0); + assert_eq!(lens.degree_at(0, Predicate::PartOf), 0); + } + + #[test] + fn is_sorted_actually_rejects_an_unsorted_slice() { + // the guard must be able to fail, or it is decoration + let rows = vec![row(3, &[]), row(1, &[2])]; + assert!(!SpineLens::new(&rows).is_sorted()); + } + + /// The lens drives the reasoner with no adapter type in between — the + /// property that keeps ogar-obo and ogar-elk free of a dependency edge. + #[test] + fn the_lens_composes_with_a_visitor_closure() { + let rows = fixture(); + let lens = SpineLens::new(&rows); + // the shape `ogar_elk::Parents` is blanket-implemented for + let f = |addr: (u32, u32), visit: &mut dyn FnMut((u32, u32))| { + lens.parents_of(addr.0, addr.1, &mut |c, n| visit((c, n))); + }; + let mut seen = Vec::new(); + f((MONDO, 1), &mut |a| seen.push(a)); + assert_eq!(seen, vec![(MONDO, 2)]); + } +} diff --git a/crates/ogar-ro/src/lib.rs b/crates/ogar-ro/src/lib.rs index 6725b51a..39c73abb 100644 --- a/crates/ogar-ro/src/lib.rs +++ b/crates/ogar-ro/src/lib.rs @@ -328,3 +328,44 @@ mod tests { assert_eq!(id & 0xFFFF, 0x1000); } } + +#[cfg(test)] +mod concept_id_collision_guard { + use super::RELATION_BODY_CONCEPT_ID; + use ogar_obo::registry::{META_STUDY_SPINE, OBO_CORE}; + + /// The mirror of `ogar_obo::registry`'s band guard, pointing the other way. + /// + /// `ogar-obo` cannot see this crate (the dependency runs `ogar-ro` → + /// `ogar-obo`), so it can only assert that its tables stay out of the + /// reserved band. This side asserts the thing that actually matters: our + /// real constant is claimed by no `ogar-obo` table. + /// + /// It exists because the reverse DID happen — `META_STUDY_SPINE` was + /// minted over `0x0306` on 2026-08-08 and nothing failed. The collision + /// was found by hand-enumerating the `0x03` domain during an unrelated + /// audit, which is not a process that scales. + /// + /// **What would make it fail:** any `ogar-obo` table growing a row at + /// `RELATION_BODY_CONCEPT_ID`. + #[test] + fn no_ogar_obo_namespace_claims_our_concept_id() { + for s in OBO_CORE.specs().iter().chain(META_STUDY_SPINE.specs()) { + assert_ne!( + s.concept_id, RELATION_BODY_CONCEPT_ID, + "ogar-obo namespace {} claims {:#06X}, which is \ + RELATION_BODY_CONCEPT_ID — a baked {} row and an ogar-ro \ + relation-body row would be the same classid", + s.prefix, RELATION_BODY_CONCEPT_ID, s.prefix + ); + } + } + + /// Anti-vacuity: the guard above compares against tables that are actually + /// populated. A pair of empty tables would pass it while proving nothing. + #[test] + fn the_tables_the_guard_checks_are_non_empty() { + assert!(OBO_CORE.len() >= 5, "core table must be populated"); + assert!(META_STUDY_SPINE.len() >= 8, "spine table must be populated"); + } +}