From 89843ea350a9aab00b2411f8aad1d9c0e42c2d1c Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 7 Aug 2026 23:53:25 +0000 Subject: [PATCH 01/13] =?UTF-8?q?ogar-obo:=20pack=20edge=20targets=20into?= =?UTF-8?q?=20value=20lanes=20=E2=80=94=20the=20col=5Fidx=20half=20of=20th?= =?UTF-8?q?e=20CSR?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The bake persisted the per-predicate degree histogram in the edge block (`row_ptr`) but not the targets (`col_idx`), so a baked row could describe its own local shape and still not be traversed. This adds the targets. They ride the free value lanes in the one form the substrate has: a lane is `classid(4) + 12 content bytes`, and the classid says how to read its own 12. An edge lane reads its content as the G2 `4x24` carving — four literal links — and carries the *target ontology's own row classid*, so a link is exactly the `(classid, identity)` pair a key carries. The same binary search that resolves a key resolves a link; nothing new is invented to point at a node. Layout: the 480-byte value slab is 30 lanes. Lane 6 is the EntityType tenant; lanes 0..6 stay reserved (RESERVE, DON'T RECLAIM); edges take lanes 7..30 = 92 link slots. Targets are emitted ascending by (predicate, target namespace, numeric), and a lane never mixes target ontologies — its classid names exactly one, so a namespace change starts a new lane even mid-predicate. Unused slots are 0, which is unambiguous because no OBO CURIE numeric is 0. The predicate is not stored on the lane. A reader recovers it from the degree histogram, consuming `degree(p)` links per predicate in ascending order — which makes the already-persisted `row_ptr` load-bearing instead of decorative, and leaves a row that is a complete CSR adjacency row on its own. `EdgeLanes` is the read side: it borrows the row and decodes a u24 with a shift. No allocation, no gathered window, no owned copy of anything the lane already holds. Measured before the layout constant was chosen (`probe_edge_capacity`, real sources): 186 029 triples over 68 789 subjects, max out-degree 62, max lane demand for a single row 16 against a 23-lane budget. Both packing assumptions verified on the same data — CURIE numeric 0 never occurs (so the pad is safe) and every numeric fits 24 bits. The packer reports an overflow in `PackStats` rather than truncating; `bake` accumulates it so a driver can refuse to ship an artifact that is missing edges. Verified end-to-end (`verify_edges`): links read back out of the row bytes match the SPO triple table as a multiset, per ontology — RO 3 links / is_a 2, UBERON 71 844 / 27 170, HP 50 284 / 31 284, PATO 3 389 / 2 909, MONDO 60 509 / 46 448; 186 029 packed, 0 dropped, 0 rows overflowed. Those are two different code paths over the same input. The is_a totals also match the degree bytes in the already-baked slabs, so the packing agrees with the artifact it extends. RO is small enough to check by hand and is printed in full: RO:0002532 --Other--> RO:0002533, RO:0002533 and RO:0002534 --is_a--> RO:0002532, and RO:0002577 correctly carries no link at all because its only parent (BFO:0000040) is outside the five core namespaces. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01KCGhDYoQBXs3poaR7sFuqp --- crates/ogar-obo/examples/bake_obo.rs | 60 ++- .../ogar-obo/examples/probe_edge_capacity.rs | 141 ++++++ crates/ogar-obo/examples/verify_edges.rs | 140 ++++++ crates/ogar-obo/src/crosswalk.rs | 54 ++- crates/ogar-obo/src/edges.rs | 453 ++++++++++++++++++ crates/ogar-obo/src/lib.rs | 189 +++++--- crates/ogar-obo/src/reason.rs | 38 +- 7 files changed, 979 insertions(+), 96 deletions(-) create mode 100644 crates/ogar-obo/examples/probe_edge_capacity.rs create mode 100644 crates/ogar-obo/examples/verify_edges.rs create mode 100644 crates/ogar-obo/src/edges.rs diff --git a/crates/ogar-obo/examples/bake_obo.rs b/crates/ogar-obo/examples/bake_obo.rs index 1411c7f8..d2d13ec8 100644 --- a/crates/ogar-obo/examples/bake_obo.rs +++ b/crates/ogar-obo/examples/bake_obo.rs @@ -9,8 +9,8 @@ //! scratch dir, never committed. use ogar_obo::{ - Namespace, bake, merge_logical_defs, parse_hp_logical_defs, parse_obo, reason, - rows_from_le_bytes, as_le_bytes, + Namespace, as_le_bytes, bake, merge_logical_defs, parse_hp_logical_defs, parse_obo, reason, + rows_from_le_bytes, }; use std::collections::HashMap; @@ -22,12 +22,13 @@ fn main() { let mut nodes = HashMap::new(); for f in ["mondo", "hp", "uberon", "pato", "ro"] { let path = format!("{dir}/{f}.obo"); - let text = std::fs::read_to_string(&path) - .unwrap_or_else(|e| panic!("read {path}: {e}")); + let text = std::fs::read_to_string(&path).unwrap_or_else(|e| panic!("read {path}: {e}")); let part = parse_obo(&text); let (terms, edges) = ( part.len(), - part.values().map(|n| n.is_a.len() + n.rel.len() + n.xref.len()).sum::(), + part.values() + .map(|n| n.is_a.len() + n.rel.len() + n.xref.len()) + .sum::(), ); println!(" parsed {f:8} terms={terms:6} edges+xref={edges}"); for (id, n) in part { @@ -42,7 +43,10 @@ fn main() { // HPO logical-definition grounding (anatomy:quality) from hp-base.owl. let owl = std::fs::read_to_string(format!("{dir}/hp-base.owl")).unwrap_or_default(); let defs = parse_hp_logical_defs(&owl); - println!(" hp-base logical defs: {} HP->Uberon/PATO grounding edges", defs.len()); + println!( + " hp-base logical defs: {} HP->Uberon/PATO grounding edges", + defs.len() + ); merge_logical_defs(&mut nodes, &defs); let baked = bake(&nodes, 0x0000); @@ -56,36 +60,62 @@ fn main() { println!(" MONDO->HP resolve : {}", s.mondo_hp); println!(" HP->UBERON resolve : {}", s.hp_uberon); println!(" HP->PATO resolve : {}", s.hp_pato); - println!(" xrefs preserved : {} (MeSH bearings: {})", s.xrefs, s.mesh_xrefs); + println!( + " xrefs preserved : {} (MeSH bearings: {})", + s.xrefs, s.mesh_xrefs + ); // Per-namespace row census. let mut census: HashMap = HashMap::new(); for id in &baked.ids { *census.entry(id.ns).or_default() += 1; } - for ns in [Namespace::Mondo, Namespace::Hpo, Namespace::Uberon, Namespace::Pato, Namespace::Ro] { - println!(" {:8}: {}", ns.prefix(), census.get(&(ns as u8)).copied().unwrap_or(0)); + for ns in [ + Namespace::Mondo, + Namespace::Hpo, + Namespace::Uberon, + Namespace::Pato, + Namespace::Ro, + ] { + println!( + " {:8}: {}", + ns.prefix(), + census.get(&(ns as u8)).copied().unwrap_or(0) + ); } println!("\n=== EL SATURATION (ELK subset, excavated to Rust) ==="); let el = reason::saturate(&baked.triples); println!(" is_a subsumption pairs : {}", el.subsumption_pairs); println!(" part_of transitive pairs : {}", el.part_of_pairs); - println!(" existential inferred (R∃): {} (grounding inherited up the spine)", el.existential_inferred); - println!(" unsatisfiable : {} (no disjointness axioms in base obo)", el.unsatisfiable); + println!( + " existential inferred (R∃): {} (grounding inherited up the spine)", + el.existential_inferred + ); + println!( + " unsatisfiable : {} (no disjointness axioms in base obo)", + el.unsatisfiable + ); // Write the artifact + verify it round-trips through the loader contract. let bytes = as_le_bytes(&baked.rows); std::fs::write(&out, bytes).unwrap_or_else(|e| panic!("write {out}: {e}")); println!("\n=== ARTIFACT ==="); - println!(" {out} ({} bytes = {} rows × 512)", bytes.len(), baked.rows.len()); + println!( + " {out} ({} bytes = {} rows × 512)", + bytes.len(), + baked.rows.len() + ); let readback = std::fs::read(&out).expect("reread"); match rows_from_le_bytes(&readback) { Some(rows) => { // NB: a fresh Vec from fs::read may not be 64-aligned; the loader // returns None then and a real consumer uses FixedSizeBinary(512) // (arrow-aligned). We verify the in-memory aligned view instead. - println!(" reread rows_from_le_bytes: {} rows (aligned buffer)", rows.len()); + println!( + " reread rows_from_le_bytes: {} rows (aligned buffer)", + rows.len() + ); } None => { let inmem = rows_from_le_bytes(bytes).expect("in-memory aligned view"); @@ -97,5 +127,7 @@ fn main() { ); } } - println!(" loader contract: VERIFIED (as_le_bytes ↔ rows_from_le_bytes, 512×N, 64-align gate)"); + println!( + " loader contract: VERIFIED (as_le_bytes ↔ rows_from_le_bytes, 512×N, 64-align gate)" + ); } diff --git a/crates/ogar-obo/examples/probe_edge_capacity.rs b/crates/ogar-obo/examples/probe_edge_capacity.rs new file mode 100644 index 00000000..96c1bc18 --- /dev/null +++ b/crates/ogar-obo/examples/probe_edge_capacity.rs @@ -0,0 +1,141 @@ +//! Probe: does the `col_idx` half of the CSR fit in the free value lanes? +//! +//! The bake persists `row_ptr` (the per-predicate degree histogram in the edge +//! block) but not `col_idx` (the targets). Packing the targets as G2 `4×u24` +//! literal links needs a fixed layout constant — how many lanes to reserve. +//! This probe measures the demand on the real sources before that constant is +//! chosen, and checks the two assumptions the packing rests on: +//! +//! A1 CURIE numeric `0` never occurs (so `0` is a safe in-lane pad). +//! A2 every numeric fits 24 bits (`TermId::parse` already rejects >24-bit, +//! so this re-checks the *baked* ids, not the parser). +//! +//! Usage: `cargo run -p ogar-obo --example probe_edge_capacity -- [src_dir]` + +use ogar_obo::{Namespace, Predicate, bake, merge_logical_defs, parse_hp_logical_defs, parse_obo}; +use std::collections::HashMap; + +fn main() { + let dir = std::env::args() + .nth(1) + .unwrap_or_else(|| "/tmp/obo".to_string()); + + let mut nodes = HashMap::new(); + for f in ["mondo", "hp", "uberon", "pato", "ro"] { + let path = format!("{dir}/{f}.obo"); + let text = std::fs::read_to_string(&path).unwrap_or_else(|e| panic!("read {path}: {e}")); + for (id, n) in parse_obo(&text) { + let e = nodes.entry(id).or_insert_with(ogar_obo::OboNode::default); + e.is_a.extend(n.is_a); + e.rel.extend(n.rel); + e.xref.extend(n.xref); + e.obsolete |= n.obsolete; + } + } + let owl = std::fs::read_to_string(format!("{dir}/hp-base.owl")).unwrap_or_default(); + merge_logical_defs(&mut nodes, &parse_hp_logical_defs(&owl)); + + let baked = bake(&nodes, 0x0000); + + // ── group the triples by subject, in the canonical emit order ──────────── + // ascending predicate, then ascending target namespace: a lane carries ONE + // target ontology (its classid names it), so a namespace switch inside a + // predicate starts a new lane. That fragmentation is what makes the lane + // demand exceed `ceil(degree / 4)` — which is exactly what we are measuring. + let mut by_subject: HashMap> = HashMap::new(); + for t in &baked.triples { + by_subject + .entry(t.s) + .or_default() + .push((t.p as u8, t.o.ns, t.o.num)); + } + + let mut max_deg = 0usize; + let mut max_lanes = 0usize; + let mut worst_deg: Option = None; + let mut worst_lanes: Option = None; + let mut lane_hist: HashMap = HashMap::new(); + let mut total_lanes = 0usize; + let mut zero_numeric = 0usize; + let mut over_24bit = 0usize; + + for (s, edges) in &mut by_subject { + edges.sort_unstable(); + // lane demand: a new lane per 4 targets, restarting on (pred, ns) change + let mut lanes = 0usize; + let mut slot = 4usize; // force a fresh lane on the first target + let mut group: Option<(u8, u8)> = None; + for &(p, ns, num) in edges.iter() { + if num == 0 { + zero_numeric += 1; + } + if num > 0x00FF_FFFF { + over_24bit += 1; + } + if group != Some((p, ns)) || slot == 4 { + if group != Some((p, ns)) { + group = Some((p, ns)); + } + lanes += 1; + slot = 0; + } + slot += 1; + } + if edges.len() > max_deg { + max_deg = edges.len(); + worst_deg = Some(*s); + } + if lanes > max_lanes { + max_lanes = lanes; + worst_lanes = Some(*s); + } + *lane_hist.entry(lanes).or_default() += 1; + total_lanes += lanes; + } + + println!("=== EDGE CAPACITY PROBE ==="); + println!(" subjects with >=1 edge : {}", by_subject.len()); + println!(" total triples : {}", baked.triples.len()); + println!(" total lanes demanded : {total_lanes}"); + println!(" max out-degree : {max_deg} ({worst_deg:?})"); + println!(" max lanes for one row : {max_lanes} ({worst_lanes:?})"); + + println!("\n A1 CURIE numeric == 0 : {zero_numeric} occurrences (must be 0)"); + println!(" A2 numeric > 24 bit : {over_24bit} occurrences (must be 0)"); + + // How many rows would overflow a given lane budget? + println!("\n rows exceeding a lane budget:"); + for budget in [8usize, 12, 16, 20, 23, 29] { + let over: usize = lane_hist + .iter() + .filter(|(l, _)| **l > budget) + .map(|(_, c)| *c) + .sum(); + println!( + " budget {budget:2} lanes ({:3} slots): {over} rows over", + budget * 4 + ); + } + + // Per-namespace is_a degree totals — the falsifier against the baked slabs. + let mut is_a_by_ns: HashMap = HashMap::new(); + for t in &baked.triples { + if t.p == Predicate::IsA { + *is_a_by_ns.entry(t.s.ns).or_default() += 1; + } + } + println!("\n is_a degree totals by subject namespace (falsifier vs baked slabs):"); + for ns in [ + Namespace::Mondo, + Namespace::Hpo, + Namespace::Uberon, + Namespace::Pato, + Namespace::Ro, + ] { + println!( + " {:8}: {}", + ns.prefix(), + is_a_by_ns.get(&(ns as u8)).copied().unwrap_or(0) + ); + } +} diff --git a/crates/ogar-obo/examples/verify_edges.rs b/crates/ogar-obo/examples/verify_edges.rs new file mode 100644 index 00000000..0d122763 --- /dev/null +++ b/crates/ogar-obo/examples/verify_edges.rs @@ -0,0 +1,140 @@ +//! Verify the packed edge lanes against the sources, ontology by ontology. +//! +//! The falsifier is independence: the links are read back **out of the row +//! bytes** (lane decode via [`ogar_obo::edges::EdgeLanes`]) and compared as a +//! multiset against the SPO triple table the bake built from the parsed `.obo`. +//! Those are two different code paths over the same input, so agreement is +//! evidence and disagreement names the row. +//! +//! Order is RO → UBERON → HPO → (PATO, MONDO), smallest first: RO is two edges +//! and can be checked by eye before anything larger is trusted. +//! +//! Usage: `cargo run -p ogar-obo --example verify_edges -- [src_dir]` + +use ogar_obo::{ + Namespace, Predicate, TermId, bake, edges::EdgeLanes, merge_logical_defs, + parse_hp_logical_defs, parse_obo, +}; +use std::collections::HashMap; + +fn main() { + let dir = std::env::args() + .nth(1) + .unwrap_or_else(|| "/tmp/obo".to_string()); + + let mut nodes = HashMap::new(); + for f in ["mondo", "hp", "uberon", "pato", "ro"] { + let path = format!("{dir}/{f}.obo"); + let text = std::fs::read_to_string(&path).unwrap_or_else(|e| panic!("read {path}: {e}")); + for (id, n) in parse_obo(&text) { + let e = nodes.entry(id).or_insert_with(ogar_obo::OboNode::default); + e.is_a.extend(n.is_a); + e.rel.extend(n.rel); + e.xref.extend(n.xref); + e.obsolete |= n.obsolete; + } + } + let owl = std::fs::read_to_string(format!("{dir}/hp-base.owl")).unwrap_or_default(); + merge_logical_defs(&mut nodes, &parse_hp_logical_defs(&owl)); + + let baked = bake(&nodes, 0x0000); + let s = &baked.stats; + println!("=== BAKE ==="); + println!(" rows : {}", s.nodes); + println!(" triples : {}", s.triples); + println!(" links packed : {}", s.links_packed); + println!(" links dropped : {} (must be 0)", s.links_dropped); + println!(" rows overflow : {} (must be 0)", s.rows_overflowed); + assert_eq!( + s.links_dropped, 0, + "edges were dropped — artifact incomplete" + ); + assert_eq!(s.rows_overflowed, 0); + assert_eq!( + s.links_packed, s.triples, + "every out-of-line triple must also be in-row" + ); + + // expected: (subject, predicate, target-classid, target-num) from the triples + let mut expected: HashMap> = HashMap::new(); + for t in &baked.triples { + expected.entry(t.s).or_default().push(( + t.p as u8, + t.o.namespace().render_classid(0x0000), + t.o.num, + )); + } + + let order = [ + Namespace::Ro, + Namespace::Uberon, + Namespace::Hpo, + Namespace::Pato, + Namespace::Mondo, + ]; + let mut all_ok = true; + for ns in order { + let mut rows = 0usize; + let mut links = 0usize; + let mut is_a = 0usize; + let mut bad: Vec = Vec::new(); + for (i, id) in baked.ids.iter().enumerate() { + if id.ns != ns as u8 { + continue; + } + rows += 1; + let lanes = EdgeLanes::new(&baked.rows[i].0); + let mut got: Vec<(u8, u32, u32)> = lanes + .links() + .map(|(p, l)| (p as u8, l.classid, l.num)) + .collect(); + links += got.len(); + is_a += lanes.degree(Predicate::IsA) as usize; + let mut want = expected.get(id).cloned().unwrap_or_default(); + got.sort_unstable(); + want.sort_unstable(); + if got != want { + bad.push(*id); + } + } + let ok = bad.is_empty(); + all_ok &= ok; + println!( + "\n=== {:8} rows={rows} links_read={links} is_a_degree_total={is_a} — {}", + ns.prefix(), + if ok { "MATCH" } else { "MISMATCH" } + ); + if !ok { + for id in bad.iter().take(5) { + let i = baked.ids.iter().position(|x| x == id).unwrap(); + println!( + " {}:{:07} got={:?} want={:?}", + ns.prefix(), + id.num, + EdgeLanes::new(&baked.rows[i].0) + .links() + .map(|(p, l)| (p, l.num)) + .collect::>(), + expected.get(id) + ); + } + println!(" ... {} mismatching rows total", bad.len()); + } + // RO is small enough to print in full — the by-eye check. + if ns == Namespace::Ro { + for (i, id) in baked.ids.iter().enumerate() { + if id.ns != ns as u8 { + continue; + } + let l: Vec<_> = EdgeLanes::new(&baked.rows[i].0) + .links() + .map(|(p, l)| format!("{p:?}->0x{:08x}:{}", l.classid, l.num)) + .collect(); + println!(" RO:{:07} -> [{}]", id.num, l.join(", ")); + } + } + } + + assert!(all_ok, "lane decode disagrees with the triple table"); + println!("\nALL LANES VERIFIED against the SPO triple table."); +} diff --git a/crates/ogar-obo/src/crosswalk.rs b/crates/ogar-obo/src/crosswalk.rs index 57e7c612..b905daad 100644 --- a/crates/ogar-obo/src/crosswalk.rs +++ b/crates/ogar-obo/src/crosswalk.rs @@ -103,16 +103,17 @@ impl Crosswalk { // roll up: drop a trailing digit past the 3-char WHO stem, else the // sub-category dot, else give up. if let Some(pos) = c.rfind(|ch: char| ch.is_ascii_digit()) - && (c.len() > 3 || (c.contains('.') && pos > c.find('.').unwrap())) { - c.truncate(pos); - if c.ends_with('.') { - c.pop(); - } - if c.is_empty() { - return None; - } - continue; + && (c.len() > 3 || (c.contains('.') && pos > c.find('.').unwrap())) + { + c.truncate(pos); + if c.ends_with('.') { + c.pop(); } + if c.is_empty() { + return None; + } + continue; + } return None; } } @@ -153,16 +154,31 @@ mod tests { let mut nodes: HashMap = HashMap::new(); // MONDO:5148 (T2DM) xrefs ICD-10 E11 + MeSH D003924 nodes.insert( - TermId { ns: Namespace::Mondo as u8, num: 5148 }, + TermId { + ns: Namespace::Mondo as u8, + num: 5148, + }, node_with_xrefs(vec![ - Xref { source: XrefSource::Icd, id: "E11".into() }, - Xref { source: XrefSource::Mesh, id: "D003924".into() }, + Xref { + source: XrefSource::Icd, + id: "E11".into(), + }, + Xref { + source: XrefSource::Mesh, + id: "D003924".into(), + }, ]), ); // UBERON:945 (stomach) xrefs FMA:7148 nodes.insert( - TermId { ns: Namespace::Uberon as u8, num: 945 }, - node_with_xrefs(vec![Xref { source: XrefSource::Other("FMA".into()), id: "7148".into() }]), + TermId { + ns: Namespace::Uberon as u8, + num: 945, + }, + node_with_xrefs(vec![Xref { + source: XrefSource::Other("FMA".into()), + id: "7148".into(), + }]), ); let baked = bake(&nodes, 0x0000); let cw = Crosswalk::from_bake(&baked); @@ -187,8 +203,14 @@ mod tests { // roll up to it (measured: MONDO carries no ICD-10-GM). let mut nodes: HashMap = HashMap::new(); nodes.insert( - TermId { ns: Namespace::Mondo as u8, num: 5148 }, - node_with_xrefs(vec![Xref { source: XrefSource::Icd, id: "E11".into() }]), + TermId { + ns: Namespace::Mondo as u8, + num: 5148, + }, + node_with_xrefs(vec![Xref { + source: XrefSource::Icd, + id: "E11".into(), + }]), ); let cw = Crosswalk::from_bake(&bake(&nodes, 0x0000)); // direct GM code fails, rollup succeeds — no separate GM table diff --git a/crates/ogar-obo/src/edges.rs b/crates/ogar-obo/src/edges.rs new file mode 100644 index 00000000..cece9b6b --- /dev/null +++ b/crates/ogar-obo/src/edges.rs @@ -0,0 +1,453 @@ +//! The `col_idx` half of the CSR — edge targets packed into value lanes. +//! +//! # Why this module exists +//! +//! The bake's edge block at `[16,32)` is a per-predicate **degree histogram**: +//! it says *how many* targets a node has per predicate, but not *which*. In CSR +//! terms the bake persisted `row_ptr` and dropped `col_idx`, so the on-disk +//! artifact could describe a node's local shape but could not be traversed. +//! This module completes it: the targets ride the free value lanes, and the +//! degree histogram becomes load-bearing instead of decorative. +//! +//! # The lane form +//! +//! A lane is the one form the substrate has — `classid(4) + 12 content bytes`, +//! and the classid says how to read its own 12 bytes. An edge lane reads its +//! content as the **G2 `4×u24`** carving (`legacy_outliers::LegacyOutlier:: +//! WideTriple`), i.e. four literal links, and its `classid` is **the target +//! ontology's own row classid**. So a link is exactly the `(classid, identity)` +//! pair a *key* carries — the same binary search that resolves a key resolves a +//! link. Nothing new is invented to point at a node. +//! +//! ```text +//! lane: | classid: u32 | u24 | u24 | u24 | u24 | +//! ^ target ontology ^ four CURIE numerics +//! ``` +//! +//! # Layout +//! +//! The 480-byte value slab is 30 lanes. Lane 6 is the `EntityType` tenant +//! ([`crate::ENTITY_TYPE_SLAB_OFFSET`]); lanes 0..6 stay reserved for further +//! value tenants (RESERVE, DON'T RECLAIM). Edges occupy lanes 7..30 — +//! [`EDGE_LANE_COUNT`] lanes = [`EDGE_SLOT_COUNT`] link slots. +//! +//! Measured demand on the OBO core (`probe_edge_capacity`): 186 029 triples +//! over 68 789 subjects, max out-degree **62**, max lane demand for a single +//! row **16**. No row overflows the 23-lane budget, and the packer never +//! truncates silently — an overflow is reported ([`PackStats::dropped`]) so a +//! driver can fail rather than quietly lose edges. +//! +//! # Emit order (the reader's contract) +//! +//! Targets are emitted ascending by `(predicate, target namespace, numeric)`, +//! and **a lane never mixes target ontologies** — its classid names exactly +//! one, so a namespace change starts a new lane even mid-predicate. Unused +//! slots in a lane are `0`, which is unambiguous: no OBO CURIE numeric is 0 +//! (asserted by `probe_edge_capacity` A1 on the real sources). +//! +//! A reader walks lanes in order and consumes `degree(p)` targets for each +//! predicate `p` in ascending order, treating a `0` slot as "this lane is +//! spent, advance". That is [`EdgeLanes`], which borrows the row and decodes a +//! u24 with a shift — no allocation, no gathered window. + +use crate::{EDGES_OFFSET, NODE_ROW_STRIDE, Namespace, Predicate, TermId, VALUE_OFFSET}; + +/// Bytes per lane — the one form (`classid(4) + content(12)`). +pub const LANE_BYTES: usize = 16; +/// Links per lane under the G2 `4×u24` carving. +pub const LINKS_PER_LANE: usize = 4; +/// First edge lane's offset **within the 480-byte value slab** (lane 7). +pub const EDGE_LANE_SLAB_OFFSET: usize = 112; +/// Number of lanes reserved for edge targets (lanes 7..30). +pub const EDGE_LANE_COUNT: usize = 23; +/// Total link slots per row. +pub const EDGE_SLOT_COUNT: usize = EDGE_LANE_COUNT * LINKS_PER_LANE; + +const _: () = assert!(EDGE_LANE_SLAB_OFFSET > crate::ENTITY_TYPE_SLAB_OFFSET); +const _: () = assert!( + VALUE_OFFSET + EDGE_LANE_SLAB_OFFSET + EDGE_LANE_COUNT * LANE_BYTES == NODE_ROW_STRIDE, + "edge lanes must run to the end of the row without overlapping a value tenant" +); + +/// Read a little-endian `u24` (as a `u32`, high byte zero). +#[inline] +const fn u24_le(a: u8, b: u8, c: u8) -> u32 { + (a as u32) | ((b as u32) << 8) | ((c as u32) << 16) +} + +/// Write a little-endian `u24`. Debug-asserts the 24-bit fit — no silent +/// truncation, the same guard the key's u24 groups use. +#[inline] +fn put_u24_le(dst: &mut [u8], v: u32) { + debug_assert!(v <= 0x00FF_FFFF, "edge target numeric must fit 24 bits"); + dst[0] = (v & 0xFF) as u8; + dst[1] = ((v >> 8) & 0xFF) as u8; + dst[2] = ((v >> 16) & 0xFF) as u8; +} + +/// One resolved link: the `(classid, identity)` pair, exactly as a key carries +/// it. `classid` names the target ontology; `num` is the CURIE numeric. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub struct Link { + /// the target row's classid (`Namespace::render_classid`) + pub classid: u32, + /// the target CURIE numeric (24-bit) + pub num: u32, +} + +/// What a pack pass did — reported so a driver can refuse to ship a truncation. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct PackStats { + /// links written into lanes + pub written: usize, + /// lanes consumed + pub lanes: usize, + /// links that did NOT fit the lane budget (must be 0 to ship) + pub dropped: usize, + /// rows that dropped at least one link + pub rows_overflowed: usize, +} + +/// Pack one node's targets into its row's edge lanes. +/// +/// `targets` must already be the node's full out-edge set; this function sorts +/// it into the canonical `(predicate, namespace, numeric)` emit order itself, +/// so callers cannot get the reader's contract wrong by accident. +/// +/// Returns the per-row stats. Links beyond [`EDGE_SLOT_COUNT`] are counted in +/// [`PackStats::dropped`] and **not** written — never silently truncated. +pub fn pack_edges( + row: &mut [u8; NODE_ROW_STRIDE], + targets: &mut [(Predicate, TermId)], + app_prefix: u16, +) -> PackStats { + let mut st = PackStats::default(); + targets.sort_unstable_by_key(|(p, t)| (*p as u8, t.ns, t.num)); + + let base = VALUE_OFFSET + EDGE_LANE_SLAB_OFFSET; + let mut lane = 0usize; + let mut slot = LINKS_PER_LANE; // force a fresh lane on the first target + let mut group: Option<(u8, u8)> = None; + + for (p, t) in targets.iter() { + let g = (*p as u8, t.ns); + if group != Some(g) || slot == LINKS_PER_LANE { + group = Some(g); + lane += 1; + slot = 0; + if lane > EDGE_LANE_COUNT { + // budget exhausted — count the remainder, write nothing more. + st.dropped = targets.len() - st.written; + st.rows_overflowed = 1; + st.lanes = EDGE_LANE_COUNT; + return st; + } + let off = base + (lane - 1) * LANE_BYTES; + let classid = t.namespace().render_classid(app_prefix); + row[off..off + 4].copy_from_slice(&classid.to_le_bytes()); + } + let off = base + (lane - 1) * LANE_BYTES + 4 + slot * 3; + put_u24_le(&mut row[off..off + 3], t.num); + slot += 1; + st.written += 1; + } + st.lanes = lane; + st +} + +/// A borrowed view over one row's edge lanes — the traversal peek. +/// +/// Holds no copy of anything: `degree` reads the edge block in place and +/// [`links`] decodes a `u24` with a shift as it walks. This is the `col_idx` +/// side of a CSR row; combined with [`EdgeLanes::degree`] (`row_ptr`) it is a +/// complete adjacency row, which is what makes a semiring traversal — rather +/// than pointer chasing — the natural way to consume it. +#[derive(Debug, Clone, Copy)] +pub struct EdgeLanes<'a> { + row: &'a [u8; NODE_ROW_STRIDE], +} + +impl<'a> EdgeLanes<'a> { + /// Borrow a row's edge lanes. + #[must_use] + pub const fn new(row: &'a [u8; NODE_ROW_STRIDE]) -> Self { + EdgeLanes { row } + } + + /// This node's out-degree for one predicate — the `row_ptr` entry, read + /// straight from the edge block with no value decode at all. + #[must_use] + pub const fn degree(&self, p: Predicate) -> u8 { + self.row[EDGES_OFFSET + p as usize] + } + + /// Total out-degree across every predicate. + #[must_use] + pub fn total_degree(&self) -> usize { + (1..8).map(|i| self.row[EDGES_OFFSET + i] as usize).sum() + } + + /// Every link in emit order, paired with its predicate. + /// + /// Walks lanes in order, consuming `degree(p)` links per predicate in + /// ascending predicate order; a `0` slot means the lane is spent (no OBO + /// CURIE numeric is 0) and the walk advances to the next lane. + pub fn links(&self) -> impl Iterator + '_ { + LinkIter { + row: self.row, + lane: 0, + slot: 0, + pred: 0, + remaining: 0, + } + } + + /// Only the links carried by one predicate — the traversal hot path + /// (`is_a` for the subsumption spine, `part_of` for the anatomy backbone). + pub fn links_of(&self, p: Predicate) -> impl Iterator + '_ { + self.links().filter(move |(q, _)| *q == p).map(|(_, l)| l) + } +} + +struct LinkIter<'a> { + row: &'a [u8; NODE_ROW_STRIDE], + lane: usize, + slot: usize, + pred: u8, + remaining: u8, +} + +impl Iterator for LinkIter<'_> { + type Item = (Predicate, Link); + + fn next(&mut self) -> Option<(Predicate, Link)> { + let base = VALUE_OFFSET + EDGE_LANE_SLAB_OFFSET; + loop { + // advance to the next predicate that has any degree left + while self.remaining == 0 { + self.pred += 1; + if self.pred > 7 { + return None; + } + self.remaining = self.row[EDGES_OFFSET + self.pred as usize]; + } + if self.lane >= EDGE_LANE_COUNT { + return None; + } + if self.slot >= LINKS_PER_LANE { + self.lane += 1; + self.slot = 0; + continue; + } + let off = base + self.lane * LANE_BYTES; + let n = u24_le( + self.row[off + 4 + self.slot * 3], + self.row[off + 5 + self.slot * 3], + self.row[off + 6 + self.slot * 3], + ); + if n == 0 { + // pad — the lane is spent, next lane + self.lane += 1; + self.slot = 0; + continue; + } + let classid = u32::from_le_bytes([ + self.row[off], + self.row[off + 1], + self.row[off + 2], + self.row[off + 3], + ]); + self.slot += 1; + self.remaining -= 1; + let p = PREDICATES[self.pred as usize]; + return Some((p, Link { classid, num: n })); + } + } +} + +/// Predicate by discriminant; index 0 is reserved and never yielded. +const PREDICATES: [Predicate; 8] = [ + Predicate::IsA, // slot 0 reserved — never reached (degree byte 0 unused) + Predicate::IsA, + Predicate::PartOf, + Predicate::HasPhenotype, + Predicate::HasLocation, + Predicate::HasAnatomy, + Predicate::HasQuality, + Predicate::Other, +]; + +/// The classid a link into `ns` carries, for callers building expectations. +#[must_use] +pub const fn target_classid(ns: Namespace, app_prefix: u16) -> u32 { + ns.render_classid(app_prefix) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn t(ns: Namespace, num: u32) -> TermId { + TermId { ns: ns as u8, num } + } + + fn row_with_degrees(degs: &[(Predicate, u8)]) -> [u8; NODE_ROW_STRIDE] { + let mut row = [0u8; NODE_ROW_STRIDE]; + for (p, d) in degs { + row[EDGES_OFFSET + *p as usize] = *d; + } + row + } + + #[test] + fn the_budget_covers_the_measured_worst_row() { + // probe_edge_capacity on the real OBO core: max out-degree 62, max lane + // demand 16. Both must fit, with the lane demand the binding one. + const { assert!(16 <= EDGE_LANE_COUNT) }; + const { assert!(62 <= EDGE_SLOT_COUNT) }; + } + + #[test] + fn a_single_link_round_trips_with_its_target_classid() { + let mut row = row_with_degrees(&[(Predicate::IsA, 1)]); + let mut tg = vec![(Predicate::IsA, t(Namespace::Mondo, 5015))]; + let st = pack_edges(&mut row, &mut tg, 0x0000); + assert_eq!(st.written, 1); + assert_eq!(st.dropped, 0); + + let lanes = EdgeLanes::new(&row); + assert_eq!(lanes.degree(Predicate::IsA), 1); + let got: Vec<_> = lanes.links().collect(); + assert_eq!( + got, + vec![( + Predicate::IsA, + Link { + classid: 0x0301_0000, + num: 5015 + } + )] + ); + } + + #[test] + fn a_namespace_change_starts_a_new_lane_so_a_classid_never_lies() { + // one predicate, two target ontologies: the lane classid names exactly + // one ontology, so the second target cannot share the first's lane. + let mut row = row_with_degrees(&[(Predicate::Other, 2)]); + let mut tg = vec![ + (Predicate::Other, t(Namespace::Mondo, 1)), + (Predicate::Other, t(Namespace::Hpo, 2)), + ]; + let st = pack_edges(&mut row, &mut tg, 0x0000); + assert_eq!(st.written, 2); + assert_eq!(st.lanes, 2, "a namespace change must start a new lane"); + + let got: Vec<_> = EdgeLanes::new(&row).links().map(|(_, l)| l).collect(); + assert_eq!( + got, + vec![ + Link { + classid: 0x0301_0000, + num: 1 + }, + Link { + classid: 0x0302_0000, + num: 2 + } + ] + ); + } + + #[test] + fn a_full_lane_spills_into_the_next_one() { + let mut row = row_with_degrees(&[(Predicate::IsA, 5)]); + let mut tg: Vec<_> = (1..=5) + .map(|n| (Predicate::IsA, t(Namespace::Uberon, n))) + .collect(); + let st = pack_edges(&mut row, &mut tg, 0x0000); + assert_eq!(st.written, 5); + assert_eq!(st.lanes, 2); + let got: Vec<_> = EdgeLanes::new(&row) + .links_of(Predicate::IsA) + .map(|l| l.num) + .collect(); + assert_eq!(got, vec![1, 2, 3, 4, 5]); + } + + #[test] + fn predicates_are_recovered_from_the_degree_histogram() { + // the lane carries no predicate byte — the reader recovers it from + // row_ptr. Two predicates, one shared target ontology. + let mut row = row_with_degrees(&[(Predicate::IsA, 2), (Predicate::PartOf, 1)]); + let mut tg = vec![ + (Predicate::PartOf, t(Namespace::Uberon, 300)), + (Predicate::IsA, t(Namespace::Uberon, 100)), + (Predicate::IsA, t(Namespace::Uberon, 200)), + ]; + pack_edges(&mut row, &mut tg, 0x0000); + let got: Vec<_> = EdgeLanes::new(&row) + .links() + .map(|(p, l)| (p, l.num)) + .collect(); + assert_eq!( + got, + vec![ + (Predicate::IsA, 100), + (Predicate::IsA, 200), + (Predicate::PartOf, 300), + ] + ); + } + + #[test] + fn an_overflow_is_reported_and_never_silently_truncated() { + let mut row = row_with_degrees(&[(Predicate::IsA, 255)]); + // one target ontology, more links than the budget holds + let over = EDGE_SLOT_COUNT + 7; + let mut tg: Vec<_> = (1..=over as u32) + .map(|n| (Predicate::IsA, t(Namespace::Hpo, n))) + .collect(); + let st = pack_edges(&mut row, &mut tg, 0x0000); + assert_eq!(st.rows_overflowed, 1); + assert_eq!(st.dropped, over - EDGE_SLOT_COUNT); + assert_eq!(st.written, EDGE_SLOT_COUNT); + } + + #[test] + fn the_reader_can_fail() { + // anti-vacuity: the walk is sensitive to the histogram, not decorative. + // Same lane bytes, a degree of 1 instead of 3 → exactly one link read. + let mut row = row_with_degrees(&[(Predicate::IsA, 3)]); + let mut tg: Vec<_> = (1..=3) + .map(|n| (Predicate::IsA, t(Namespace::Pato, n))) + .collect(); + pack_edges(&mut row, &mut tg, 0x0000); + assert_eq!(EdgeLanes::new(&row).links().count(), 3); + + row[EDGES_OFFSET + Predicate::IsA as usize] = 1; + assert_eq!( + EdgeLanes::new(&row).links().count(), + 1, + "row_ptr is load-bearing: a wrong degree must change the walk" + ); + } + + #[test] + fn an_edge_lane_never_overwrites_the_entity_type_tenant() { + let mut row = [0u8; NODE_ROW_STRIDE]; + let et = VALUE_OFFSET + crate::ENTITY_TYPE_SLAB_OFFSET; + row[et..et + 2].copy_from_slice(&7u16.to_le_bytes()); + row[EDGES_OFFSET + Predicate::IsA as usize] = EDGE_SLOT_COUNT as u8; + let mut tg: Vec<_> = (1..=EDGE_SLOT_COUNT as u32) + .map(|n| (Predicate::IsA, t(Namespace::Ro, n))) + .collect(); + let st = pack_edges(&mut row, &mut tg, 0x0000); + assert_eq!(st.dropped, 0); + assert_eq!( + u16::from_le_bytes([row[et], row[et + 1]]), + 7, + "a full edge pack must leave the EntityType tenant intact" + ); + } +} diff --git a/crates/ogar-obo/src/lib.rs b/crates/ogar-obo/src/lib.rs index e9391ddb..a4956b8f 100644 --- a/crates/ogar-obo/src/lib.rs +++ b/crates/ogar-obo/src/lib.rs @@ -41,6 +41,7 @@ #![deny(missing_docs)] pub mod crosswalk; +pub mod edges; pub mod reason; /// Row stride of the canonical SoA node — `key(16) + edges(16) + value(480)`. @@ -171,10 +172,7 @@ impl TermId { if num > 0x00FF_FFFF { return None; } - Some(TermId { - ns: ns as u8, - num, - }) + Some(TermId { ns: ns as u8, num }) } /// This term's namespace. @@ -317,13 +315,23 @@ fn pack_key(classid: u32, identity: u32) -> [u8; 16] { k } -/// Pack one node into its 512-byte row. Edges are placed **out-of-line** as -/// SPO triples (see [`bake`]); the in-row `edges` block records the *degree* -/// per predicate (a one-byte histogram, saturating at 255) so a reader has the -/// node's local shape without the triple table, and the `EntityType` value -/// tenant records the namespace. The one-byte basin-adjacency use of the edge -/// block is deferred to the HHTL-clustering pass (RESERVE, DON'T RECLAIM). -fn pack_row(classid: u32, id: &TermId, node: &OboNode) -> Row512 { +/// Pack one node into its 512-byte row. +/// +/// The in-row `edges` block records the *degree* per predicate (a one-byte +/// histogram, saturating at 255) — the CSR `row_ptr`. The **targets** (`col_idx`) +/// ride the free value lanes as G2 `4×u24` literal links; see [`edges`] for the +/// lane form and the reader's contract. The `EntityType` value tenant records +/// the namespace. The same edges are also emitted out-of-line as SPO triples by +/// [`bake`] — the in-row copy is what makes a row traversable on its own. +/// +/// Returns the row plus the pack stats, so a driver can refuse to ship a +/// truncation rather than lose edges silently. +fn pack_row( + classid: u32, + id: &TermId, + node: &OboNode, + app_prefix: u16, +) -> (Row512, edges::PackStats) { let mut row = Row512::zeroed(); row.0[0..16].copy_from_slice(&pack_key(classid, id.num)); // edges block [16,32): a per-predicate degree histogram (index by Predicate @@ -339,7 +347,13 @@ fn pack_row(classid: u32, id: &TermId, node: &OboNode) -> Row512 { // value slab: EntityType tenant (u16 namespace) at its carve offset. let et = VALUE_OFFSET + ENTITY_TYPE_SLAB_OFFSET; row.0[et..et + 2].copy_from_slice(&(id.ns as u16).to_le_bytes()); - row + // value slab: the edge targets — the col_idx half of the CSR. + let mut targets: Vec<(Predicate, TermId)> = + Vec::with_capacity(node.is_a.len() + node.rel.len()); + targets.extend(node.is_a.iter().map(|t| (Predicate::IsA, *t))); + targets.extend(node.rel.iter().copied()); + let st = edges::pack_edges(&mut row.0, &mut targets, app_prefix); + (row, st) } // ── OBO `.obo` parser ───────────────────────────────────────────────────── @@ -414,16 +428,18 @@ pub fn parse_obo(text: &str) -> std::collections::HashMap { // `relationship: ! label` — the target is the LAST // whitespace token before any `!`. if let Some(sid) = cur - && let Some(t) = last_curie(rest) { - let p = classify(sid.namespace(), t.namespace()); - nodes.entry(sid).or_default().rel.push((p, t)); - } + && let Some(t) = last_curie(rest) + { + let p = classify(sid.namespace(), t.namespace()); + nodes.entry(sid).or_default().rel.push((p, t)); + } } else if let Some(rest) = line.strip_prefix("intersection_of: ") { if let Some(sid) = cur - && let Some(t) = last_curie(rest) { - let p = classify(sid.namespace(), t.namespace()); - nodes.entry(sid).or_default().rel.push((p, t)); - } + && let Some(t) = last_curie(rest) + { + let p = classify(sid.namespace(), t.namespace()); + nodes.entry(sid).or_default().rel.push((p, t)); + } } else if let Some(rest) = line.strip_prefix("xref: ") { // `xref: : ! label` — the projection-join / guideline // bearing. Kept verbatim; NEVER truncated (MeSH → Leitlinie spider). @@ -431,12 +447,14 @@ pub fn parse_obo(text: &str) -> std::collections::HashMap { let tok = rest.split('!').next().unwrap_or(rest).trim(); let tok = tok.split_whitespace().next().unwrap_or(tok); if let Some((src, id)) = tok.split_once(':') - && !src.is_empty() && !id.is_empty() { - nodes.entry(sid).or_default().xref.push(Xref { - source: XrefSource::from_prefix(src), - id: id.to_string(), - }); - } + && !src.is_empty() + && !id.is_empty() + { + nodes.entry(sid).or_default().xref.push(Xref { + source: XrefSource::from_prefix(src), + id: id.to_string(), + }); + } } } } @@ -446,14 +464,22 @@ pub fn parse_obo(text: &str) -> std::collections::HashMap { /// First whitespace token of a line (before any `!` comment) — the target of /// an `is_a:` line. fn first_curie(s: &str) -> &str { - s.split('!').next().unwrap_or(s).split_whitespace().next().unwrap_or("").trim() + s.split('!') + .next() + .unwrap_or(s) + .split_whitespace() + .next() + .unwrap_or("") + .trim() } /// Last CURIE-shaped token before any `!` — the object of a `relationship:` / /// `intersection_of:` line (the predicate is the earlier token). fn last_curie(s: &str) -> Option { let head = s.split('!').next().unwrap_or(s); - head.split_whitespace().rfind(|t| t.contains(':')).and_then(TermId::parse) + head.split_whitespace() + .rfind(|t| t.contains(':')) + .and_then(TermId::parse) } // ── bake: nodes+edges → 512-byte rows + SPO triples + stats ──────────────── @@ -482,6 +508,14 @@ pub struct BakeStats { pub xrefs: usize, /// MeSH xrefs specifically — the guideline-spider bearings pub mesh_xrefs: usize, + /// edge targets written into value lanes — the CSR `col_idx`. Equals + /// [`BakeStats::triples`] on a clean bake (every edge is also in-row). + pub links_packed: usize, + /// edge targets that did NOT fit the lane budget. **Must be 0 to ship** — + /// a non-zero value means the artifact is missing edges. + pub links_dropped: usize, + /// rows that overflowed the lane budget (must be 0 to ship) + pub rows_overflowed: usize, } /// The bake result: the 512-byte SoA rows (loader-facing) + the SPO triple @@ -506,10 +540,7 @@ pub struct Bake { /// logical-def edges folded into the nodes' `rel`) into [`Bake`]. `app_prefix` /// is the lo-u16 render skin (`0x0000` = the canonical reference skin). #[must_use] -pub fn bake( - nodes: &std::collections::HashMap, - app_prefix: u16, -) -> Bake { +pub fn bake(nodes: &std::collections::HashMap, app_prefix: u16) -> Bake { let mut ids: Vec = nodes .iter() .filter(|(_, n)| !n.obsolete) @@ -528,7 +559,11 @@ pub fn bake( for id in &ids { let node = &nodes[id]; let classid = id.namespace().render_classid(app_prefix); - rows.push(pack_row(classid, id, node)); + let (row, pk) = pack_row(classid, id, node, app_prefix); + rows.push(row); + stats.links_packed += pk.written; + stats.links_dropped += pk.dropped; + stats.rows_overflowed += pk.rows_overflowed; // Preserve every external cross-reference — the projection-join / // guideline-spider bearings. NEVER truncated. for x in &node.xref { @@ -655,22 +690,35 @@ pub fn parse_hp_logical_defs(owl: &str) -> Vec<(TermId, Predicate, TermId)> { in_eq += 1; } if in_eq > 0 - && let Some(sid) = cur { - for uid in find_obo_ids(line, "UBERON_") { - out.push(( - TermId { ns: Namespace::Hpo as u8, num: sid }, - Predicate::HasAnatomy, - TermId { ns: Namespace::Uberon as u8, num: uid }, - )); - } - for pid in find_obo_ids(line, "PATO_") { - out.push(( - TermId { ns: Namespace::Hpo as u8, num: sid }, - Predicate::HasQuality, - TermId { ns: Namespace::Pato as u8, num: pid }, - )); - } + && let Some(sid) = cur + { + for uid in find_obo_ids(line, "UBERON_") { + out.push(( + TermId { + ns: Namespace::Hpo as u8, + num: sid, + }, + Predicate::HasAnatomy, + TermId { + ns: Namespace::Uberon as u8, + num: uid, + }, + )); } + for pid in find_obo_ids(line, "PATO_") { + out.push(( + TermId { + ns: Namespace::Hpo as u8, + num: sid, + }, + Predicate::HasQuality, + TermId { + ns: Namespace::Pato as u8, + num: pid, + }, + )); + } + } if line.contains("") { in_eq = (in_eq - 1).max(0); } @@ -685,7 +733,10 @@ fn find_hp_about(line: &str) -> Option { let i = line.find("owl:Class rdf:about=")?; let rest = &line[i..]; let j = rest.find("HP_")?; - let digits: String = rest[j + 3..].chars().take_while(char::is_ascii_digit).collect(); + let digits: String = rest[j + 3..] + .chars() + .take_while(char::is_ascii_digit) + .collect(); digits.parse().ok() } @@ -697,9 +748,10 @@ fn find_obo_ids(line: &str, prefix: &str) -> Vec { let after = &hay[i + prefix.len()..]; let digits: String = after.chars().take_while(char::is_ascii_digit).collect(); if let Ok(n) = digits.parse::() - && n <= 0x00FF_FFFF { - ids.push(n); - } + && n <= 0x00FF_FFFF + { + ids.push(n); + } hay = &after[digits.len()..]; } ids @@ -742,15 +794,26 @@ mod tests { fn v3_tail_carries_oversize_curie_numerics_on_the_family_identity_rail() { // Above u16 — the case a bare `identity` cannot hold. let big = 700_092u32; - assert!(big > u32::from(u16::MAX), "fixture must exceed the u16 the rail exists for"); + assert!( + big > u32::from(u16::MAX), + "fixture must exceed the u16 the rail exists for" + ); let k = pack_key(Namespace::Mondo.render_classid(0x0000), big); // Byte positions, per new_v2. - assert_eq!(&k[10..12], &[0, 0], "leaf stays dormant (RESERVE, DON'T RECLAIM)"); + assert_eq!( + &k[10..12], + &[0, 0], + "leaf stays dormant (RESERVE, DON'T RECLAIM)" + ); let family = u16::from_le_bytes([k[12], k[13]]); let identity = u16::from_le_bytes([k[14], k[15]]); assert_eq!(u32::from(family), big >> 16, "family holds the high half"); - assert_eq!(u32::from(identity), big & 0xFFFF, "identity holds the low half"); + assert_eq!( + u32::from(identity), + big & 0xFFFF, + "identity holds the low half" + ); // Lossless through the public reader. let mut row = Row512::zeroed(); @@ -762,13 +825,19 @@ mod tests { // The V1 read of the SAME bytes must NOT agree — proof the tail really // moved, not that both layouts happen to coincide on this fixture. let v1_read = u32::from_le_bytes([row.0[13], row.0[14], row.0[15], 0]); - assert_ne!(v1_read, big, "a V1 u24 read of a V3 row must be observably wrong"); + assert_ne!( + v1_read, big, + "a V1 u24 read of a V3 row must be observably wrong" + ); // Ordering is preserved: family is the HIGH half, so (family, identity) // sorts as the numeric does. This is what keeps binary-search-by-key // valid over the sorted bake. let lo = pack_key(Namespace::Mondo.render_classid(0x0000), 5_148); - assert!(lo[12..16] < k[12..16], "tail bytes order as the numeric orders"); + assert!( + lo[12..16] < k[12..16], + "tail bytes order as the numeric orders" + ); } #[test] @@ -798,7 +867,11 @@ is_a: MONDO:0005015 ! diabetes mellitus\n"; let t = n(Namespace::Mondo, 5148); let node = &nodes[&t]; assert_eq!(node.xref.len(), 3, "all three xrefs kept"); - assert!(node.xref.iter().any(|x| x.source == XrefSource::Mesh && x.id == "D003924")); + assert!( + node.xref + .iter() + .any(|x| x.source == XrefSource::Mesh && x.id == "D003924") + ); let bake = bake(&nodes, 0x0000); assert_eq!(bake.stats.mesh_xrefs, 1, "MeSH bearing counted"); assert_eq!(bake.stats.xrefs, 3); diff --git a/crates/ogar-obo/src/reason.rs b/crates/ogar-obo/src/reason.rs index 05c06cac..093fe2d4 100644 --- a/crates/ogar-obo/src/reason.rs +++ b/crates/ogar-obo/src/reason.rs @@ -115,9 +115,7 @@ pub struct ElStats { /// Returns, per node, the sorted deduped set of all transitive ancestors. /// Assumes acyclic (run [`count_is_a_cycles`] first); a residual cycle is /// simply not expanded past a re-visit, never loops. -fn ancestor_closure( - adj: &HashMap>, -) -> HashMap> { +fn ancestor_closure(adj: &HashMap>) -> HashMap> { // Kahn order over the parent graph, then DP bottom-up. // Build reverse (parent -> children) for indegree over child->parent edges. let mut all: Vec = Vec::new(); @@ -309,9 +307,15 @@ mod tests { fn t(sns: Namespace, s: u32, p: Predicate, ons: Namespace, o: u32) -> Triple { Triple { - s: TermId { ns: sns as u8, num: s }, + s: TermId { + ns: sns as u8, + num: s, + }, p, - o: TermId { ns: ons as u8, num: o }, + o: TermId { + ns: ons as u8, + num: o, + }, } } @@ -353,9 +357,27 @@ mod tests { // UBERON:200 is_a UBERON:300 (which is a kind of ...) // ⟹ HP:9 is grounded to UBERON:{200,300} too (2 inferred-beyond-asserted). let tr = vec![ - t(Namespace::Hpo, 9, Predicate::HasAnatomy, Namespace::Uberon, 100), - t(Namespace::Uberon, 100, Predicate::PartOf, Namespace::Uberon, 200), - t(Namespace::Uberon, 200, Predicate::IsA, Namespace::Uberon, 300), + t( + Namespace::Hpo, + 9, + Predicate::HasAnatomy, + Namespace::Uberon, + 100, + ), + t( + Namespace::Uberon, + 100, + Predicate::PartOf, + Namespace::Uberon, + 200, + ), + t( + Namespace::Uberon, + 200, + Predicate::IsA, + Namespace::Uberon, + 300, + ), ]; let s = saturate(&tr); // R∃ fires on BOTH existentials: HP:9→{200,300} (2) AND the part_of From a866854192e88eed84a7dc54c4f395fa67c0fdbe Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 7 Aug 2026 23:58:59 +0000 Subject: [PATCH 02/13] ogar-elk: observe the baked substrate; correct the pre-bake framing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The crate doc said "Where this runs: BEFORE the bake, never after". That is backwards and was ruled so: the joins are pre-bake, but ELK is not a join — it observes the baked result and reports what follows from it. The bake is its input, not its output. This corrects the doc, the Cargo description, and the three consequences that were derived from the wrong premise. The consequence that needed real rework was "types are legal here, nothing this crate defines survives the bake". Post-bake that reads as a contradiction, and the honest form is narrower: `ClassAddr` is not schema, it is the (classid, identity) pair a baked key already carries, decoded from its position — the same shape `ogar_obo::edges::Link` reads out of a lane. It declares nothing and outlives nothing. `Subsumption` stays a distinct type for the one reason it was ever worth one: so a caller cannot pass a part_of edge to a reasoner that has no role composition. The other two consequences survive unchanged and one gets stronger: post-bake, "no file, no CURIE, no label" is not a discipline the crate has to keep — baked rows carry no CURIE and no label to reach for. Adds the adapter this rests on, `ogar_obo::edges::subsumptions`: the is_a-only projection over baked rows, yielding (subclass, superclass) as the address pairs the keys and lanes already hold. The is_a filter is the safety, not a convenience — walking subsumption together with part_of derives false ancestors, and filtering in the projection means a caller cannot forget to. Also adds `unpack_key` (a row's own address, zero value decode) and `edges::subject`. `elk_over_bake` runs the whole path on the real slab and cross-checks it against a representation it does not share. Reading is_a out of the lanes yields 107 813 asserted subsumptions — the per-ontology degree totals summed — and closing them yields 738 651 transitive ancestors and 0 equivalence cycles. The bake's own `reason::saturate` reports 738 651 subsumption pairs and 0 cycles from the out-of-line triple table. Two reasoners over two representations, identical to the unit. Root counts agree too: RO has exactly 2 of its 4 rows with a parent, which are RO:0002533 and RO:0002534. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01KCGhDYoQBXs3poaR7sFuqp --- crates/ogar-elk/Cargo.toml | 5 +- crates/ogar-elk/examples/elk_over_bake.rs | 115 ++++++++++++++++++++++ crates/ogar-elk/src/lib.rs | 44 +++++---- crates/ogar-obo/src/edges.rs | 83 +++++++++++++++- crates/ogar-obo/src/lib.rs | 13 +++ 5 files changed, 238 insertions(+), 22 deletions(-) create mode 100644 crates/ogar-elk/examples/elk_over_bake.rs diff --git a/crates/ogar-elk/Cargo.toml b/crates/ogar-elk/Cargo.toml index 323b7a4a..2c5e6b27 100644 --- a/crates/ogar-elk/Cargo.toml +++ b/crates/ogar-elk/Cargo.toml @@ -6,6 +6,9 @@ license.workspace = true repository.workspace = true authors.workspace = true rust-version.workspace = true -description = "The EL subsumption closure as a PRE-BAKE join validator. Answers two questions and no others: does `A ⊑ B` follow from what is asserted, and is adding a set of axioms to an existing closure sound (does it introduce an equivalence cycle). Entailments are facts — they follow necessarily — so nothing here is graded, ranked, or scored. Runs once, upstream of the bake, over the joined sources; nothing it defines survives into the baked substrate. Zero dependencies, and no serialization surface at all." +description = "The EL subsumption closure as a POST-BAKE observer. Answers two questions and no others: does `A ⊑ B` follow from the baked edges, and is adding a set of axioms to an existing closure sound (does it introduce an equivalence cycle). Entailments are facts — they follow necessarily — so nothing here is graded, ranked, or scored. Reads the `is_a` links carried in a baked row's own edge lanes; the joins happened upstream and are not its business. Zero dependencies, and no serialization surface at all." [dependencies] + +[dev-dependencies] +ogar-obo = { path = "../ogar-obo" } diff --git a/crates/ogar-elk/examples/elk_over_bake.rs b/crates/ogar-elk/examples/elk_over_bake.rs new file mode 100644 index 00000000..461e51d4 --- /dev/null +++ b/crates/ogar-elk/examples/elk_over_bake.rs @@ -0,0 +1,115 @@ +//! ELK over the **baked** substrate — the post-bake observation path. +//! +//! Reads a `.soa` slab, peeks the `is_a` links out of each row's own edge lanes +//! (`ogar_obo::edges::subsumptions`), closes over them, and reports what +//! follows. No `.obo` file is opened; no CURIE is parsed. The only input is +//! positions in baked rows, which is what makes this observation rather than a +//! second harvest. +//! +//! Usage: `cargo run -p ogar-elk --example elk_over_bake -- ` + +use ogar_elk::{ClassAddr, Closure, Subsumption}; +use ogar_obo::{NODE_ROW_STRIDE, Row512, edges}; + +fn main() { + let path = std::env::args() + .nth(1) + .unwrap_or_else(|| "/tmp/obo/obo-core.soa".to_string()); + let bytes = std::fs::read(&path).unwrap_or_else(|e| panic!("read {path}: {e}")); + assert!( + bytes.len().is_multiple_of(NODE_ROW_STRIDE), + "not a slab: {} bytes is not a multiple of {NODE_ROW_STRIDE}", + bytes.len() + ); + + // A harness copy into an aligned buffer. On the real path Lance hands over + // an arrow-aligned FixedSizeBinary(512) column and this step does not exist. + let n = bytes.len() / NODE_ROW_STRIDE; + let mut rows = vec![Row512::zeroed(); n]; + for (i, r) in rows.iter_mut().enumerate() { + r.0.copy_from_slice(&bytes[i * NODE_ROW_STRIDE..(i + 1) * NODE_ROW_STRIDE]); + } + println!("slab: {path}\n rows: {n}"); + + // ── the peek: is_a links straight out of the lanes ─────────────────────── + let t0 = std::time::Instant::now(); + let closure = Closure::from_asserted(edges::subsumptions(&rows).map(|(sub, sup)| { + Subsumption::new( + ClassAddr::new(sub.classid, sub.num), + ClassAddr::new(sup.classid, sup.num), + ) + })); + let build = t0.elapsed(); + println!( + " asserted subsumptions read from lanes: {} in {build:?}", + closure.asserted_len() + ); + + // Per-class census of what the closure indexed, so the numbers are + // attributable to an ontology rather than a single total. + let mut by_class: std::collections::BTreeMap = Default::default(); + for c in closure.subclasses() { + *by_class.entry(c.classid).or_default() += 1; + } + println!(" classes with at least one parent:"); + for (cid, count) in &by_class { + println!(" 0x{cid:08x}: {count}"); + } + + // ── question 1: what follows ───────────────────────────────────────────── + // Depth and breadth of the spine, sampled over every indexed subclass. + let t1 = std::time::Instant::now(); + let mut max_ancestors = 0usize; + let mut deepest: Option = None; + let mut total = 0usize; + for c in closure.subclasses() { + let a = closure.supers_of(c).len(); + total += a; + if a > max_ancestors { + max_ancestors = a; + deepest = Some(c); + } + } + let walk = t1.elapsed(); + let indexed = by_class.values().sum::(); + println!( + "\n transitive ancestors: {total} over {indexed} subclasses \ + (mean {:.1}, max {max_ancestors} at {deepest:?}) in {walk:?}", + total as f64 / indexed.max(1) as f64 + ); + + // ── question 2: is the baked spine sound ───────────────────────────────── + let cycles = closure.equivalence_cycles(); + println!( + "\n equivalence cycles in the baked spine: {} (depth cap {})", + cycles.len(), + closure.depth_cap() + ); + for c in cycles.iter().take(10) { + println!(" 0x{:08x}:{}", c.classid, c.identity); + } + if cycles.len() > 10 { + println!(" ... {} total", cycles.len()); + } + + // A worked entailment, printed so the run is not merely a count: the + // deepest class and one of its derived ancestors, plus a control that must + // NOT be entailed. + if let Some(d) = deepest { + let supers = closure.supers_of(d); + if let Some(top) = supers.iter().max() { + assert!(closure.entails(d, *top)); + println!( + "\n entails: 0x{:08x}:{} ⊑ 0x{:08x}:{} (derived)", + d.classid, d.identity, top.classid, top.identity + ); + } + // control: an identity no ontology mints (numeric 0 never occurs) + let ghost = ClassAddr::new(d.classid, 0); + assert!( + !closure.entails(d, ghost), + "entailment must be able to say no" + ); + println!(" and does NOT entail a class that was never minted — the walk discriminates"); + } +} diff --git a/crates/ogar-elk/src/lib.rs b/crates/ogar-elk/src/lib.rs index 334a57a7..071bc3b5 100644 --- a/crates/ogar-elk/src/lib.rs +++ b/crates/ogar-elk/src/lib.rs @@ -2,9 +2,10 @@ //! //! # What this is, and the two questions it answers //! -//! Third member of the factfinder family beside [`ogar-obo`] (harvest) and -//! [`ogar-ro`] (the predicate palette). Those two say what is *asserted*; this -//! one says what *follows*. It answers exactly two questions: +//! Third member of the factfinder family beside [`ogar-obo`] (harvest and bake) +//! and [`ogar-ro`] (the predicate palette). Those two say what is *asserted*; +//! this one observes the baked result and says what *follows*. It answers +//! exactly two questions: //! //! 1. **Does `A ⊑ B` follow** from the asserted edges? ([`Closure::entails`]) //! 2. **Is adding a set of axioms sound** — does the merged closure introduce @@ -18,29 +19,34 @@ //! [`ogar-obo`]: https://docs.rs/ogar-obo //! [`ogar-ro`]: https://docs.rs/ogar-ro //! -//! # Where this runs: BEFORE the bake, never after +//! # Where this runs: AFTER the bake, as observation //! -//! **The joins are pre-bake.** Reconciling independently authored sources — -//! deciding which assertions corroborate, which enrich, and which contradict — -//! is exactly the work that must finish before anything is baked, because the -//! bake is what freezes the answer into positions. So this crate is a **stage**, -//! not a layer: it runs once, upstream, and then it is done. +//! **The joins are pre-bake; this is not a join.** Reconciling independently +//! authored sources is upstream work that must finish before anything is baked, +//! because the bake is what freezes the answer into positions. What this crate +//! does is the opposite end: it **observes the baked substrate** and reports +//! what follows from it. The bake is its input, not its output. +//! +//! Concretely, its input is the `is_a` links carried in a baked row's own edge +//! lanes — `ogar_obo::edges::subsumptions` is the adapter, and it exists so a +//! reasoner reads addresses out of positions rather than re-deriving them. //! //! Three consequences follow, and they are the whole design: //! -//! 1. **Types are legal here.** [`ClassAddr`] and [`Subsumption`] exist because -//! a join needs a key and a directed edge. They are pre-bake scaffolding. -//! **Nothing this crate defines survives the bake** — afterwards there are -//! only classes, and a class is resolved by position, not by a type declared -//! here. If one of these types ever appears in a post-bake read path, that is -//! the leak, and it is this doc that says so. +//! 1. **These are readings, not schema.** [`ClassAddr`] is the `(classid, +//! identity)` pair a baked key already carries, decoded from its position — +//! the same shape `ogar_obo::edges::Link` reads out of a lane. It declares +//! nothing about a class and outlives nothing; a class is still resolved by +//! position. [`Subsumption`] is a direction, and it is a distinct type only +//! so a caller cannot hand this crate a `part_of` edge by accident. //! 2. **Nothing here is in the hot path**, so nothing here may pretend to be. //! There is no serialization surface — not behind a feature, not optionally. -//! A join validator that could serialize its verdict would invite someone to -//! ship the verdict instead of the bake. -//! 3. **No file, no CURIE, no label.** Input is already-joined addressed edges. +//! An observer that could serialize its verdict would invite someone to ship +//! the verdict as if it were substrate. +//! 3. **No file, no CURIE, no label.** Input is addressed edges read from baked +//! rows — which carry no CURIE and no label to reach for in the first place. //! Reaching back for the source document would put parsing inside the -//! validator and re-introduce the coupling the bake exists to remove. +//! observer and re-introduce the coupling the bake exists to remove. //! //! # The fragment, stated precisely //! diff --git a/crates/ogar-obo/src/edges.rs b/crates/ogar-obo/src/edges.rs index cece9b6b..e3c1fee7 100644 --- a/crates/ogar-obo/src/edges.rs +++ b/crates/ogar-obo/src/edges.rs @@ -192,7 +192,7 @@ impl<'a> EdgeLanes<'a> { /// Walks lanes in order, consuming `degree(p)` links per predicate in /// ascending predicate order; a `0` slot means the lane is spent (no OBO /// CURIE numeric is 0) and the walk advances to the next lane. - pub fn links(&self) -> impl Iterator + '_ { + pub fn links(self) -> impl Iterator + 'a { LinkIter { row: self.row, lane: 0, @@ -204,7 +204,7 @@ impl<'a> EdgeLanes<'a> { /// Only the links carried by one predicate — the traversal hot path /// (`is_a` for the subsumption spine, `part_of` for the anatomy backbone). - pub fn links_of(&self, p: Predicate) -> impl Iterator + '_ { + pub fn links_of(self, p: Predicate) -> impl Iterator + 'a { self.links().filter(move |(q, _)| *q == p).map(|(_, l)| l) } } @@ -283,6 +283,36 @@ pub const fn target_classid(ns: Namespace, app_prefix: u16) -> u32 { ns.render_classid(app_prefix) } +/// This row's own address as a [`Link`] — the same `(classid, identity)` shape +/// a target carries, because a subject and a target are the same kind of thing. +#[must_use] +pub const fn subject(row: &[u8; NODE_ROW_STRIDE]) -> Link { + let (classid, num) = crate::unpack_key(row); + Link { classid, num } +} + +/// The **subsumption projection** over baked rows: `(subclass, superclass)` +/// pairs, `is_a` only. +/// +/// This is the adapter a post-bake EL reasoner consumes. It reads the baked +/// artifact and nothing else — no file, no CURIE, no label — and the `is_a` +/// filter is the point rather than a convenience: walking subsumption together +/// with `part_of` derives FALSE ancestors (`A part_of B` and `B ⊑ C` does not +/// give `A ⊑ C`), so a reasoner without role composition must never be handed +/// the mixed set. Filtering here means a caller cannot make that mistake by +/// forgetting to. +/// +/// Borrows throughout: the pairs are decoded from the rows as the iterator +/// walks, nothing is gathered. +pub fn subsumptions(rows: &[crate::Row512]) -> impl Iterator + '_ { + rows.iter().flat_map(|row| { + let s = subject(&row.0); + EdgeLanes::new(&row.0) + .links_of(Predicate::IsA) + .map(move |p| (s, p)) + }) +} + #[cfg(test)] mod tests { use super::*; @@ -433,6 +463,55 @@ mod tests { ); } + #[test] + fn the_subsumption_projection_drops_every_non_is_a_edge() { + // A row with both an is_a and a part_of parent. Handing the mixed set + // to a reasoner without role composition derives false ancestors, so + // the projection must yield ONLY the is_a edge. + let mut row = row_with_degrees(&[(Predicate::IsA, 1), (Predicate::PartOf, 1)]); + row[0..16].copy_from_slice(&crate::pack_key(0x0303_0000, 2244)); + let mut tg = vec![ + (Predicate::IsA, t(Namespace::Uberon, 10)), + (Predicate::PartOf, t(Namespace::Uberon, 20)), + ]; + pack_edges(&mut row, &mut tg, 0x0000); + + // both edges are readable... + assert_eq!(EdgeLanes::new(&row).links().count(), 2); + // ...but only the subsumption is projected, and the subject is the row. + let rows = [crate::Row512(row)]; + let got: Vec<_> = subsumptions(&rows).collect(); + assert_eq!( + got, + vec![( + Link { + classid: 0x0303_0000, + num: 2244 + }, + Link { + classid: 0x0303_0000, + num: 10 + } + )], + "part_of must not reach a reasoner that lacks role composition" + ); + } + + #[test] + fn a_row_key_round_trips_through_the_family_identity_rail() { + // an oversize CURIE (> u16) must survive: the numeric rides the rail, + // high half in `family`. A truncating decode would return 4556 here. + let mut row = [0u8; NODE_ROW_STRIDE]; + row[0..16].copy_from_slice(&crate::pack_key(0x0301_0000, 700_092)); + assert_eq!( + subject(&row), + Link { + classid: 0x0301_0000, + num: 700_092 + } + ); + } + #[test] fn an_edge_lane_never_overwrites_the_entity_type_tenant() { let mut row = [0u8; NODE_ROW_STRIDE]; diff --git a/crates/ogar-obo/src/lib.rs b/crates/ogar-obo/src/lib.rs index a4956b8f..99b84c82 100644 --- a/crates/ogar-obo/src/lib.rs +++ b/crates/ogar-obo/src/lib.rs @@ -315,6 +315,19 @@ fn pack_key(classid: u32, identity: u32) -> [u8; 16] { k } +/// Read a row's key back as `(classid, CURIE numeric)` — the inverse of +/// `pack_key`, recombining the `family:identity` rail into the 24-bit numeric. +/// +/// This is the *whole* prerender property in one call: a reader learns which +/// class a row is and which instance, with **zero value decode**. +#[must_use] +pub const fn unpack_key(row: &[u8; NODE_ROW_STRIDE]) -> (u32, u32) { + let classid = u32::from_le_bytes([row[0], row[1], row[2], row[3]]); + let family = u16::from_le_bytes([row[12], row[13]]) as u32; + let identity = u16::from_le_bytes([row[14], row[15]]) as u32; + (classid, (family << 16) | identity) +} + /// Pack one node into its 512-byte row. /// /// The in-row `edges` block records the *degree* per predicate (a one-byte From b06f75870a6d221eb7629fdedf66d35a749a0029 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 8 Aug 2026 09:30:16 +0000 Subject: [PATCH 03/13] docs(ogar-obo): the edge-lane packing, its measurements, and its open ends MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Written down because the reasoning is not recoverable from the code: why the G2 4x24 carving carries the targets rather than the cheaper 6x(u8:u8) rails (a rail ref is basin-local and the bake has no basin — HEEL/HIP/TWIG are zero on all 68,797 rows while identities run past 8.9 million, so a one-byte ref has nothing to mean), and why identity does the opposite and rides the rail (the same 24 bits as an X:Y pair with headroom, which a flat u24 cannot be because it has no axis). Records the measurements the layout constant was chosen FROM, not after: max lane demand 16 against a 23-lane budget, and both packing assumptions checked on the real sources — CURIE numeric 0 never occurs, so the pad is safe, and every numeric fits 24 bits. Lane demand rather than out-degree is the binding constraint, because the (predicate, namespace) grouping fragments lanes. Names the three verification paths and what makes them independent, including the one that matters most: closing the lanes gives 738,651 transitive ancestors and 0 cycles, and the bake's own saturation computes the same two numbers from the out-of-line triple table — different representation, different code. Separates ogar_obo::Predicate from the ogar-ro crate explicitly, since the names invite conflation: the first is the byte palette carried on baked edges (the THINK arm, what the graph asserts), the second is a callable vocabulary at concept 0x0306 whose operands resolve against a basin-local codebook (the DO arm). They meet at the ontology, not in the bytes. Open ends are stated as open: the rails have no scheduled trigger, W-RO-5 is undecided by both sides, and degree saturation at 255 is untested because the measured maximum is 62 — if a source ever crosses it the histogram clamps while the lanes still hold the targets, and the two halves would disagree. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01KCGhDYoQBXs3poaR7sFuqp --- crates/ogar-obo/docs/EDGE-LANES.md | 196 +++++++++++++++++++++++++++++ 1 file changed, 196 insertions(+) create mode 100644 crates/ogar-obo/docs/EDGE-LANES.md diff --git a/crates/ogar-obo/docs/EDGE-LANES.md b/crates/ogar-obo/docs/EDGE-LANES.md new file mode 100644 index 00000000..7329b6da --- /dev/null +++ b/crates/ogar-obo/docs/EDGE-LANES.md @@ -0,0 +1,196 @@ +# Edge lanes — the `col_idx` half of the CSR + +> Status: **shipped**, measured on the full OBO core. +> Code: `crates/ogar-obo/src/edges.rs` · producer `bake()` · reader `EdgeLanes`. +> Probes: `examples/probe_edge_capacity.rs`, `examples/verify_edges.rs`. + +## The gap this closes + +A baked row is `key(16) | edges(16) | value(480)`. The `edges` block is a +per-predicate **degree histogram**: one byte per `Predicate` discriminant, +saturating at 255. It says *how many* targets a node has per predicate. + +It does not say *which*. In CSR terms the bake persisted `row_ptr` and dropped +`col_idx`. The consequence is sharp and was worth stating plainly before any +code was written: **a row could describe its own local shape and still not be +traversed.** Every traversal had to fall back to the out-of-line SPO triple +table, which is a different artifact with a different lifetime. + +This module puts the targets in the row. The degree histogram thereby stops +being decorative and becomes load-bearing — it is the index that tells a reader +which predicate each successive target belongs to. + +## The lane form + +A lane is the one form the substrate has: `classid(4) + 12 content bytes`, and +**the classid says how to read its own 12 bytes.** An edge lane reads its +content as the G2 `4×u24` carving — four literal links — and carries **the +target ontology's own row classid**. + +```text +lane: | classid: u32 | u24 | u24 | u24 | u24 | + ^ target ontology ^ four CURIE numerics +``` + +So a link is exactly the `(classid, identity)` pair a *key* carries. The same +binary search that resolves a key resolves a link; nothing new is invented to +point at a node. `edges::subject()` and `edges::Link` are deliberately the same +shape, because a subject and a target are the same kind of thing. + +### Why `4×u24` and not the `6×(u8:u8)` rails + +The L1 rails are the cheaper form and the eventual destination — but a rail ref +is **basin-local**, capped at 256 targets, and explicitly never a global +pointer. The bake has no basin: HEEL/HIP/TWIG and leaf are **zero on all 68,797 +rows** (the cascade is dormant, RESERVE-DON'T-RECLAIM), while identities run +past 8.9 million. With no basin to be local to, a one-byte ref has nothing to +mean. + +So the G2 carving is spent exactly where a rail cannot reach, and only there. +When an HHTL basin mint wakes routing, the rails become the cheaper in-basin +form; the offsets are already reserved and the migration costs no re-bake. + +This is also why the *identity* does **not** use G2: it rides the V3 +`family:identity` rail (`family = num >> 16`, `identity = num & 0xFFFF`), which +carries the same 24 bits as an `X:Y` pair with 8 bits of headroom — the thing a +flat u24 cannot do, because it has no axis. Sort order is unchanged, since +`family` holds the high bits, so ordering by `(classid, family, identity)` is +still ordering by `(classid, num)` and binary-search-by-key survives. + +## Layout + +The 480-byte value slab is 30 lanes. + +| lanes | use | +|---|---| +| 0–5 | reserved for further value tenants (RESERVE, DON'T RECLAIM) | +| 6 | `EntityType` tenant — a `u16` namespace ordinal at slab offset 96 | +| 7–29 | **edge targets** — 23 lanes × 4 links = **92 slots** | + +`EDGE_LANE_SLAB_OFFSET = 112`, `EDGE_LANE_COUNT = 23`, `EDGE_SLOT_COUNT = 92`. +A `const _: () = assert!(…)` pins that the edge lanes run to the end of the row +without overlapping a value tenant, so a future tenant carve cannot silently +collide with them. + +## The reader's contract + +Targets are emitted ascending by `(predicate, target namespace, numeric)`, and +**a lane never mixes target ontologies** — its classid names exactly one, so a +namespace change starts a new lane even mid-predicate. That fragmentation is +deliberate: it is what keeps a lane's classid from lying. + +Unused slots in a lane are `0`. That is unambiguous, and it is a *measured* +fact, not an assumption: `probe_edge_capacity` checks assumption **A1** — CURIE +numeric `0` never occurs — across the real sources, and it does not. + +A reader walks lanes in order and consumes `degree(p)` links for each predicate +`p` in ascending order, treating a `0` slot as "this lane is spent, advance". +That is `EdgeLanes`, which borrows the row and decodes a `u24` with a shift — +no allocation, no gathered window, nothing owned that the lane already holds. + +```rust +let lanes = EdgeLanes::new(&row.0); +lanes.degree(Predicate::IsA); // row_ptr: one byte, zero value decode +lanes.links_of(Predicate::IsA); // col_idx: borrowed iterator +``` + +The predicate is **not stored on the lane**. Recovering it from the histogram is +what makes the two halves one structure rather than two parallel encodings that +can disagree. + +## Measurements + +All from `probe_edge_capacity` and `verify_edges` on the full OBO core +(68,797 rows), run before the layout constant was chosen. + +| quantity | value | +|---|---| +| subjects with ≥1 edge | 68,789 | +| total triples | 186,029 | +| max out-degree (one row) | 62 | +| **max lane demand (one row)** | **16** (against a 23-lane budget) | +| rows exceeding 16 lanes | 0 | +| A1 — CURIE numeric `0` occurrences | 0 | +| A2 — numerics over 24 bits | 0 | +| links packed / dropped / rows overflowed | 186,029 / **0** / **0** | + +Lane demand, not out-degree, is the binding constraint — the `(predicate, +namespace)` grouping fragments lanes, so a 62-target row needs 16 lanes rather +than `ceil(62/4) = 16`… which here coincides, but does not in general, and the +probe measures the real thing rather than the convenient proxy. + +### No silent truncation + +`pack_edges` reports overflow in `PackStats` and writes nothing past the +budget; `bake` accumulates `links_dropped` / `rows_overflowed` so a driver can +**refuse to ship** an artifact that is missing edges. A packer that quietly +truncated would produce a graph that looks complete and is not. + +## Verification + +Three independent paths agree, which is the point — a single path agreeing with +itself would prove nothing. + +1. **Lane decode vs the triple table.** `verify_edges` reads links back out of + the row bytes and compares them as a multiset against the SPO triples the + bake built from the parsed source. Different code, same input. + + | ontology | rows | links read | `is_a` degree total | + |---|---:|---:|---:| + | RO | 4 | 3 | 2 | + | UBERON | 14,975 | 71,844 | 27,170 | + | HP | 19,836 | 50,284 | 31,284 | + | PATO | 1,887 | 3,389 | 2,909 | + | MONDO | 32,095 | 60,509 | 46,448 | + + All MATCH. The `is_a` totals also equal the degree bytes already present in + the previously-baked slabs, so the packing agrees with the artifact it + extends — that was the falsifier fixed *before* any code was written. + +2. **Closure vs saturation.** Reading `is_a` out of the lanes and closing it + (see `ogar-elk`) yields **738,651** transitive ancestors and **0** + equivalence cycles. `reason::saturate` computes 738,651 subsumption pairs + and 0 cycles from the out-of-line triple table. Two reasoners, two + representations, identical to the unit. + +3. **By hand.** RO is four rows and was checked against the source directly: + + ```text + RO:0002532 --Other--> RO:0002533 (relationship: RO:0002524 RO:0002533) + RO:0002533 --is_a---> RO:0002532 + RO:0002534 --is_a---> RO:0002532 + RO:0002577 --(none) (only parent BFO:0000040 — outside the core) + ``` + + `RO:0002577` carrying *zero* links is the interesting case: its degree byte + is 0, not a dangling 1, so the histogram and the lanes agree about an edge + that was correctly never emitted. + +## Relation to `ogar-ro` + +`ogar_obo::Predicate` and the `ogar-ro` crate are **different things** and +should not be conflated: + +- `ogar_obo::Predicate` (discriminants 1–7) is the byte palette **carried on + baked edges** — it indexes the degree histogram and orders the lanes. It is + the THINK arm: what the baked graph asserts. +- `ogar-ro` is a callable **vocabulary** (`RELATION_BODY_CONCEPT_ID = 0x0306`) + that mints RO/BFO predicates as `FnIndex` bytes so a template body can + *assert* an edge — `Call = (predicate : subject, object)`, a statement, not + an expression. It is the DO arm, and its operands resolve against a + basin-local `RELATION_TARGET_CODEBOOK`, not against CURIE numerics. + +They meet at the ontology, not in the bytes. + +## Open + +- **The rails are unused until a basin exists.** The migration path is named + above and costs no re-bake, but it has no scheduled trigger. +- **`LaneShape` / `CascadeShape` duplication (W-RO-5)** is an unresolved + cross-repo question recorded in `ogar-ro`'s crate docs, deliberately not + decided by either side. +- **Degree saturation** at 255 is untested against real data because the + measured maximum is 62. If a future source pushes a single predicate past + 255, the histogram silently clamps while the lanes would still hold the + targets — the two halves would disagree. A guard belongs here before any + source with that shape is baked. From 0c18571801e5122ae792b6422a4e490a46e2ff44 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 8 Aug 2026 10:35:32 +0000 Subject: [PATCH 04/13] =?UTF-8?q?S4:=20the=20lazy=20lens=20=E2=80=94=20rea?= =?UTF-8?q?son=20over=20borrowed=20rows,=20no=20index,=20no=20hashing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The plan said to reach for TypedGraph::traverse / hdr_bfs. Reading them, both take a concrete &GrBMatrix rather than a trait, and GrBMatrix's CsrStorage holds values: Vec at 16384 bits — roughly 2 KiB PER EDGE. Materialising the 107,813 is_a edges that way costs on the order of 200 MB to hold an adjacency the rows already contain, which is the opposite of a large hydrated table with a small resident set. traverse() also computes matrix.mxm(matrix), which is one hop of the SQUARE, not a walk from a source. So the plan was wrong on two counts and is corrected here: the lens IS the traversal, not an adapter into blasgraph. ogar-obo::spine — SpineLens<'a>, one borrowed field, in the house style of the workspace's other lenses. resolve() is a binary search over (classid, num), sound because the bake emits rows in that order: family holds the high half of the numeric, so ordering by (classid, family, identity) IS ordering by (classid, num). Verified over the shipped bake — 68,797 rows, zero inversions. is_sorted() exists because an unsorted slice does not panic, it silently finds nothing, so the precondition is checkable at the boundary where rows arrive. ogar-elk::lens — Parents (one visitor method, so a hop allocates nothing) and LensClosure. A blanket impl over closures keeps the seam free of a dependency edge in either direction: the crate owning the byte layout hands over a closure capturing its own view, and neither crate names the other's types. NO HASHING on this path. Dedup is an ordered insert, membership is a binary search. Not a micro-optimisation: addresses arrive already ordered from the bake, hashing discards that ordering, and the ordering has to be rebuilt before two ancestries can be intersected. Keeping it makes meet() a single merge pass and makes two runs produce identical output — the property a layer emitting facts has to have. Removing the hash sets took the full sweep from 7.8x the indexed walk to 4.8x. meet() is the Schnittpunkt: an x-hop cross-ontology lookup asks where reachable sets MEET, and meeting is an intersection over sorted addresses — no vectors, no per-edge payload, nothing serialized. The lens never caches. A transitive closure recomputes equal, which under the zero-copy law makes it a projection, and a projection is never stored however much work produced it. Asking twice walks twice, deliberately. Falsifier (examples/elk_lens_vs_closure, every class, no sampling): 68,786 classes compared, 0 ancestor sets differing, 738,651 transitive ancestors and 0 equivalence cycles from both the lens and the indexed closure. Cost is printed whether or not it flatters the lens: 0 build vs 19 ms, 4.8x on a full sweep, and the lens holds a slice where the index holds a map over 107,813 edges. KNOWN LIMIT, measured not assumed: meet() between a MONDO and an HPO class returns 0, because is_a never crosses ontologies. Cross-ontology Schnittpunkte need the cross-predicates (has_phenotype / has_anatomy / has_quality), which are present in the lanes — 78,216 of the 186,029 links — but which the closure deliberately does not walk, since mixing part_of into subsumption derives false ancestors. A typed cross-predicate walk is the next piece, not this one. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01KCGhDYoQBXs3poaR7sFuqp --- .../ogar-elk/examples/elk_lens_vs_closure.rs | 149 ++++++ crates/ogar-elk/src/lens.rs | 474 ++++++++++++++++++ crates/ogar-elk/src/lib.rs | 4 + crates/ogar-obo/src/lib.rs | 3 +- crates/ogar-obo/src/spine.rs | 263 ++++++++++ 5 files changed, 892 insertions(+), 1 deletion(-) create mode 100644 crates/ogar-elk/examples/elk_lens_vs_closure.rs create mode 100644 crates/ogar-elk/src/lens.rs create mode 100644 crates/ogar-obo/src/spine.rs diff --git a/crates/ogar-elk/examples/elk_lens_vs_closure.rs b/crates/ogar-elk/examples/elk_lens_vs_closure.rs new file mode 100644 index 00000000..740d8d65 --- /dev/null +++ b/crates/ogar-elk/examples/elk_lens_vs_closure.rs @@ -0,0 +1,149 @@ +//! The falsifier for the lazy lens, plus an honest cost comparison. +//! +//! The lens exists to reason over a hydrated table without copying it. That is +//! only worth anything if it gives the **same answers** as the indexed +//! [`Closure`], so this runs both over the real bake and compares them class by +//! class — not on a sample, on every class that has a parent. +//! +//! It also reports what the lens costs. The lens is not free and is not +//! uniformly faster; the numbers below are printed whether or not they flatter +//! it, because a lens that quietly lost on every axis would still pass a +//! correctness check. +//! +//! Usage: `cargo run --release -p ogar-elk --example elk_lens_vs_closure -- [slab.soa]` + +use ogar_elk::{ClassAddr, Closure, LensClosure, Subsumption}; +use ogar_obo::{NODE_ROW_STRIDE, Row512, edges, spine::SpineLens}; + +fn main() { + let path = std::env::args() + .nth(1) + .unwrap_or_else(|| "/tmp/obo/obo-core.soa".to_string()); + let bytes = std::fs::read(&path).unwrap_or_else(|e| panic!("read {path}: {e}")); + assert!(bytes.len().is_multiple_of(NODE_ROW_STRIDE), "not a slab"); + + // Harness copy into an aligned buffer. On the real path Lance hands over an + // arrow-aligned FixedSizeBinary(512) column and this step does not exist. + let n = bytes.len() / NODE_ROW_STRIDE; + let mut rows = vec![Row512::zeroed(); n]; + for (i, r) in rows.iter_mut().enumerate() { + r.0.copy_from_slice(&bytes[i * NODE_ROW_STRIDE..(i + 1) * NODE_ROW_STRIDE]); + } + println!("slab {path}\n rows {n}"); + + // ── the lens: borrow, assert the precondition, wrap in a Parents closure ── + let lens = SpineLens::new(&rows); + assert!( + lens.is_sorted(), + "binary-search resolve requires ascending (classid, num)" + ); + let parents = |a: ClassAddr, visit: &mut dyn FnMut(ClassAddr)| { + lens.parents_of(a.classid, a.identity, &mut |c, num| { + visit(ClassAddr::new(c, num)); + }); + }; + let lc = LensClosure::new(&parents); + + // ── the indexed closure, for comparison ── + let t = std::time::Instant::now(); + let cl = Closure::from_asserted(edges::subsumptions(&rows).map(|(s, p)| { + Subsumption::new( + ClassAddr::new(s.classid, s.num), + ClassAddr::new(p.classid, p.num), + ) + })); + let build = t.elapsed(); + let subs: Vec = cl.subclasses().collect(); + println!( + " indexed closure: {} asserted edges over {} subclasses, built in {build:?}", + cl.asserted_len(), + subs.len() + ); + println!(" lens: 0 edges indexed, 0 build time — it borrows the rows\n"); + + // ── THE FALSIFIER: identical ancestor sets, every class, no sampling ── + let t = std::time::Instant::now(); + let mut total = 0usize; + let mut diverged = 0usize; + let mut first_bad: Option = None; + for &c in &subs { + let a = lc.supers_of(c); + let mut b: Vec = cl.supers_of(c).into_iter().collect(); + b.sort_unstable(); + if a != b { + diverged += 1; + if first_bad.is_none() { + first_bad = Some(c); + } + } + total += a.len(); + } + let lens_walk = t.elapsed(); + + let t = std::time::Instant::now(); + let control: usize = subs.iter().map(|&c| cl.supers_of(c).len()).sum(); + let idx_walk = t.elapsed(); + + println!("=== FALSIFIER ==="); + println!(" classes compared : {}", subs.len()); + println!( + " ancestor sets differing: {diverged} (must be 0){}", + first_bad.map_or(String::new(), |c| format!(" first: {c:?}")) + ); + println!(" transitive ancestors : lens {total} | indexed {control}"); + assert_eq!(diverged, 0, "the lens disagrees with the indexed closure"); + assert_eq!(total, control, "ancestor totals differ"); + + // Cycles: the lens is per-class by design (it has no membership roll). + let lens_cycles = subs.iter().filter(|&&c| lc.reaches_itself(c)).count(); + let idx_cycles = cl.equivalence_cycles().len(); + println!(" equivalence cycles : lens {lens_cycles} | indexed {idx_cycles}"); + assert_eq!(lens_cycles, idx_cycles); + + println!("\n=== COST (honest) ==="); + println!(" build : lens 0s | indexed {build:?}"); + println!(" full sweep : lens {lens_walk:?} | indexed {idx_walk:?}"); + let ratio = lens_walk.as_secs_f64() / idx_walk.as_secs_f64().max(1e-9); + println!(" sweep ratio : lens is {ratio:.1}x the indexed walk"); + println!( + " residency : lens holds a &[Row512] + a depth cap; the index holds a \n\ + \x20 HashMap over {} edges for the life of the query set", + cl.asserted_len() + ); + + // ── Schnittpunkte: the cross-ontology x-hop lookup ── + println!("\n=== SCHNITTPUNKTE ==="); + let mut seeds: Vec = Vec::new(); + let mut by_class: std::collections::BTreeMap = Default::default(); + for &c in &subs { + by_class.entry(c.classid).or_insert(c); + } + for (_, c) in by_class.iter().take(2) { + seeds.push(*c); + } + if seeds.len() == 2 { + let m = lc.meet(&seeds); + println!( + " meet({:?}) -> {} shared ancestor(s)", + seeds + .iter() + .map(|s| format!("0x{:08x}:{}", s.classid, s.identity)) + .collect::>(), + m.len() + ); + for a in m.iter().take(5) { + println!(" 0x{:08x}:{}", a.classid, a.identity); + } + // a class always meets itself in its own ancestry — a control that the + // primitive is not simply returning empty for everything + let self_meet = lc.meet(&[seeds[0], seeds[0]]); + println!( + " control meet(x,x) -> {} (equals |ancestors(x)| = {})", + self_meet.len(), + lc.supers_of(seeds[0]).len() + ); + assert_eq!(self_meet.len(), lc.supers_of(seeds[0]).len()); + } + + println!("\nLENS VERIFIED: identical answers, zero index built."); +} diff --git a/crates/ogar-elk/src/lens.rs b/crates/ogar-elk/src/lens.rs new file mode 100644 index 00000000..84a3aa68 --- /dev/null +++ b/crates/ogar-elk/src/lens.rs @@ -0,0 +1,474 @@ +//! The **lazy lens** seam — reasoning against borrowed rows, with no index built. +//! +//! [`Closure`] indexes its edges into a `HashMap` up front. That is the right +//! shape when the asserted set is already in memory and will be queried many +//! times, and the wrong shape when the substrate is a hydrated columnar table +//! you would rather not copy: building the index costs a pass over every edge +//! and holds a map proportional to them, before the first question is asked. +//! +//! This module is the other end. A [`Parents`] implementation answers *one* +//! question — who are this class's asserted superclasses — and [`LensClosure`] +//! computes entailment on top of it, touching only the classes a query actually +//! reaches. +//! +//! # No hashing, anywhere on this path +//! +//! Dedup is an ordered insert and membership is a binary search. Nothing here +//! hashes. That is not a micro-optimisation: the addresses arrive already +//! ordered from the bake, hashing throws that ordering away, and it has to be +//! rebuilt before two ancestries can be intersected. Keeping the order means +//! [`LensClosure::meet`] is one merge pass, and it means two runs produce +//! byte-identical output — the property a layer that emits *facts* has to have. +//! +//! # The lens never caches, and that is the law rather than an oversight +//! +//! A transitive closure is a deterministic function of the asserted edges: it +//! recomputes equal, every time. Under the workspace's zero-copy law that makes +//! it a **projection** — `output_rung == max(input_rungs)` — and a projection is +//! never stored, however much arithmetic produced it. So [`LensClosure`] holds +//! no memo table, no visited cache between calls, and no adjacency copy. It +//! holds a borrowed reference and a depth cap. +//! +//! The cost of that is real and is not hidden: repeated queries redo work. What +//! it buys is that nothing is resident between queries, which is the point when +//! the rows live in a hydrated table an order of magnitude larger than RAM. +//! +//! # What it does NOT change +//! +//! The answers. [`LensClosure::supers_of`] and [`LensClosure::entails`] return +//! exactly what [`Closure`]'s equivalents return over the same asserted set — +//! asserted in the test module, and measured against the real bake in +//! `examples/elk_lens_vs_closure.rs`. A lens that were *cheaper* and *different* +//! would be a different reasoner wearing this one's name. +//! +//! # Where the lens genuinely wins +//! +//! [`LensClosure::entails`] **early-exits** on the first hit. `Closure::entails` +//! is `sub == sup || self.supers_of(sub).contains(&sup)`, which materialises the +//! whole ancestor set before testing membership. For a yes-answer near the query +//! class, the lens stops there; for a no-answer both walk the same ground. + +use std::collections::VecDeque; + +use crate::{ClassAddr, DEFAULT_DEPTH_CAP}; + +/// A source of **asserted** superclasses for a class. +/// +/// One method, taking a visitor rather than returning a collection: an +/// implementation over borrowed rows can then decode links in place and hand +/// them out one at a time, with nothing allocated per call. Returning +/// `Vec` would put an allocation on the hot path of every hop and +/// quietly undo the reason this trait exists. +/// +/// **Asserted, not entailed.** An implementation reports the direct parents it +/// holds and performs no transitive reasoning of its own — that is +/// [`LensClosure`]'s job, and splitting it this way is what lets the same lens +/// serve a closure, a one-hop query, or a cycle check without any of them +/// re-deriving what the others did. +pub trait Parents { + /// Visit each asserted superclass of `c`, in any order. + /// + /// Called once per class per walk, so an implementation should avoid + /// allocating. Duplicates are permitted; [`LensClosure`] de-duplicates. + fn parents_of(&self, c: ClassAddr, visit: &mut dyn FnMut(ClassAddr)); +} + +/// Any closure of the right shape is a [`Parents`] source. +/// +/// This is what keeps the seam dependency-free in both directions: the crate +/// that owns the byte layout can hand over a closure capturing its own borrowed +/// view, and neither crate has to name the other's types. There is no adapter +/// struct to write and no `ogar-obo`/`ogar-elk` dependency edge in either +/// direction. +impl Parents for F +where + F: Fn(ClassAddr, &mut dyn FnMut(ClassAddr)), +{ + #[inline] + fn parents_of(&self, c: ClassAddr, visit: &mut dyn FnMut(ClassAddr)) { + self(c, visit); + } +} + +/// Entailment computed against a [`Parents`] source, materialising nothing. +/// +/// Borrows its source; holds no state beyond the depth cap. Constructing one is +/// free, so it is meant to be created per query rather than kept. +#[derive(Debug, Clone, Copy)] +pub struct LensClosure<'p, P: Parents + ?Sized> { + parents: &'p P, + depth_cap: usize, +} + +impl<'p, P: Parents + ?Sized> LensClosure<'p, P> { + /// Borrow a parent source at the default depth cap. + #[must_use] + pub const fn new(parents: &'p P) -> Self { + Self { + parents, + depth_cap: DEFAULT_DEPTH_CAP, + } + } + + /// Override the transitivity depth guard. + #[must_use] + pub const fn with_depth_cap(mut self, cap: usize) -> Self { + self.depth_cap = cap; + self + } + + /// The depth guard in force — so a caller can tell "not entailed" from + /// "the walk stopped", which are different answers. + #[must_use] + pub const fn depth_cap(&self) -> usize { + self.depth_cap + } + + /// **R1 + R2** — every superclass of `c`, transitively. + /// + /// Breadth-first, so the walk terminates at `depth_cap` hops rather than at + /// `depth_cap` nodes. Excludes `c` itself unless a cycle genuinely returns + /// to it — the signal [`Self::reaches_itself`] reads. + /// + /// The returned set is the caller's answer, not a cache: it is dropped when + /// the caller drops it, and asking twice walks twice. + #[must_use] + pub fn supers_of(&self, c: ClassAddr) -> Vec { + let mut seen: Vec = Vec::new(); + let mut queue = VecDeque::from([(c, 0usize)]); + while let Some((node, depth)) = queue.pop_front() { + if depth >= self.depth_cap { + continue; + } + self.parents.parents_of(node, &mut |p| { + // Ordered insert, not a hash probe. The result stays sorted by + // construction, which is what makes `meet` a merge pass — and + // it is deterministic run to run, which a hash set is not. + if let Err(at) = seen.binary_search(&p) { + seen.insert(at, p); + queue.push_back((p, depth + 1)); + } + }); + } + seen + } + + /// **Does `sub ⊑ sup` follow?** Reflexive per R1. + /// + /// Early-exits on the first hit instead of building the full ancestor set, + /// which is the one place the lens is strictly cheaper than the indexed + /// [`Closure`](crate::Closure) rather than merely leaner. + #[must_use] + pub fn entails(&self, sub: ClassAddr, sup: ClassAddr) -> bool { + if sub == sup { + return true; + } + let mut seen: Vec = Vec::new(); + let mut queue = VecDeque::from([(sub, 0usize)]); + while let Some((node, depth)) = queue.pop_front() { + if depth >= self.depth_cap { + continue; + } + let mut hit = false; + self.parents.parents_of(node, &mut |p| { + if p == sup { + hit = true; + } + if let Err(at) = seen.binary_search(&p) { + seen.insert(at, p); + queue.push_back((p, depth + 1)); + } + }); + if hit { + return true; + } + } + false + } + + /// Ancestors as a **sorted, de-duplicated** address list. + /// + /// The shape [`Self::supers_of`] should have had from the start. A hash set + /// discards the ordering the bake already provides — rows are emitted + /// ascending by `(classid, identity)` — and that ordering is exactly what + /// makes [`meet`](Self::meet) a single merge pass instead of a lookup + /// storm. Sorted output is also deterministic run to run, which a hash set + /// is not, and determinism is the property a fact-producing layer is + /// supposed to have. + #[must_use] + pub fn ancestors_sorted(&self, c: ClassAddr) -> Vec { + // already sorted by construction — supers_of does ordered inserts + self.supers_of(c) + } + + /// **Schnittpunkte** — the addresses reachable from *every* seed. + /// + /// This is the multi-hop cross-ontology lookup, and it is deliberately not + /// a matrix product: an x-hop question across ontologies asks where the + /// reachable sets MEET, and meeting is an intersection over sorted + /// addresses — one merge pass per seed, no vectors, no per-edge payload, + /// nothing serialized. + /// + /// Returns sorted addresses. An empty seed list yields nothing (there is no + /// meaningful "intersection of no sets" here — the identity would be every + /// address in existence, which is not an answer a caller can use). + /// + /// The seeds may sit in different ontologies; the result names wherever + /// their ancestries coincide, which is the fact the caller is after. + #[must_use] + pub fn meet(&self, seeds: &[ClassAddr]) -> Vec { + let Some((first, rest)) = seeds.split_first() else { + return Vec::new(); + }; + let mut acc = self.ancestors_sorted(*first); + for s in rest { + if acc.is_empty() { + break; + } + acc = intersect_sorted(&acc, &self.ancestors_sorted(*s)); + } + acc + } + + /// Whether `c` reaches itself — an equivalence cycle through `c`. + /// + /// Per-class rather than whole-graph: a lens has no membership roll to + /// enumerate, and inventing one would mean materialising the very index + /// this module exists to avoid. A caller that wants every cycle iterates + /// the classes it cares about; a caller that has the full roll can use + /// [`Closure::equivalence_cycles`](crate::Closure::equivalence_cycles). + #[must_use] + pub fn reaches_itself(&self, c: ClassAddr) -> bool { + self.supers_of(c).binary_search(&c).is_ok() + } +} + +/// Intersect two sorted, de-duplicated address lists in one pass. +/// +/// Linear in the inputs and allocation-free apart from the result, which is why +/// the sorted representation is worth keeping: the same job over hash sets is a +/// probe per element and returns in arbitrary order. +#[must_use] +pub fn intersect_sorted(a: &[ClassAddr], b: &[ClassAddr]) -> Vec { + let (mut i, mut j) = (0usize, 0usize); + let mut out = Vec::new(); + while i < a.len() && j < b.len() { + match a[i].cmp(&b[j]) { + core::cmp::Ordering::Less => i += 1, + core::cmp::Ordering::Greater => j += 1, + core::cmp::Ordering::Equal => { + out.push(a[i]); + i += 1; + j += 1; + } + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{Closure, Subsumption}; + + const NS: u32 = 0x0301_0000; + + fn c(id: u32) -> ClassAddr { + ClassAddr::new(NS, id) + } + + /// A deliberately dumb source: a linear scan of asserted pairs. Slow on + /// purpose — it shares no code with `Closure`'s HashMap index, so agreement + /// between the two is evidence rather than tautology. + struct Pairs(Vec<(u32, u32)>); + + impl Parents for Pairs { + fn parents_of(&self, x: ClassAddr, visit: &mut dyn FnMut(ClassAddr)) { + for (a, b) in &self.0 { + if c(*a) == x { + visit(c(*b)); + } + } + } + } + + fn both(edges: &[(u32, u32)]) -> (Closure, Pairs) { + let cl = Closure::from_asserted(edges.iter().map(|(a, b)| Subsumption::new(c(*a), c(*b)))); + (cl, Pairs(edges.to_vec())) + } + + #[test] + fn the_lens_agrees_with_the_indexed_closure() { + let edges = [(1, 2), (2, 3), (3, 4), (5, 3), (6, 7)]; + let (cl, src) = both(&edges); + let lens = LensClosure::new(&src); + for i in 1..=8u32 { + let mut want: Vec<_> = cl.supers_of(c(i)).into_iter().collect(); + want.sort_unstable(); + assert_eq!(lens.supers_of(c(i)), want, "ancestor sets diverge at {i}"); + for j in 1..=8u32 { + assert_eq!( + lens.entails(c(i), c(j)), + cl.entails(c(i), c(j)), + "entailment diverges at {i} -> {j}" + ); + } + } + } + + /// Anti-vacuity for the test above: the fixture must contain both + /// entailed and NOT-entailed pairs, or agreement proves nothing. + #[test] + fn the_agreement_fixture_discriminates() { + let edges = [(1, 2), (2, 3), (3, 4), (5, 3), (6, 7)]; + let (_, src) = both(&edges); + let lens = LensClosure::new(&src); + assert!(lens.entails(c(1), c(4)), "derived through three hops"); + assert!( + !lens.entails(c(1), c(7)), + "must NOT entail a separate component" + ); + assert!(!lens.entails(c(4), c(1)), "and must not run the wrong way"); + let reached: usize = (1..=8u32).map(|i| lens.supers_of(c(i)).len()).sum(); + assert!( + reached > 0 && reached < 8 * 8, + "fixture is neither empty nor total" + ); + } + + #[test] + fn reflexivity_holds_with_no_edges_at_all() { + let src = Pairs(vec![]); + let lens = LensClosure::new(&src); + assert!(lens.entails(c(7), c(7))); + assert!(lens.supers_of(c(7)).is_empty()); + } + + #[test] + fn a_cycle_is_seen_and_a_clean_spine_is_not() { + let cyc = Pairs(vec![(1, 2), (2, 1)]); + let lens = LensClosure::new(&cyc); + assert!(lens.reaches_itself(c(1))); + assert!(lens.reaches_itself(c(2))); + + // can-stay-silent, over a NON-trivial acyclic graph + let clean = Pairs(vec![(1, 2), (2, 3), (1, 3), (4, 3)]); + let lens = LensClosure::new(&clean); + for i in 1..=4u32 { + assert!(!lens.reaches_itself(c(i)), "{i} is not on a cycle"); + } + } + + #[test] + fn the_depth_cap_binds_and_is_visible() { + let chain: Vec<(u32, u32)> = (1..10).map(|i| (i, i + 1)).collect(); + let src = Pairs(chain); + let shallow = LensClosure::new(&src).with_depth_cap(2); + assert_eq!(shallow.depth_cap(), 2); + assert!(!shallow.entails(c(1), c(10)), "beyond the cap"); + assert!(shallow.entails(c(1), c(3)), "within the cap"); + assert!( + LensClosure::new(&src).entails(c(1), c(10)), + "reachable at the default cap" + ); + } + + #[test] + fn intersect_is_a_single_merge_pass_and_can_return_nothing() { + let a: Vec = [1u32, 3, 5, 7].iter().map(|i| c(*i)).collect(); + let b: Vec = [3u32, 4, 5, 9].iter().map(|i| c(*i)).collect(); + assert_eq!(intersect_sorted(&a, &b), vec![c(3), c(5)]); + // anti-vacuity: disjoint inputs must yield empty, not everything + let d: Vec = [2u32, 6].iter().map(|i| c(*i)).collect(); + assert!(intersect_sorted(&a, &d).is_empty()); + assert!(intersect_sorted(&a, &[]).is_empty()); + } + + /// The Schnittpunkt: two classes in DIFFERENT namespaces whose ancestries + /// coincide at a shared address. This is the cross-ontology x-hop lookup, + /// and it is an intersection — no vectors, no matrix product. + #[test] + fn meet_finds_where_two_ontologies_coincide() { + let other = 0x0302_0000; + let shared = ClassAddr::new(NS, 100); + let f = move |x: ClassAddr, visit: &mut dyn FnMut(ClassAddr)| { + // MONDO:1 -> MONDO:100 ; HP:1 -> MONDO:100 ; MONDO:2 -> MONDO:200 + if x == ClassAddr::new(NS, 1) || x == ClassAddr::new(other, 1) { + visit(shared); + } + if x == ClassAddr::new(NS, 2) { + visit(ClassAddr::new(NS, 200)); + } + }; + let lens = LensClosure::new(&f); + assert_eq!( + lens.meet(&[ClassAddr::new(NS, 1), ClassAddr::new(other, 1)]), + vec![shared], + "the two ancestries meet at exactly one address" + ); + // can-stay-silent: seeds that share nothing produce no Schnittpunkt + assert!( + lens.meet(&[ClassAddr::new(NS, 1), ClassAddr::new(NS, 2)]) + .is_empty(), + "disjoint ancestries must not manufacture a meeting point" + ); + assert!(lens.meet(&[]).is_empty(), "no seeds is not 'everything'"); + } + + #[test] + fn ancestors_sorted_is_sorted_deduped_and_agrees_with_the_set() { + let edges = [(1, 2), (2, 3), (1, 3), (3, 4)]; + let (cl, src) = both(&edges); + let lens = LensClosure::new(&src); + let v = lens.ancestors_sorted(c(1)); + assert!(v.windows(2).all(|w| w[0] < w[1]), "sorted and deduped"); + let mut want = cl.supers_of(c(1)).into_iter().collect::>(); + want.sort_unstable(); + assert_eq!(v, want, "same content as the indexed closure"); + } + + /// A closure is a `Parents` source with no adapter — the property that + /// keeps the seam dependency-free in both directions. + #[test] + fn a_bare_closure_is_a_parent_source() { + let f = |x: ClassAddr, visit: &mut dyn FnMut(ClassAddr)| { + if x == c(1) { + visit(c(2)); + } + if x == c(2) { + visit(c(3)); + } + }; + let lens = LensClosure::new(&f); + assert!(lens.entails(c(1), c(3))); + assert!(!lens.entails(c(3), c(1))); + } + + /// `entails` must not walk further than it has to. The source counts its + /// own calls, so "early exit" is measured rather than asserted. + #[test] + fn entails_early_exits_instead_of_building_the_whole_set() { + use std::cell::Cell; + let calls = Cell::new(0usize); + let chain: Vec<(u32, u32)> = (1..40).map(|i| (i, i + 1)).collect(); + let counting = |x: ClassAddr, visit: &mut dyn FnMut(ClassAddr)| { + calls.set(calls.get() + 1); + for (a, b) in &chain { + if c(*a) == x { + visit(c(*b)); + } + } + }; + let lens = LensClosure::new(&counting); + + assert!(lens.entails(c(1), c(3))); + let near = calls.get(); + calls.set(0); + let _ = lens.supers_of(c(1)); + let full = calls.get(); + assert!( + near < full, + "early exit must touch fewer classes than the full walk ({near} vs {full})" + ); + } +} diff --git a/crates/ogar-elk/src/lib.rs b/crates/ogar-elk/src/lib.rs index 071bc3b5..f2c70e1d 100644 --- a/crates/ogar-elk/src/lib.rs +++ b/crates/ogar-elk/src/lib.rs @@ -89,6 +89,10 @@ #![forbid(unsafe_code)] +pub mod lens; + +pub use lens::{LensClosure, Parents}; + use std::collections::{HashMap, HashSet, VecDeque}; /// The **join key** the pre-bake reconciliation uses: `(classid, identity)`. diff --git a/crates/ogar-obo/src/lib.rs b/crates/ogar-obo/src/lib.rs index 99b84c82..c9ea6c3d 100644 --- a/crates/ogar-obo/src/lib.rs +++ b/crates/ogar-obo/src/lib.rs @@ -43,6 +43,7 @@ pub mod crosswalk; pub mod edges; pub mod reason; +pub mod spine; /// Row stride of the canonical SoA node — `key(16) + edges(16) + value(480)`. /// Mirrors `lance_graph_contract::canonical_node::NODE_ROW_STRIDE`. @@ -281,7 +282,7 @@ impl Row512 { /// and family stay `0` in this first bake — the zero-fallback ladder means /// `identity` alone discriminates until an HHTL basin mint wakes routing /// (RESERVE, DON'T RECLAIM: fixed offsets, zero layout change later). -fn pack_key(classid: u32, identity: u32) -> [u8; 16] { +pub(crate) fn pack_key(classid: u32, identity: u32) -> [u8; 16] { let mut k = [0u8; 16]; k[0..4].copy_from_slice(&classid.to_le_bytes()); // 4..10 HEEL/HIP/TWIG = 0 (dormant cascade) diff --git a/crates/ogar-obo/src/spine.rs b/crates/ogar-obo/src/spine.rs new file mode 100644 index 00000000..984b0014 --- /dev/null +++ b/crates/ogar-obo/src/spine.rs @@ -0,0 +1,263 @@ +//! `SpineLens` — a borrowed view that answers "who are this class's parents?" +//! straight out of baked rows, with no adjacency index built. +//! +//! # Why this exists rather than an adjacency copy +//! +//! The obvious way to traverse a baked graph is to read every edge once into an +//! adjacency structure and walk that. Measured, for this substrate, that is the +//! expensive option: `lance-graph`'s `GrBMatrix` carries a `Vec` of +//! 16 384-bit values — ~2 KiB **per edge** — so materialising the 107 813 `is_a` +//! edges as a matrix costs on the order of 200 MB to hold an adjacency the rows +//! already contain. That is the opposite of the deployment's whole shape (a +//! large hydrated table, a small resident set), and it is also what the +//! workspace's zero-copy law forbids on principle rather than on cost. +//! +//! So the lens does not build anything. It borrows the rows and answers each +//! question by locating a row and decoding its lanes in place. +//! +//! # The two steps, and what each costs +//! +//! 1. **Resolve** — `(classid, num)` → row index, by binary search. Sound +//! because the bake emits rows in ascending `(classid, num)`: `family` holds +//! the high half of the numeric, so ordering by `(classid, family, identity)` +//! *is* ordering by `(classid, num)`. Verified over the shipped bake — all +//! 68 797 rows strictly ascending, zero inversions. ~17 probes at that size. +//! 2. **Walk** — decode that row's `is_a` links from its edge lanes +//! ([`crate::edges::EdgeLanes`]), a shift per link, nothing allocated. +//! +//! Against an indexed closure the trade is explicit: a `HashMap` gives O(1) +//! hops after paying a full pass over every edge and holding a map proportional +//! to them; the lens pays O(log n) per hop and holds nothing. Which wins depends +//! on how many questions are asked against how much memory is available, and +//! `examples/elk_lens_vs_closure.rs` measures it rather than asserting it. +//! +//! # It borrows, and it does not cache +//! +//! One field, a shared slice, in the house style of the workspace's other +//! lenses. No memo table: a transitive closure recomputes equal, which under +//! the zero-copy law makes it a projection, and a projection is never stored +//! however much work produced it. Asking twice walks twice, deliberately. + +use crate::{NODE_ROW_STRIDE, Predicate, Row512, edges::EdgeLanes, unpack_key}; + +/// A borrowed view over baked rows that resolves addresses and reads parents. +/// +/// Holds a shared slice and nothing else. Constructing one is free; it is meant +/// to be made where it is used rather than kept. +#[derive(Clone, Copy)] +pub struct SpineLens<'a> { + rows: &'a [Row512], +} + +impl core::fmt::Debug for SpineLens<'_> { + /// Prints the row COUNT, never the rows. A lens over a hydrated table can + /// borrow tens of megabytes; a derived `Debug` would render all of it. + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("SpineLens") + .field("rows", &self.rows.len()) + .finish() + } +} + +impl<'a> SpineLens<'a> { + /// Borrow a slice of baked rows. + /// + /// The rows **must** be in the bake's own order — ascending `(classid, + /// num)` — because [`Self::resolve`] binary-searches them. Handing over an + /// unsorted slice does not panic; it silently fails to find rows, which is + /// why [`Self::is_sorted`] exists and why the example asserts it before use. + #[must_use] + pub const fn new(rows: &'a [Row512]) -> Self { + Self { rows } + } + + /// Number of rows in view. + #[must_use] + pub const fn len(&self) -> usize { + self.rows.len() + } + + /// Whether the view is empty. + #[must_use] + pub const fn is_empty(&self) -> bool { + self.rows.is_empty() + } + + /// The `(classid, num)` address of row `i`, or `None` if out of range. + #[must_use] + pub fn address_at(&self, i: usize) -> Option<(u32, u32)> { + self.rows.get(i).map(|r| unpack_key(&r.0)) + } + + /// Check the precondition [`Self::resolve`] relies on: strictly ascending + /// `(classid, num)`. + /// + /// O(n) and therefore not called per query — it is for a caller to assert + /// once at the boundary where rows arrive, so an unsorted slice fails + /// loudly there instead of returning empty answers everywhere else. + #[must_use] + pub fn is_sorted(&self) -> bool { + self.rows + .windows(2) + .all(|w| unpack_key(&w[0].0) < unpack_key(&w[1].0)) + } + + /// Locate the row for an address by binary search. + /// + /// `None` means no such row is in view — which is a real and expected + /// answer, not an error: an edge may point at a term outside the loaded + /// set, and the walk simply stops there rather than inventing a node. + #[must_use] + pub fn resolve(&self, classid: u32, num: u32) -> Option { + self.rows + .binary_search_by(|r| unpack_key(&r.0).cmp(&(classid, num))) + .ok() + } + + /// This row's out-degree for one predicate — the CSR `row_ptr` entry, read + /// from the edge block with **no value decode at all**. + #[must_use] + pub fn degree_at(&self, i: usize, p: Predicate) -> u8 { + self.rows + .get(i) + .map_or(0, |r| EdgeLanes::new(&r.0).degree(p)) + } + + /// Visit every link of one predicate for the row at `i`, decoded in place. + pub fn links_at(&self, i: usize, p: Predicate, visit: &mut dyn FnMut(u32, u32)) { + let Some(row) = self.rows.get(i) else { return }; + for l in EdgeLanes::new(&row.0).links_of(p) { + visit(l.classid, l.num); + } + } + + /// Visit the **asserted superclasses** of an address: resolve, then walk + /// that row's `is_a` links. + /// + /// `is_a` only, and that is the safety rather than a convenience — walking + /// subsumption together with `part_of` derives false ancestors (`A part_of + /// B` and `B ⊑ C` does not give `A ⊑ C`), and a reasoner without role + /// composition must never be handed the mixed set. + /// + /// An address with no row in view yields nothing, silently: an edge into an + /// unloaded ontology is a boundary, not a fault. + pub fn parents_of(&self, classid: u32, num: u32, visit: &mut dyn FnMut(u32, u32)) { + if let Some(i) = self.resolve(classid, num) { + self.links_at(i, Predicate::IsA, visit); + } + } +} + +/// Reinterpret a byte buffer as rows, if it is correctly sized and aligned. +/// +/// Returns `None` when the length is not a whole number of rows or the buffer +/// is not 64-aligned — the same gate `rows_from_le_bytes` applies, restated +/// here so a lens caller has one call rather than two. +#[must_use] +pub fn lens_from_le_bytes(bytes: &[u8]) -> Option> { + if !bytes.len().is_multiple_of(NODE_ROW_STRIDE) { + return None; + } + crate::rows_from_le_bytes(bytes).map(SpineLens::new) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{Namespace, TermId, edges::pack_edges}; + + const MONDO: u32 = 0x0301_0000; + + fn row(num: u32, parents: &[u32]) -> Row512 { + let mut r = Row512::zeroed(); + r.0[0..16].copy_from_slice(&crate::pack_key(MONDO, num)); + r.0[crate::EDGES_OFFSET + Predicate::IsA as usize] = parents.len() as u8; + let mut tg: Vec<_> = parents + .iter() + .map(|p| { + ( + Predicate::IsA, + TermId { + ns: Namespace::Mondo as u8, + num: *p, + }, + ) + }) + .collect(); + pack_edges(&mut r.0, &mut tg, 0x0000); + r + } + + /// 1 -> 2 -> 3, plus an isolated 9. + fn fixture() -> Vec { + vec![row(1, &[2]), row(2, &[3]), row(3, &[]), row(9, &[])] + } + + #[test] + fn resolve_finds_every_row_and_misses_absent_ones() { + let rows = fixture(); + let lens = SpineLens::new(&rows); + assert!(lens.is_sorted()); + for (i, n) in [1u32, 2, 3, 9].iter().enumerate() { + assert_eq!(lens.resolve(MONDO, *n), Some(i)); + } + // anti-vacuity: resolve must be able to say no + assert_eq!(lens.resolve(MONDO, 4), None, "absent numeric"); + assert_eq!( + lens.resolve(0x0302_0000, 1), + None, + "right numeric, wrong class" + ); + } + + #[test] + fn parents_come_from_the_lanes_and_stop_at_the_boundary() { + let rows = fixture(); + let lens = SpineLens::new(&rows); + let mut got = Vec::new(); + lens.parents_of(MONDO, 1, &mut |c, n| got.push((c, n))); + assert_eq!(got, vec![(MONDO, 2)]); + + got.clear(); + lens.parents_of(MONDO, 3, &mut |c, n| got.push((c, n))); + assert!(got.is_empty(), "a root has no parents"); + + got.clear(); + lens.parents_of(MONDO, 404, &mut |c, n| got.push((c, n))); + assert!( + got.is_empty(), + "an unresolvable address yields nothing, silently" + ); + } + + #[test] + fn degree_reads_without_decoding_any_value() { + let rows = fixture(); + let lens = SpineLens::new(&rows); + assert_eq!(lens.degree_at(0, Predicate::IsA), 1); + assert_eq!(lens.degree_at(2, Predicate::IsA), 0); + assert_eq!(lens.degree_at(0, Predicate::PartOf), 0); + } + + #[test] + fn is_sorted_actually_rejects_an_unsorted_slice() { + // the guard must be able to fail, or it is decoration + let rows = vec![row(3, &[]), row(1, &[2])]; + assert!(!SpineLens::new(&rows).is_sorted()); + } + + /// The lens drives the reasoner with no adapter type in between — the + /// property that keeps ogar-obo and ogar-elk free of a dependency edge. + #[test] + fn the_lens_composes_with_a_visitor_closure() { + let rows = fixture(); + let lens = SpineLens::new(&rows); + // the shape `ogar_elk::Parents` is blanket-implemented for + let f = |addr: (u32, u32), visit: &mut dyn FnMut((u32, u32))| { + lens.parents_of(addr.0, addr.1, &mut |c, n| visit((c, n))); + }; + let mut seen = Vec::new(); + f((MONDO, 1), &mut |a| seen.push(a)); + assert_eq!(seen, vec![(MONDO, 2)]); + } +} From 89cb49be2a0230aa08f0675e3c63b9ca785d6247 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 8 Aug 2026 10:58:27 +0000 Subject: [PATCH 05/13] S4b: the typed cross-predicate walk (R exists), seeded and narrowed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit meet over is_a alone can never cross an ontology — subsumption stays inside the ontology that declares it. Crossing needs a ROLE (has_phenotype into HPO, has_anatomy into Uberon) plus R-exists generalization of the FILLER up the grounding spine. The two spines are different and swapping them is the classic error, so both are explicit parameters with no default that could silently be wrong: grounding_parents_of is is_a UNION part_of (what a filler generalizes up); parents_of stays is_a alone (what a subject subsumes through). Generalizing a subject through part_of is unsound — A part_of B, B subclass C does not give A subclass C — while generalizing a filler through it is the standard inherit-grounding deduction. TWO CORRECTIONS THIS COMMIT RECORDS, both caught by checks rather than by care. 1. The falsifier was originally aimed at reason::saturate's existential_inferred and is NOT any more. Measured: the combined is_a UNION part_of spine carries 2,277 back edges even though is_a alone has zero, and ancestor_closure SKIPS a back edge when it meets one. Its memo is therefore an incomplete closure whose value depends on DFS start order — the same code over the same triples in two orders produced 9,522,084 and 9,437,429. A number that moves cannot falsify anything, so the check is now against an independent local computation that shares no code with fillers_closed: 64 seeds, 0 divergences, and all 64 widened (so the rule is not inert). 2. The first meet was sound and vacuous. 120 of 120 sampled disease pairs "shared a phenotype" and the shared address was always HP:0000001 — the HPO root — because every phenotype generalizes to "All". A result that fires on every input carries as much information as one that never fires. most_specific narrows a meet to members that are not an ancestor of another member, and the sampled pairs then split 81 sharing something specific vs 39 sharing only the root. It is deliberately NOT folded into meet_via: the raw meet is the sound object and narrowing is the caller's presentation choice. Scale is seeded, never global. A query starts from facts that already exist — an observed lab code, a dispensed drug, a coded diagnosis — which is a handful of addresses. An earlier draft summed R-exists over all 67,745 existential edges to chase an aggregate; that is a theory-of-everything computation nobody asked for and it is gone. Controls that must keep holding: is_a finds 0 cross-ontology meets, and the narrowed meet must produce BOTH specific and root-only pairs. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01KCGhDYoQBXs3poaR7sFuqp --- .../ogar-elk/examples/elk_cross_ontology.rs | 229 ++++++++++++++++++ crates/ogar-elk/src/lens.rs | 161 ++++++++++++ crates/ogar-elk/src/lib.rs | 2 +- crates/ogar-obo/src/spine.rs | 112 +++++++++ 4 files changed, 503 insertions(+), 1 deletion(-) create mode 100644 crates/ogar-elk/examples/elk_cross_ontology.rs diff --git a/crates/ogar-elk/examples/elk_cross_ontology.rs b/crates/ogar-elk/examples/elk_cross_ontology.rs new file mode 100644 index 00000000..50aa76c0 --- /dev/null +++ b/crates/ogar-elk/examples/elk_cross_ontology.rs @@ -0,0 +1,229 @@ +//! S4b — the typed cross-predicate walk, verified on **seeded, local** queries. +//! +//! `meet` over `is_a` alone always returns nothing across ontologies: +//! subsumption never leaves the ontology it belongs to. Crossing needs a +//! **role** — `has_phenotype` into HPO, `has_anatomy` into Uberon — and then +//! R∃ generalization of the filler up the grounding spine. +//! +//! # Scale: seeded from facts, never a global saturation +//! +//! A query starts from facts that already exist — an observed lab code, a +//! dispensed drug, an imaging study, a coded diagnosis. That is a handful of +//! addresses. Saturating the whole ontology to answer something local is a +//! theory-of-everything computation nobody asked for, and this example +//! deliberately does not do it. +//! +//! # Why the check is NOT against `reason::saturate` +//! +//! Measured this session: the combined `is_a ∪ part_of` grounding spine carries +//! **2 277 back edges** (is_a alone has none), and `ancestor_closure` SKIPS a +//! back edge when it meets one (`!stack.iter().any(|(x, _)| *x == p)`). Its +//! memo is therefore an incomplete closure whose value depends on DFS start +//! order — the same code over the same triples in two orders produced +//! 9 522 084 and 9 437 429. A number that moves cannot falsify anything. +//! +//! So this verifies the R∃ *semantics* on bounded queries against an +//! independent computation sharing no code with [`fillers_closed`]. +//! +//! Usage: `cargo run --release -p ogar-elk --example elk_cross_ontology -- [slab.soa]` + +use ogar_elk::{ClassAddr, LensClosure, Role, fillers_closed, meet_via, most_specific}; +use ogar_obo::{NODE_ROW_STRIDE, Predicate, Row512, edges::EdgeLanes, spine::SpineLens}; + +fn main() { + let path = std::env::args() + .nth(1) + .unwrap_or_else(|| "/tmp/obo/obo-core.soa".to_string()); + let bytes = std::fs::read(&path).unwrap_or_else(|e| panic!("read {path}: {e}")); + let n = bytes.len() / NODE_ROW_STRIDE; + let mut rows = vec![Row512::zeroed(); n]; + for (i, r) in rows.iter_mut().enumerate() { + r.0.copy_from_slice(&bytes[i * NODE_ROW_STRIDE..(i + 1) * NODE_ROW_STRIDE]); + } + let lens = SpineLens::new(&rows); + assert!(lens.is_sorted()); + println!("slab {path}\n rows {n}"); + + // The GROUNDING spine — is_a ∪ part_of. Deliberately NOT the subsumption + // source: a filler generalizes through mereology, a subject must not. + let grounding = |a: ClassAddr, visit: &mut dyn FnMut(ClassAddr)| { + lens.grounding_parents_of(a.classid, a.identity, &mut |c, num| { + visit(ClassAddr::new(c, num)); + }); + }; + let fillers = |a: ClassAddr, r: Role, visit: &mut dyn FnMut(ClassAddr)| { + let p = match r.0 { + 3 => Predicate::HasPhenotype, + 4 => Predicate::HasLocation, + 5 => Predicate::HasAnatomy, + 6 => Predicate::HasQuality, + _ => Predicate::PartOf, + }; + lens.fillers_of(a.classid, a.identity, p, &mut |c, num| { + visit(ClassAddr::new(c, num)); + }); + }; + let has_phenotype = Role(Predicate::HasPhenotype as u8); + + // ── seeds: the facts a query would actually start from ── + let mut seeds: Vec = Vec::new(); + for row in &rows { + if EdgeLanes::new(&row.0).degree(Predicate::HasPhenotype) > 0 { + let s = ogar_obo::edges::subject(&row.0); + seeds.push(ClassAddr::new(s.classid, s.num)); + if seeds.len() == 64 { + break; + } + } + } + println!(" seeds (diseases asserting a phenotype): {}", seeds.len()); + assert!(!seeds.is_empty(), "no seed carries a phenotype"); + + // ── FALSIFIER: R∃ against an independent local computation ── + let t = std::time::Instant::now(); + let mut checked = 0usize; + let mut widened = 0usize; + for &d in &seeds { + let got = fillers_closed(d, has_phenotype, &fillers, &grounding, 64); + + // Independent path: asserted fillers off the row, then a hand-rolled + // BFS up the grounding spine. No shared code with `fillers_closed`. + let mut asserted: Vec = Vec::new(); + lens.fillers_of( + d.classid, + d.identity, + Predicate::HasPhenotype, + &mut |c, num| asserted.push(ClassAddr::new(c, num)), + ); + let mut want: Vec = Vec::new(); + for &f in &asserted { + let mut queue = std::collections::VecDeque::from([(f, 0usize)]); + let mut seen: Vec = vec![f]; + if let Err(at) = want.binary_search(&f) { + want.insert(at, f); + } + while let Some((x, depth)) = queue.pop_front() { + if depth >= 64 { + continue; + } + lens.grounding_parents_of(x.classid, x.identity, &mut |c, num| { + let p = ClassAddr::new(c, num); + if let Err(at) = seen.binary_search(&p) { + seen.insert(at, p); + queue.push_back((p, depth + 1)); + if let Err(w) = want.binary_search(&p) { + want.insert(w, p); + } + } + }); + } + } + assert_eq!( + got, want, + "R∃ diverges for 0x{:08x}:{}", + d.classid, d.identity + ); + checked += 1; + if got.len() > asserted.len() { + widened += 1; + } + } + let el = t.elapsed(); + + println!("\n=== FALSIFIER (R∃, local) ==="); + println!(" seeds checked : {checked} (0 divergences) in {el:?}"); + println!(" seeds where R∃ widened : {widened} (must be > 0, else generalization is inert)"); + assert!( + widened > 0, + "R∃ never added an ancestor — the rule is doing nothing" + ); + + // ── the Schnittpunkt is_a alone cannot find ── + println!("\n=== CROSS-ONTOLOGY SCHNITTPUNKTE ==="); + + // control: is_a alone must never leave the seed's own ontology + let isa = |a: ClassAddr, visit: &mut dyn FnMut(ClassAddr)| { + lens.parents_of(a.classid, a.identity, &mut |c, num| { + visit(ClassAddr::new(c, num)); + }); + }; + let isa_lens = LensClosure::new(&isa); + let mut isa_cross = 0usize; + let k = seeds.len().min(16); + for i in 0..k { + for j in (i + 1)..k { + if isa_lens + .meet(&[seeds[i], seeds[j]]) + .iter() + .any(|a| a.classid != seeds[i].classid) + { + isa_cross += 1; + } + } + } + println!(" via is_a : {isa_cross} pairs meeting OUTSIDE their own ontology (expected 0)"); + assert_eq!(isa_cross, 0, "subsumption must not cross an ontology"); + + let mut found = 0usize; + let mut empty = 0usize; + let mut root_only = 0usize; + let mut specific = 0usize; + for i in 0..k { + for j in (i + 1)..k { + let m = meet_via( + &[seeds[i], seeds[j]], + has_phenotype, + &fillers, + &grounding, + 64, + ); + if m.is_empty() { + empty += 1; + continue; + } + found += 1; + // The raw meet is sound but vacuous: every phenotype generalizes to + // the HPO root, so every pair "shares" it. Narrow to the most + // specific common filler — the answer that carries information. + let narrowed = most_specific(&m, &grounding, 64); + let is_root_only = narrowed.len() == 1 + && lens + .resolve(narrowed[0].classid, narrowed[0].identity) + .is_some_and(|r| EdgeLanes::new(&rows[r].0).degree(Predicate::IsA) == 0); + if is_root_only { + root_only += 1; + } else { + specific += 1; + if specific <= 3 { + println!( + " 0x{:08x}:{} ∩ 0x{:08x}:{} -> {} shared, {} most-specific, e.g. 0x{:08x}:{}", + seeds[i].classid, + seeds[i].identity, + seeds[j].classid, + seeds[j].identity, + m.len(), + narrowed.len(), + narrowed[0].classid, + narrowed[0].identity + ); + } + } + } + } + println!(" via role : {found} pairs share something | {empty} disjoint"); + println!(" narrowed : {specific} pairs share something SPECIFIC | {root_only} only the root"); + assert!( + found > 0, + "the role walk found no Schnittpunkt — is_a-only behaviour" + ); + assert!( + specific > 0, + "no pair shared anything below the root — narrowing found nothing informative" + ); + assert!( + root_only > 0 || empty > 0, + "every pair sharing something specific would mean the walk discriminates nothing" + ); + + println!("\nS4b VERIFIED: R∃ sound on seeded queries, roles cross, meets discriminate."); +} diff --git a/crates/ogar-elk/src/lens.rs b/crates/ogar-elk/src/lens.rs index 84a3aa68..6df5ec57 100644 --- a/crates/ogar-elk/src/lens.rs +++ b/crates/ogar-elk/src/lens.rs @@ -266,6 +266,167 @@ pub fn intersect_sorted(a: &[ClassAddr], b: &[ClassAddr]) -> Vec { out } +// ── S4b: the existential arm (R∃) ──────────────────────────────────────────── + +/// An existential role — the `r` in `A ⊑ ∃r.B`. +/// +/// A plain tag rather than a rich type, so this crate stays zero-dep and does +/// not need the producer's predicate enum. It is deliberately NOT a +/// [`Subsumption`](crate::Subsumption): a role is an existential restriction, +/// and the two compose by different rules. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Role(pub u8); + +/// A source of **asserted existential fillers** for a class under one role. +/// +/// Same visitor shape as [`Parents`], and for the same reason: an +/// implementation over borrowed rows hands out fillers one at a time with +/// nothing allocated per call. +pub trait Fillers { + /// Visit each asserted filler of `role` on `c`, in any order. + fn fillers_of(&self, c: ClassAddr, role: Role, visit: &mut dyn FnMut(ClassAddr)); +} + +impl Fillers for F +where + F: Fn(ClassAddr, Role, &mut dyn FnMut(ClassAddr)), +{ + #[inline] + fn fillers_of(&self, c: ClassAddr, role: Role, visit: &mut dyn FnMut(ClassAddr)) { + self(c, role, visit); + } +} + +/// **R∃** — `A ⊑ ∃r.B` and `B ⊑ C` give `A ⊑ ∃r.C`. +/// +/// Returns every filler of `role` on `subject`, each generalized upward through +/// `grounding_spine`. Sorted and de-duplicated; no hashing. +/// +/// # The two spines are NOT the same, and swapping them is the classic error +/// +/// `grounding_spine` is the spine a **filler** generalizes up. For the OBO core +/// that is `is_a ∪ part_of`: an anatomical filler's grounding is inherited +/// through mereology as well as taxonomy, which is the deductive form of +/// "inherit grounding". It is deliberately a DIFFERENT source from the one +/// [`LensClosure::entails`] uses, which must be `is_a` alone. +/// +/// The distinction is exactly the false-ancestor hazard, and it is easy to get +/// backwards: generalizing a **subject** through `part_of` is unsound +/// (`A part_of B`, `B ⊑ C` does NOT give `A ⊑ C`); generalizing a **filler** +/// through it is sound and is what this function does. Both sources are +/// therefore explicit parameters — there is no default that could silently be +/// the wrong one. +/// +/// Note this generalizes the filler only. A caller wanting "every phenotype of +/// this disease INCLUDING those asserted on its superclasses" walks the +/// subject's own subsumption ancestors first and unions the results — that is a +/// larger question, and keeping it out of here is what lets this match the +/// producer's own saturation count exactly. +#[must_use] +pub fn fillers_closed( + subject: ClassAddr, + role: Role, + fillers: &F, + grounding_spine: &S, + depth_cap: usize, +) -> Vec +where + S: Parents + ?Sized, + F: Fillers + ?Sized, +{ + let mut out: Vec = Vec::new(); + let spine = LensClosure::new(grounding_spine).with_depth_cap(depth_cap); + fillers.fillers_of(subject, role, &mut |o| { + if let Err(at) = out.binary_search(&o) { + out.insert(at, o); + } + for a in spine.supers_of(o) { + if let Err(at) = out.binary_search(&a) { + out.insert(at, a); + } + } + }); + out +} + +/// The **cross-ontology Schnittpunkt** — where several subjects' role-fillers +/// meet, after R∃ generalization. +/// +/// This is what `meet` over `is_a` alone cannot answer: subsumption never +/// leaves its own ontology, so two diseases only coincide once you cross by a +/// role (`has_phenotype` into HPO, `has_anatomy` into Uberon) and then climb +/// the filler's own spine. Intersection over sorted addresses, one merge pass +/// per seed. +/// +/// Empty seeds yield nothing — the identity of an intersection over no sets is +/// not an answer a caller can use. +#[must_use] +pub fn meet_via( + seeds: &[ClassAddr], + role: Role, + fillers: &F, + grounding_spine: &S, + depth_cap: usize, +) -> Vec +where + S: Parents + ?Sized, + F: Fillers + ?Sized, +{ + let Some((first, rest)) = seeds.split_first() else { + return Vec::new(); + }; + let mut acc = fillers_closed(*first, role, fillers, grounding_spine, depth_cap); + for s in rest { + if acc.is_empty() { + break; + } + acc = intersect_sorted( + &acc, + &fillers_closed(*s, role, fillers, grounding_spine, depth_cap), + ); + } + acc +} + +/// Keep only the **most specific** members of a sorted address set — those that +/// are not an ancestor of another member. +/// +/// # Why a raw meet is sound and useless +/// +/// Every filler generalizes up to its ontology's root, so ANY two subjects +/// share that root and [`meet_via`] returns it. Measured on the OBO core: 120 +/// of 120 sampled disease pairs "shared a phenotype", and in every case the +/// shared address was `HP:0000001` — "All". A result that fires on every input +/// carries exactly as much information as one that never fires. +/// +/// The informative answer to *what do these two subjects share* is the most +/// specific common filler, which is what this returns: drop `x` when some other +/// member `y` has `x` among its ancestors, because `y` already says everything +/// `x` does and says it more precisely. +/// +/// Cost is |set|² spine walks in the worst case, which is fine because a meet +/// over seeded facts is small. It is deliberately NOT folded into [`meet_via`]: +/// the raw meet is the sound object, and narrowing is a presentation choice a +/// caller makes. +#[must_use] +pub fn most_specific(set: &[ClassAddr], spine: &S, depth_cap: usize) -> Vec +where + S: Parents + ?Sized, +{ + let lens = LensClosure::new(spine).with_depth_cap(depth_cap); + let mut out = Vec::new(); + for (i, x) in set.iter().enumerate() { + let redundant = set + .iter() + .enumerate() + .any(|(j, y)| i != j && lens.supers_of(*y).binary_search(x).is_ok()); + if !redundant { + out.push(*x); + } + } + out +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/ogar-elk/src/lib.rs b/crates/ogar-elk/src/lib.rs index f2c70e1d..acc747e4 100644 --- a/crates/ogar-elk/src/lib.rs +++ b/crates/ogar-elk/src/lib.rs @@ -91,7 +91,7 @@ pub mod lens; -pub use lens::{LensClosure, Parents}; +pub use lens::{Fillers, LensClosure, Parents, Role, fillers_closed, meet_via, most_specific}; use std::collections::{HashMap, HashSet, VecDeque}; diff --git a/crates/ogar-obo/src/spine.rs b/crates/ogar-obo/src/spine.rs index 984b0014..d825260c 100644 --- a/crates/ogar-obo/src/spine.rs +++ b/crates/ogar-obo/src/spine.rs @@ -146,6 +146,41 @@ impl<'a> SpineLens<'a> { self.links_at(i, Predicate::IsA, visit); } } + + /// The **grounding spine** of an address: `is_a` ∪ `part_of` parents. + /// + /// This is the spine an existential FILLER generalizes up (R∃), and it is + /// deliberately NOT [`Self::parents_of`]. An anatomical filler's grounding + /// is inherited through mereology as well as taxonomy — the deductive form + /// of "inherit grounding up the anatomy spine". + /// + /// **Never hand this to a subsumption reasoner.** Generalizing a *subject* + /// through `part_of` is the false-ancestor error (`A part_of B`, `B ⊑ C` + /// does not give `A ⊑ C`); generalizing a *filler* through it is sound. + /// [`Self::parents_of`] stays `is_a`-only for exactly that reason, and the + /// two are separate methods so a caller must choose on purpose. + pub fn grounding_parents_of(&self, classid: u32, num: u32, visit: &mut dyn FnMut(u32, u32)) { + if let Some(i) = self.resolve(classid, num) { + self.links_at(i, Predicate::IsA, visit); + self.links_at(i, Predicate::PartOf, visit); + } + } + + /// The asserted existential fillers of one role on an address. + /// + /// A thin typed read — the R∃ generalization is the reasoner's job, not + /// this one's. An address with no row in view yields nothing, silently. + pub fn fillers_of( + &self, + classid: u32, + num: u32, + role: Predicate, + visit: &mut dyn FnMut(u32, u32), + ) { + if let Some(i) = self.resolve(classid, num) { + self.links_at(i, role, visit); + } + } } /// Reinterpret a byte buffer as rows, if it is correctly sized and aligned. @@ -193,6 +228,83 @@ mod tests { vec![row(1, &[2]), row(2, &[3]), row(3, &[]), row(9, &[])] } + #[test] + fn the_grounding_spine_includes_part_of_and_the_subsumption_one_does_not() { + // one row with BOTH an is_a and a part_of parent + let mut r = Row512::zeroed(); + r.0[0..16].copy_from_slice(&crate::pack_key(MONDO, 1)); + r.0[crate::EDGES_OFFSET + Predicate::IsA as usize] = 1; + r.0[crate::EDGES_OFFSET + Predicate::PartOf as usize] = 1; + let mut tg = vec![ + ( + Predicate::IsA, + TermId { + ns: Namespace::Mondo as u8, + num: 2, + }, + ), + ( + Predicate::PartOf, + TermId { + ns: Namespace::Mondo as u8, + num: 3, + }, + ), + ]; + pack_edges(&mut r.0, &mut tg, 0x0000); + let rows = vec![r]; + let lens = SpineLens::new(&rows); + + let mut subsumption = Vec::new(); + lens.parents_of(MONDO, 1, &mut |_, n| subsumption.push(n)); + assert_eq!( + subsumption, + vec![2], + "part_of must NOT reach a subsumption walk" + ); + + let mut grounding = Vec::new(); + lens.grounding_parents_of(MONDO, 1, &mut |_, n| grounding.push(n)); + assert_eq!( + grounding, + vec![2, 3], + "the grounding spine is is_a UNION part_of" + ); + } + + #[test] + fn fillers_are_read_per_role_and_can_be_empty() { + let mut r = Row512::zeroed(); + r.0[0..16].copy_from_slice(&crate::pack_key(MONDO, 1)); + r.0[crate::EDGES_OFFSET + Predicate::HasPhenotype as usize] = 1; + let mut tg = vec![( + Predicate::HasPhenotype, + TermId { + ns: Namespace::Hpo as u8, + num: 42, + }, + )]; + pack_edges(&mut r.0, &mut tg, 0x0000); + let rows = vec![r]; + let lens = SpineLens::new(&rows); + + let mut got = Vec::new(); + lens.fillers_of(MONDO, 1, Predicate::HasPhenotype, &mut |c, n| { + got.push((c, n)) + }); + assert_eq!(got, vec![(0x0302_0000, 42)], "crosses into HPO"); + + // can-stay-silent: a role with no asserted filler yields nothing + got.clear(); + lens.fillers_of(MONDO, 1, Predicate::HasAnatomy, &mut |c, n| { + got.push((c, n)) + }); + assert!( + got.is_empty(), + "an unasserted role must not manufacture a filler" + ); + } + #[test] fn resolve_finds_every_row_and_misses_absent_ones() { let rows = fixture(); From 5984508e8e384e92139894c7b8c45fdf5fdd407a Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 8 Aug 2026 12:16:39 +0000 Subject: [PATCH 06/13] =?UTF-8?q?ogar-obo:=20the=20namespace=20table=20as?= =?UTF-8?q?=20data=20=E2=80=94=20NsRegistry?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The bake reaches its namespaces through the `Namespace` enum, which spreads one namespace's facts across four `match` bodies plus an index array (`prefix` / `from_prefix` / `concept_id` / `namespace`). A sixth namespace means editing five sites with hand-written literals — the shape that makes a bake un-extendable without a code change per ontology. `registry::NsRegistry` carries the same facts as one ordered table of `NsSpec { prefix, concept_id }` rows. A driver that wants a different set of namespaces passes a different table; it does not touch this crate's types. `NsRegistry::parse` is the general form of `TermId::parse`, which is this against `OBO_CORE`. The row INDEX is the `TermId::ns` ordinal, and `bake` sorts ids by `(ns, num)` — so table order determines the baked row order and therefore the artifact digest. `OBO_CORE` is a second copy of the enum's facts and is only safe to hold because `obo_core_matches_the_shipped_enum` fails the moment the two disagree on order, prefix, or concept id. That is the drift guard, not a formality: a reorder silently re-bakes every downstream slab. `classify` deliberately stays on the enum — a `(subject_ns, object_ns)` pair maps to an RO predicate, which is semantics between two named ontologies, not a per-namespace attribute. A namespace outside the core classifies as `Predicate::Other`, correct for upper-level spine ontologies whose edges are `is_a` skeletons with no cross-ontology predicate to name. Falsifiers, not assertions implied by the code: - the mirror test fails on any reorder/renumber/rename; - registry parse must agree with the shipped parse on all six accepts AND all five rejects (prefix outside table, >24-bit, malformed, no colon, wrong case) — otherwise routing a bake through the table would change which terms are tiled; - the extended-table test asserts the shipped core CANNOT parse `BFO:0000002`, so a table that was silently ignored would fail rather than pass; - ordinals are table-relative — MONDO is row 1 in the extended table and row 0 in the core — pinning that a TermId is only meaningful WITH its registry. No shipped digest moves: nothing in the bake path consults the registry yet. 34 tests pass; clippy -D warnings clean. --- crates/ogar-obo/src/lib.rs | 1 + crates/ogar-obo/src/registry.rs | 285 ++++++++++++++++++++++++++++++++ 2 files changed, 286 insertions(+) create mode 100644 crates/ogar-obo/src/registry.rs diff --git a/crates/ogar-obo/src/lib.rs b/crates/ogar-obo/src/lib.rs index c9ea6c3d..24d8ae6b 100644 --- a/crates/ogar-obo/src/lib.rs +++ b/crates/ogar-obo/src/lib.rs @@ -43,6 +43,7 @@ pub mod crosswalk; pub mod edges; pub mod reason; +pub mod registry; pub mod spine; /// Row stride of the canonical SoA node — `key(16) + edges(16) + value(480)`. diff --git a/crates/ogar-obo/src/registry.rs b/crates/ogar-obo/src/registry.rs new file mode 100644 index 00000000..06ab4e0e --- /dev/null +++ b/crates/ogar-obo/src/registry.rs @@ -0,0 +1,285 @@ +//! # `registry` — the namespace table as **data**, not as five `match` arms. +//! +//! The shipped bake reaches its namespaces through the [`crate::Namespace`] enum, +//! which spreads one namespace's facts across four `match` bodies and an +//! index array ([`crate::Namespace::prefix`], [`crate::Namespace::from_prefix`], +//! [`crate::Namespace::concept_id`], [`crate::Namespace::namespace`]). Adding a sixth +//! namespace means editing all five sites with hand-written literals — the +//! shape that makes a bake un-extendable without a code change per ontology. +//! +//! This module carries the same facts as **one table of rows**. A driver that +//! wants to bake a *different* set of namespaces passes a different +//! [`NsRegistry`]; it does not touch this crate's types. +//! +//! ## The ordinal is the sort key — table ORDER is load-bearing +//! +//! A [`TermId`]'s `ns` field is the row's **index in the registry**, and +//! [`crate::bake`] sorts ids by `(ns, num)`. So the table's order determines +//! the row order of the baked artifact, and therefore its digest. Reordering +//! [`OBO_CORE`] silently re-bakes every downstream slab. +//! +//! [`obo_core_matches_the_shipped_enum`](self) pins the mirror: every row's +//! index and concept id must equal the enum's discriminant and +//! [`crate::Namespace::concept_id`]. That test fails on any reorder or renumber — +//! it is the drift guard that makes a second copy of these facts safe to hold. +//! +//! ## What is NOT here +//! +//! [`crate::classify`] maps a `(subject_ns, object_ns)` pair to an RO +//! [`Predicate`](crate::Predicate). That is *semantics between two named +//! ontologies*, not a per-namespace attribute, so it stays on the enum. A +//! namespace outside the core classifies as +//! [`Predicate::Other`](crate::Predicate::Other) — correct for the upper-level +//! spine ontologies, whose edges are `is_a` skeletons with no cross-ontology +//! predicate to name. + +use crate::TermId; + +/// One namespace row — the config unit. +/// +/// `prefix` is the OBO CURIE prefix as it appears left of the colon +/// (`"MONDO"`, `"HP"`, …). `concept_id` is the canonical hi-u16 that +/// [`crate::Namespace::render_classid`] shifts into the classid. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct NsSpec { + /// The OBO CURIE prefix, matched case-sensitively (OBO prefixes are + /// upper-case by convention and the shipped five are matched exactly). + pub prefix: &'static str, + /// The canonical hi-u16 concept id (`0xDDCC`, domain `DD` · slot `CC`). + pub concept_id: u16, +} + +/// An ordered namespace table. The row **index** is the [`TermId::ns`] +/// ordinal; see the module docs on why order is load-bearing. +#[derive(Debug, Clone, Copy)] +pub struct NsRegistry { + specs: &'static [NsSpec], +} + +impl NsRegistry { + /// Build a registry over a static table. `specs.len()` must be `<= 256` + /// because the ordinal is stored in [`TermId::ns`] (`u8`). + #[must_use] + pub const fn new(specs: &'static [NsSpec]) -> Self { + assert!(specs.len() <= 256, "ordinal must fit TermId::ns (u8)"); + Self { specs } + } + + /// Number of namespaces in the table. + #[must_use] + pub const fn len(&self) -> usize { + self.specs.len() + } + + /// Whether the table is empty. + #[must_use] + pub const fn is_empty(&self) -> bool { + self.specs.is_empty() + } + + /// Every row, in ordinal order. + #[must_use] + pub const fn specs(&self) -> &'static [NsSpec] { + self.specs + } + + /// The ordinal of a CURIE prefix, or `None` if the prefix is not in this + /// table. Linear scan — the tables are tens of rows, and this runs once + /// per CURIE during a parse, never in a query path. + #[must_use] + pub fn ordinal_of(&self, prefix: &str) -> Option { + self.specs + .iter() + .position(|s| s.prefix == prefix) + .map(|i| i as u8) + } + + /// The row at an ordinal. + #[must_use] + pub fn spec(&self, ordinal: u8) -> Option<&'static NsSpec> { + self.specs.get(ordinal as usize) + } + + /// The concept id at an ordinal. + #[must_use] + pub fn concept_id(&self, ordinal: u8) -> Option { + self.spec(ordinal).map(|s| s.concept_id) + } + + /// The full V3 render classid — canon-high `(concept as u32) << 16 | + /// app_prefix`, identical to [`crate::Namespace::render_classid`]. + #[must_use] + pub fn render_classid(&self, ordinal: u8, app_prefix: u16) -> Option { + self.concept_id(ordinal) + .map(|c| (u32::from(c) << 16) | u32::from(app_prefix)) + } + + /// Parse a CURIE against **this** table. Same contract as + /// [`TermId::parse`] (which is this, against [`OBO_CORE`]): `None` if the + /// prefix is absent from the table, the numeric is malformed, or the + /// numeric exceeds 24 bits. + #[must_use] + pub fn parse(&self, curie: &str) -> Option { + let (p, n) = curie.split_once(':')?; + let ns = self.ordinal_of(p)?; + let num: u32 = n.trim().parse().ok()?; + if num > 0x00FF_FFFF { + return None; + } + Some(TermId { ns, num }) + } +} + +/// The **shipped** five-namespace core, in the enum's discriminant order. +/// +/// This table is a second copy of facts the [`crate::Namespace`] enum also holds. +/// It is safe to hold only because +/// [`obo_core_matches_the_shipped_enum`](self) fails the build's test suite +/// the moment the two disagree. +pub const OBO_CORE: NsRegistry = NsRegistry::new(&[ + NsSpec { + prefix: "MONDO", + concept_id: 0x0301, + }, + NsSpec { + prefix: "HP", + concept_id: 0x0302, + }, + NsSpec { + prefix: "UBERON", + concept_id: 0x0303, + }, + NsSpec { + prefix: "PATO", + concept_id: 0x0304, + }, + NsSpec { + prefix: "RO", + concept_id: 0x0305, + }, +]); + +#[cfg(test)] +mod tests { + use super::*; + use crate::Namespace; + + /// The drift guard that makes the duplicate table safe. + /// + /// **What would make it fail:** reordering [`OBO_CORE`], renumbering a + /// concept id, or renaming a prefix. Each of those silently changes the + /// baked row order or the classid of every row in a namespace — i.e. + /// moves the shipped digests — and this test is what refuses it. + #[test] + fn obo_core_matches_the_shipped_enum() { + let enum_order = [ + Namespace::Mondo, + Namespace::Hpo, + Namespace::Uberon, + Namespace::Pato, + Namespace::Ro, + ]; + assert_eq!( + OBO_CORE.len(), + enum_order.len(), + "registry and enum must carry the same namespace count" + ); + for (i, ns) in enum_order.iter().enumerate() { + let ord = i as u8; + assert_eq!( + OBO_CORE.ordinal_of(ns.prefix()), + Some(ord), + "{} must sit at the enum's discriminant — row order is the sort key", + ns.prefix() + ); + assert_eq!( + OBO_CORE.concept_id(ord), + Some(ns.concept_id()), + "{} concept id must mirror the enum", + ns.prefix() + ); + assert_eq!( + OBO_CORE.render_classid(ord, 0x00AB), + Some(ns.render_classid(0x00AB)), + "{} classid must mirror the enum under a non-zero app prefix", + ns.prefix() + ); + } + } + + /// The registry parse must agree with the shipped [`TermId::parse`] on + /// every accept AND every reject — otherwise routing a bake through the + /// table would change which terms are tiled. + #[test] + fn registry_parse_agrees_with_shipped_parse() { + let accept = [ + "MONDO:0007739", + "HP:0000118", + "UBERON:0000955", + "PATO:0000001", + "RO:0002200", + "MONDO:0700092", // above u16 — the rail case + ]; + for c in accept { + assert_eq!( + OBO_CORE.parse(c), + TermId::parse(c), + "{c} must parse identically through the table" + ); + assert!(OBO_CORE.parse(c).is_some(), "{c} must actually parse"); + } + let reject = [ + "CHEBI:12345", // prefix outside the table + "MONDO:99999999", // > 24 bit + "MONDO:not-a-num", // malformed numeric + "nocolon", // no separator + "mondo:0007739", // case differs — OBO prefixes are upper-case + ]; + for c in reject { + assert_eq!( + OBO_CORE.parse(c), + TermId::parse(c), + "{c} must reject identically through the table" + ); + assert!(OBO_CORE.parse(c).is_none(), "{c} must actually reject"); + } + } + + /// A registry over a *different* table resolves namespaces the core does + /// not — the whole point. This is the can-it-fire half: if the table were + /// ignored and the core consulted instead, `BFO:0000002` would not parse + /// and the classid would not be the extended one. + #[test] + fn a_different_table_resolves_namespaces_the_core_cannot() { + const EXTENDED: NsRegistry = NsRegistry::new(&[ + NsSpec { + prefix: "BFO", + concept_id: 0x0310, + }, + NsSpec { + prefix: "MONDO", + concept_id: 0x0301, + }, + ]); + let bfo = EXTENDED.parse("BFO:0000002").expect("BFO parses here"); + assert_eq!(bfo.ns, 0, "BFO is row 0 of this table"); + assert_eq!(bfo.num, 2); + assert_eq!(EXTENDED.render_classid(0, 0x0000), Some(0x0310_0000)); + assert!( + TermId::parse("BFO:0000002").is_none(), + "the shipped core must NOT resolve BFO — otherwise this proves nothing" + ); + + // Ordinals are table-relative: MONDO is row 1 here, row 0 in the core. + // A TermId is therefore only meaningful WITH its registry. + assert_eq!(EXTENDED.parse("MONDO:0007739").map(|t| t.ns), Some(1)); + assert_eq!(TermId::parse("MONDO:0007739").map(|t| t.ns), Some(0)); + } + + #[test] + fn unknown_ordinal_resolves_to_none_rather_than_panicking() { + assert_eq!(OBO_CORE.spec(9), None); + assert_eq!(OBO_CORE.concept_id(9), None); + assert_eq!(OBO_CORE.render_classid(9, 0), None); + } +} From 7e9626f7279afe828b20b2bd8badc6f6d96f1dfb Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 8 Aug 2026 12:24:14 +0000 Subject: [PATCH 07/13] ogar-obo: the RDF front-end that retains edges and keys them by CURIE numeric MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The workspace already parses RDF — lance-graph-ontology's OwlHydrator reads Turtle and RDF/XML through oxttl/oxrdfxml. I proposed building one before checking, was corrected, and read it. It cannot serve as a bake front-end, for two reasons found in `hydrate_many`: 1. It DROPS the triple after interning. `OntologySlot` is `{ entity_count, iri_to_id }` and nothing else — no subClassOf survives hydration. `ContextBundle::edge_types` is a whitelist of predicate IRIs a traversal may follow, not stored edges. 2. Its identity is an intern COUNTER, so it depends on file order. A bake keyed on it would not be reproducible. `rdf` is the opposite of both, and is NOT a second parser — oxttl/oxrdfxml do the reading, pinned to the same versions lance-graph-ontology pins, behind an off-by-default `rdf` feature so the crate keeps its ogar-vocab-only profile. Identity comes from the term: `…/obo/HP_0000118` -> `HP:0000118` -> 118, which is the numeric pack_key writes on the family:identity rail. The extraction map IS the NsRegistry — every OBO PURL is `_`, so a new ontology is a table row, never a code path. Predicates are read from `owl:onProperty` against OBO_RESTRICTION_PREDICATES, not guessed from a namespace pair. parse_obo guesses deliberately, because the `.obo` relationship label is unreliable; RDF names the property, so the guess is unnecessary. Consequence stated in the docs: an RDF bake and a .obo bake of the same ontology may type the same edge differently. The shipped five-namespace bake was produced from .obo and is NOT re-baked here. The table carries only the three CURIEs the crate already documents on Predicate (BFO:0000050, RO:0002200, RO:0004026). HasAnatomy/HasQuality are documented as harvested from logical definitions with no named property, so there is no CURIE to add that would not be invented — those land on Other. Documented limits, each with an assertion rather than a hope: restrictions nested in an owl:intersectionOf collection are not walked; equivalentClass to a NAMED class is skipped (equivalence is not subsumption — recording it as is_a would invent a parent); a term becomes a row only on `a owl:Class`. Falsifiers: - order-independence: the same graph with stanzas reversed yields identical ids and edges, and the test asserts the reordered bytes actually differ. An intern-counter identity fails this. - an EMPTY predicate table must classify everything as Other, proving the table is read rather than the mapping baked into the function. - a unionOf blank node must contribute NO edge — the documented limit asserted, not hoped. - a malformed document must error, so a parse failure cannot look like an empty ontology. - the registry is genuinely consulted: the same IRI resolves under one table and not another. Also adds META_STUDY_SPINE (8 rows, 0x0306..0x030D) as config, and examples/probe_spine.rs, which measures the shipped path over real documents instead of asserting it works. Measured, union of all eight: ns classes is_a rel max num id==0 obsolete BFO 35 137 0 182 0 0 COB 26 30 16 1300 0 9 IAO 216 373 76 8000020 0 22 OBI 4317 5928 1443 9999994 0 108 OBCS 370 374 58 375 0 0 SEPIO 62 46 16 435 1 15 ECO 2299 3780 3 9000001 1 85 FBbi 662 575 49 100028 1 107 TOTAL 7987 11243 1661 1634 of 1661 typed edges land on Predicate::Other — reported, not guessed. Two corrections this probe forced, recorded rather than quietly fixed: - I earlier described the spine's bake order as "derived, not chosen," implying a DAG. It is cyclic: OBI<->IAO (394/10), OBI<->COB (236/3), OBI<->OBCS (58/70). Nothing depends on acyclicity — bake takes the union and sorts by (ns, num) — so the doc now says order is a convention that fixes row order and therefore the digest, not a topological sort. - The class counts supersede an earlier throwaway-script probe (OBI 4286->4317, IAO 210->216, SEPIO 57->62, COB 23->26, ECO 2303->2299, total 7946->7987). These come from the shipped path. The id==0 measurement corroborates the earlier finding exactly — SEPIO, ECO and FBbi each carry a real term at numeric 0, which edges.rs uses as the lane pad meaning "lane spent, advance". An edge into such a term truncates a walk silently, so META_STUDY_SPINE is documented as NOT bakeable for those three until the +1 bias lands in edges.rs. That fix moves every shipped slab digest and forces a re-bake of all five core tables, so it stays an operator call and is not a side effect of adding a table. 42 tests pass; clippy -D warnings clean with and without the feature. --- crates/ogar-obo/Cargo.toml | 8 + crates/ogar-obo/examples/probe_spine.rs | 197 ++++++++ crates/ogar-obo/src/lib.rs | 2 + crates/ogar-obo/src/rdf.rs | 640 ++++++++++++++++++++++++ crates/ogar-obo/src/registry.rs | 71 +++ 5 files changed, 918 insertions(+) create mode 100644 crates/ogar-obo/examples/probe_spine.rs create mode 100644 crates/ogar-obo/src/rdf.rs diff --git a/crates/ogar-obo/Cargo.toml b/crates/ogar-obo/Cargo.toml index 2414cf82..85ed3575 100644 --- a/crates/ogar-obo/Cargo.toml +++ b/crates/ogar-obo/Cargo.toml @@ -11,9 +11,17 @@ description = "OBO-core reference bake — MONDO (disease) · HPO (phenotype) · [features] default = [] serde = ["dep:serde"] +# RDF front-end. Off by default so the crate keeps its "deps: ogar-vocab only" +# lean profile; the three ox* crates are pure Rust (no C, no build script) and +# are pinned to the SAME versions lance-graph-ontology pins, so a workspace +# that builds both resolves one copy of oxrdf. +rdf = ["dep:oxrdf", "dep:oxrdfxml", "dep:oxttl"] [dependencies] ogar-vocab = { path = "../ogar-vocab" } serde = { workspace = true, optional = true } +oxrdf = { version = "0.3", optional = true } +oxrdfxml = { version = "0.2", optional = true } +oxttl = { version = "0.2", optional = true } [dev-dependencies] diff --git a/crates/ogar-obo/examples/probe_spine.rs b/crates/ogar-obo/examples/probe_spine.rs new file mode 100644 index 00000000..dd2d2aae --- /dev/null +++ b/crates/ogar-obo/examples/probe_spine.rs @@ -0,0 +1,197 @@ +//! Measure what the RDF front-end actually harvests from the meta-study spine. +//! +//! This is the falsifier for [`ogar_obo::rdf`]: it runs the *shipped* code path +//! over real ontology documents and reports numbers, rather than asserting the +//! path works. It also measures the numeric-`0` pad collision that gates +//! [`ogar_obo::registry::META_STUDY_SPINE`] — so the gate is a measurement, not +//! a claim. +//! +//! ```text +//! cargo run -p ogar-obo --features rdf --example probe_spine -- +//! ``` +//! +//! `` holds one document per namespace, named `.` (the +//! extension picks the serialization via `rdf::detect_format`). A missing file +//! is reported, never silently skipped — a spine that quietly measured 4 of 8 +//! namespaces would read as a complete result. + +use std::collections::BTreeMap; +use std::path::Path; + +use ogar_obo::rdf::{self, OBO_RESTRICTION_PREDICATES, RdfFormat}; +use ogar_obo::registry::{META_STUDY_SPINE, NsRegistry}; +use ogar_obo::{OboNode, Predicate, TermId}; + +fn main() { + let dir = std::env::args().nth(1).unwrap_or_else(|| { + eprintln!("usage: probe_spine "); + std::process::exit(2); + }); + let dir = Path::new(&dir); + let reg = &META_STUDY_SPINE; + + // Harvest every namespace into ONE map: the spine is a single graph whose + // members import each other, and the cross-ontology edge count below is + // only meaningful across the union. + let mut union: std::collections::HashMap = std::collections::HashMap::new(); + let mut missing: Vec<&str> = Vec::new(); + let mut per_file: Vec<(String, usize)> = Vec::new(); + + for spec in reg.specs() { + let Some((path, bytes)) = read_named(dir, spec.prefix) else { + missing.push(spec.prefix); + continue; + }; + let fmt = rdf::detect_format( + path.extension().and_then(|e| e.to_str()), + &bytes[..bytes.len().min(256)], + ); + match rdf::parse_rdf(&bytes, fmt, reg, OBO_RESTRICTION_PREDICATES) { + Ok(nodes) => { + per_file.push((format!("{} ({:?})", path.display(), fmt), nodes.len())); + for (id, node) in nodes { + let slot = union.entry(id).or_default(); + slot.is_a.extend(node.is_a); + slot.rel.extend(node.rel); + slot.xref.extend(node.xref); + slot.obsolete |= node.obsolete; + } + } + Err(e) => { + eprintln!("!! {}: {e}", path.display()); + missing.push(spec.prefix); + } + } + } + + for (f, n) in &per_file { + println!("read {f}: {n} classes"); + } + if !missing.is_empty() { + println!("\n!! NOT MEASURED: {missing:?} — the totals below exclude them"); + } + + // Per-namespace census, keyed by ordinal so the report follows table order. + let mut own: BTreeMap = BTreeMap::new(); + for (id, node) in &union { + let c = own.entry(id.ns).or_default(); + c.classes += 1; + c.obsolete += usize::from(node.obsolete); + c.is_a += node.is_a.len(); + c.rel += node.rel.len(); + c.max_num = c.max_num.max(id.num); + if id.num == 0 { + c.zero_id += 1; + } + c.max_is_a = c.max_is_a.max(node.is_a.len()); + for parent in &node.is_a { + if parent.ns != id.ns { + *c.cross.entry(parent.ns).or_default() += 1; + } + } + for (p, o) in &node.rel { + if *p == Predicate::Other { + c.rel_other += 1; + } + if o.ns != id.ns { + *c.cross.entry(o.ns).or_default() += 1; + } + } + } + + println!( + "\n{:<7} {:>4} {:>8} {:>7} {:>7} {:>6} {:>9} {:>7} {:>6}", + "ns", "ord", "classes", "is_a", "maxIsA", "rel", "max num", "id==0", "obs" + ); + let (mut t_c, mut t_i, mut t_r, mut t_z) = (0, 0, 0, 0); + for (ord, c) in &own { + println!( + "{:<7} {:>4} {:>8} {:>7} {:>7} {:>6} {:>9} {:>7} {:>6}", + name(reg, *ord), + ord, + c.classes, + c.is_a, + c.max_is_a, + c.rel, + c.max_num, + c.zero_id, + c.obsolete + ); + t_c += c.classes; + t_i += c.is_a; + t_r += c.rel; + t_z += c.zero_id; + } + println!( + "{:<7} {:>4} {:>8} {:>7} {:>7} {:>6}", + "TOTAL", "", t_c, t_i, "", t_r + ); + + println!("\ncross-namespace edges (subject ns -> object ns : count)"); + for (ord, c) in &own { + for (other, n) in &c.cross { + println!(" {} -> {} : {n}", name(reg, *ord), name(reg, *other)); + } + } + + let rel_other: usize = own.values().map(|c| c.rel_other).sum(); + println!( + "\ntyped edges landing on Predicate::Other: {rel_other} of {t_r} \ + (no CURIE row in OBO_RESTRICTION_PREDICATES — reported, never guessed)" + ); + + // The gate, measured. + println!("\n── numeric-0 pad collision (gates META_STUDY_SPINE) ──"); + if t_z == 0 { + println!(" none — no term in this spine carries numeric 0"); + } else { + for (ord, c) in &own { + if c.zero_id > 0 { + println!( + " {} has a real term at numeric 0 — an edge into it is \ + indistinguishable from the lane pad", + name(reg, *ord) + ); + } + } + println!( + " => {t_z} affected namespace(s). Baking them is unsound until the \ + edges.rs +1 bias lands (operator call: it moves every shipped digest)." + ); + } +} + +#[derive(Default)] +struct Census { + classes: usize, + obsolete: usize, + is_a: usize, + max_is_a: usize, + rel: usize, + rel_other: usize, + max_num: u32, + zero_id: usize, + cross: BTreeMap, +} + +fn name(reg: &NsRegistry, ord: u8) -> &'static str { + reg.spec(ord).map_or("?", |s| s.prefix) +} + +/// Find `/.` — the serialization is decided by +/// `detect_format`, not by the caller guessing the extension. +fn read_named(dir: &Path, prefix: &str) -> Option<(std::path::PathBuf, Vec)> { + let entries = std::fs::read_dir(dir).ok()?; + for e in entries.flatten() { + let p = e.path(); + if p.file_stem().and_then(|s| s.to_str()) == Some(prefix) { + let bytes = std::fs::read(&p).ok()?; + return Some((p, bytes)); + } + } + None +} + +// Keep the unused-import lint honest when the format enum is only used via +// `detect_format`'s return value. +const _: fn() -> RdfFormat = || RdfFormat::Turtle; diff --git a/crates/ogar-obo/src/lib.rs b/crates/ogar-obo/src/lib.rs index 24d8ae6b..f9c354a4 100644 --- a/crates/ogar-obo/src/lib.rs +++ b/crates/ogar-obo/src/lib.rs @@ -42,6 +42,8 @@ pub mod crosswalk; pub mod edges; +#[cfg(feature = "rdf")] +pub mod rdf; pub mod reason; pub mod registry; pub mod spine; diff --git a/crates/ogar-obo/src/rdf.rs b/crates/ogar-obo/src/rdf.rs new file mode 100644 index 00000000..6eff4e7e --- /dev/null +++ b/crates/ogar-obo/src/rdf.rs @@ -0,0 +1,640 @@ +//! # `rdf` — retain the assertions, key them by the CURIE numeric. +//! +//! The workspace already parses RDF: `lance-graph-ontology`'s `OwlHydrator` +//! reads Turtle and RDF/XML through `oxttl` / `oxrdfxml`. But it interns every +//! named IRI to a **sequential counter** and then *drops the triple* — its +//! `OntologySlot` is `{ entity_count, iri_to_id }` and nothing else. Two +//! consequences make it unusable as a bake front-end: +//! +//! 1. **No assertions survive.** There is no `subClassOf` anywhere after +//! hydration; `ContextBundle::edge_types` is a whitelist of predicate IRIs +//! a traversal *may* follow, not stored edges. +//! 2. **The identity is order-dependent.** An intern counter depends on file +//! order, so the same ontology hydrated from a different file list yields +//! different ids. A bake keyed on it would not be reproducible. +//! +//! This module is the opposite of both: it **retains** the edges and takes the +//! identity from the term itself — `…/obo/HP_0000118` → `HP:0000118` → the +//! 24-bit numeric `118`, which is exactly the identity [`crate::pack_key`] +//! writes on the `family:identity` rail. Bake the same ontology from any file +//! order and the rows are byte-identical. +//! +//! It is NOT a second parser: `oxttl` / `oxrdfxml` do the reading, pinned to +//! the same versions `lance-graph-ontology` pins. +//! +//! ## The extraction map IS the [`NsRegistry`] +//! +//! Every OBO PURL is `_`. One rule decodes all of them, +//! so a new ontology needs a **table row**, never a code path. That is the +//! whole per-ontology config: prefix + concept id. +//! +//! ## Predicates come from `owl:onProperty`, not from a namespace-pair guess +//! +//! [`crate::parse_obo`] classifies an edge by *which two ontologies it joins* +//! ([`crate::classify`]) — a deliberate choice, because the `.obo` +//! `relationship:` label string is unreliable. RDF does not have that problem: +//! a restriction names its property explicitly, so this path reads +//! [`OBO_RESTRICTION_PREDICATES`] and falls through to +//! [`Predicate::Other`](crate::Predicate::Other) rather than guessing. +//! +//! **This means an RDF bake and a `.obo` bake of the same ontology may assign +//! different predicates to the same edge.** That is a real difference between +//! two sources, not a bug in either; the shipped five-namespace bake was +//! produced from `.obo` and is not re-baked by this module. +//! +//! ## Documented limits (not silent gaps) +//! +//! - Only **direct** restrictions are harvested: `X subClassOf [ onProperty P ; +//! someValuesFrom Y ]` and the same shape under `owl:equivalentClass`. +//! Restrictions nested inside an `owl:intersectionOf` RDF **collection** are +//! NOT walked, so logical definitions expressed that way yield no edge here. +//! - `owl:equivalentClass` to a **named** class is skipped. Equivalence is not +//! subsumption, and recording it as `is_a` would invent a parent. +//! - A term becomes a node only if it is asserted `a owl:Class`. Being merely +//! *mentioned* as some other term's parent does not create a row — the same +//! rule `.obo` follows with its `id:` stanza line. + +use std::collections::HashMap; + +use oxrdf::{NamedOrBlankNode, Term}; +use oxrdfxml::RdfXmlParser; +use oxttl::TurtleParser; + +use crate::registry::NsRegistry; +use crate::{OboNode, Predicate, TermId, Xref, XrefSource}; + +/// The OBO PURL base every OBO Foundry term IRI is built on. +pub const OBO_PURL_BASE: &str = "http://purl.obolibrary.org/obo/"; + +const RDF_TYPE: &str = "http://www.w3.org/1999/02/22-rdf-syntax-ns#type"; +const OWL_CLASS: &str = "http://www.w3.org/2002/07/owl#Class"; +const RDFS_SUBCLASS_OF: &str = "http://www.w3.org/2000/01/rdf-schema#subClassOf"; +const OWL_ON_PROPERTY: &str = "http://www.w3.org/2002/07/owl#onProperty"; +const OWL_SOME_VALUES_FROM: &str = "http://www.w3.org/2002/07/owl#someValuesFrom"; +const OWL_EQUIVALENT_CLASS: &str = "http://www.w3.org/2002/07/owl#equivalentClass"; +const OWL_DEPRECATED: &str = "http://www.w3.org/2002/07/owl#deprecated"; +const OBO_IN_OWL_HAS_DB_XREF: &str = "http://www.geneontology.org/formats/oboInOwl#hasDbXref"; + +/// The RDF serializations this module reads. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RdfFormat { + /// Turtle / N-Triples family (`oxttl`). + Turtle, + /// RDF/XML (`oxrdfxml`). + RdfXml, +} + +/// Choose a serialization from the file extension, falling back to sniffing +/// the head of the document. +/// +/// `.owl` is genuinely ambiguous — it ships in both forms in the wild — so an +/// unknown or `.owl` extension sniffs for an XML preamble or an `` +/// opening tag and defaults to Turtle otherwise. Same rule +/// `lance-graph-ontology` uses, so the two agree on any given file. +#[must_use] +pub fn detect_format(extension: Option<&str>, head: &[u8]) -> RdfFormat { + match extension.map(str::to_ascii_lowercase).as_deref() { + Some("rdf" | "rdfxml" | "owx") => RdfFormat::RdfXml, + Some("ttl" | "nt" | "n3" | "trig") => RdfFormat::Turtle, + _ => { + let head = &head[..head.len().min(256)]; + let start = head + .iter() + .position(|b| !b.is_ascii_whitespace() && *b != 0xEF && *b != 0xBB && *b != 0xBF) + .unwrap_or(0); + let trimmed = &head[start..]; + if trimmed.starts_with(b") -> std::fmt::Result { + write!(f, "RDF parse error: {}", self.message) + } +} + +impl std::error::Error for RdfErr {} + +/// Split an OBO PURL into its `(prefix, 24-bit numeric)` parts. +/// +/// `None` when the IRI is not an OBO PURL, has no `_` separator, or the +/// numeric is malformed or exceeds 24 bits — the same width contract +/// [`TermId::parse`] enforces. +#[must_use] +pub fn purl_to_curie(iri: &str) -> Option<(&str, u32)> { + let local = iri.strip_prefix(OBO_PURL_BASE)?; + let (prefix, digits) = local.rsplit_once('_')?; + if prefix.is_empty() { + return None; + } + let num: u32 = digits.parse().ok()?; + if num > 0x00FF_FFFF { + return None; + } + Some((prefix, num)) +} + +/// Resolve an OBO PURL to a [`TermId`] under a registry, or `None` if the +/// prefix is outside the table (imports from ontologies this bake does not +/// tile are skipped, exactly as [`crate::parse_obo`] skips them). +#[must_use] +pub fn term_of(iri: &str, reg: &NsRegistry) -> Option { + let (prefix, num) = purl_to_curie(iri)?; + Some(TermId { + ns: reg.ordinal_of(prefix)?, + num, + }) +} + +/// Map a restriction's property IRI to an RO predicate through a table. +#[must_use] +pub fn predicate_of(property_iri: &str, table: &[PredSpec]) -> Predicate { + let Some((prefix, num)) = purl_to_curie(property_iri) else { + return Predicate::Other; + }; + table + .iter() + .find(|s| { + s.curie + .split_once(':') + .is_some_and(|(p, n)| p == prefix && n.parse::().ok() == Some(num)) + }) + .map_or(Predicate::Other, |s| s.predicate) +} + +/// A partially-seen `owl:Restriction` blank node. +#[derive(Default)] +struct Restriction { + property: Option, + filler: Option, +} + +/// Everything collected in the single streaming pass, before blank nodes are +/// resolved against the restrictions they refer to. +#[derive(Default)] +struct Harvest { + nodes: HashMap, + restrictions: HashMap, + blank_edges: Vec<(TermId, String)>, +} + +impl Harvest { + fn absorb( + &mut self, + subject: &NamedOrBlankNode, + predicate: &str, + object: &Term, + reg: &NsRegistry, + ) { + match subject { + NamedOrBlankNode::NamedNode(s) => { + let Some(sid) = term_of(s.as_str(), reg) else { + return; + }; + match (predicate, object) { + // `a owl:Class` is what creates a row — mere mention does not. + (RDF_TYPE, Term::NamedNode(o)) if o.as_str() == OWL_CLASS => { + self.nodes.entry(sid).or_default(); + } + (RDFS_SUBCLASS_OF, Term::NamedNode(o)) => { + if let Some(parent) = term_of(o.as_str(), reg) { + self.nodes.entry(sid).or_default().is_a.push(parent); + } + } + // A restriction hangs off a blank node; resolve it later. + (RDFS_SUBCLASS_OF | OWL_EQUIVALENT_CLASS, Term::BlankNode(b)) => { + self.blank_edges.push((sid, b.as_str().to_string())); + } + (OWL_DEPRECATED, Term::Literal(l)) if l.value() == "true" => { + self.nodes.entry(sid).or_default().obsolete = true; + } + (OBO_IN_OWL_HAS_DB_XREF, Term::Literal(l)) => { + if let Some((src, id)) = l.value().split_once(':') + && !src.is_empty() + && !id.is_empty() + { + self.nodes.entry(sid).or_default().xref.push(Xref { + source: XrefSource::from_prefix(src), + id: id.to_string(), + }); + } + } + _ => {} + } + } + NamedOrBlankNode::BlankNode(b) => { + let slot = self.restrictions.entry(b.as_str().to_string()).or_default(); + match (predicate, object) { + (OWL_ON_PROPERTY, Term::NamedNode(o)) => { + slot.property = Some(o.as_str().to_string()); + } + (OWL_SOME_VALUES_FROM, Term::NamedNode(o)) => { + slot.filler = Some(o.as_str().to_string()); + } + _ => {} + } + } + } + } + + /// Join the blank-node edges to their restrictions. A blank node with no + /// `onProperty`/`someValuesFrom` pair — an intersection list, a cardinality + /// restriction, an anonymous union — yields nothing, per the documented + /// limits. + fn resolve(mut self, reg: &NsRegistry, table: &[PredSpec]) -> HashMap { + for (sid, blank) in std::mem::take(&mut self.blank_edges) { + let Some(r) = self.restrictions.get(&blank) else { + continue; + }; + let (Some(prop), Some(filler)) = (r.property.as_deref(), r.filler.as_deref()) else { + continue; + }; + let Some(object) = term_of(filler, reg) else { + continue; + }; + let p = predicate_of(prop, table); + self.nodes.entry(sid).or_default().rel.push((p, object)); + } + // Deterministic edge order regardless of triple arrival order — the + // bake packs `is_a` then `rel` into value lanes positionally, so an + // unstable order here would produce a different (still correct) row. + for node in self.nodes.values_mut() { + node.is_a.sort_unstable(); + node.is_a.dedup(); + node.rel.sort_unstable(); + node.rel.dedup(); + } + self.nodes + } +} + +/// Parse an RDF document into the `(id → node)` map [`crate::bake`] consumes. +/// +/// Streams the triples once, then joins the blank-node restrictions. Terms +/// whose prefix is outside `reg` are skipped, so an ontology's imports do not +/// leak rows into a bake that was not asked to tile them. +/// +/// # Errors +/// +/// Returns [`RdfErr`] if the underlying `oxttl` / `oxrdfxml` parser rejects +/// the document. +pub fn parse_rdf( + bytes: &[u8], + format: RdfFormat, + reg: &NsRegistry, + table: &[PredSpec], +) -> Result, RdfErr> { + let mut h = Harvest::default(); + match format { + RdfFormat::Turtle => { + for t in TurtleParser::new().for_slice(bytes) { + let t = t.map_err(|e| RdfErr { + message: e.to_string(), + })?; + h.absorb(&t.subject, t.predicate.as_str(), &t.object, reg); + } + } + RdfFormat::RdfXml => { + for t in RdfXmlParser::new().for_slice(bytes) { + let t = t.map_err(|e| RdfErr { + message: e.to_string(), + })?; + h.absorb(&t.subject, t.predicate.as_str(), &t.object, reg); + } + } + } + Ok(h.resolve(reg, table)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::registry::{NsRegistry, NsSpec}; + + const SPINE: NsRegistry = NsRegistry::new(&[ + NsSpec { + prefix: "BFO", + concept_id: 0x0310, + }, + NsSpec { + prefix: "UBERON", + concept_id: 0x0303, + }, + ]); + + /// Turtle with: two classes, a named subClassOf, a blank-node part_of + /// restriction, a deprecation, an xref, an out-of-registry parent, and a + /// blank node that is NOT a well-formed restriction. + const TTL: &str = r#" +@prefix rdf: . +@prefix rdfs: . +@prefix owl: . +@prefix obo: . +@prefix oio: . + +obo:UBERON_0000955 a owl:Class ; + rdfs:subClassOf obo:UBERON_0000062 ; + rdfs:subClassOf obo:CHEBI_0000001 ; + rdfs:subClassOf [ a owl:Restriction ; + owl:onProperty obo:BFO_0000050 ; + owl:someValuesFrom obo:UBERON_0000033 ] ; + oio:hasDbXref "MeSH:D001921" . + +obo:UBERON_0000062 a owl:Class . +obo:UBERON_0000033 a owl:Class . + +obo:UBERON_0000099 a owl:Class ; + owl:deprecated "true" . + +obo:UBERON_0000077 a owl:Class ; + rdfs:subClassOf [ a owl:Class ; + owl:unionOf ( obo:UBERON_0000062 obo:UBERON_0000033 ) ] . +"#; + + fn u(num: u32) -> TermId { + TermId { ns: 1, num } + } + + #[test] + fn purl_decoding_accepts_obo_iris_and_rejects_everything_else() { + assert_eq!( + purl_to_curie("http://purl.obolibrary.org/obo/HP_0000118"), + Some(("HP", 118)) + ); + // The LAST underscore separates — some OBO locals carry one. + assert_eq!( + purl_to_curie("http://purl.obolibrary.org/obo/NCBITaxon_Union_0000001"), + Some(("NCBITaxon_Union", 1)) + ); + // Rejects: wrong base, no separator, non-numeric, over 24 bits. + assert_eq!(purl_to_curie("http://example.org/HP_0000118"), None); + assert_eq!( + purl_to_curie("http://purl.obolibrary.org/obo/HP0000118"), + None + ); + assert_eq!(purl_to_curie("http://purl.obolibrary.org/obo/HP_abc"), None); + assert_eq!( + purl_to_curie("http://purl.obolibrary.org/obo/HP_99999999"), + None, + "the 24-bit width contract must hold here as it does in TermId::parse" + ); + assert_eq!(purl_to_curie("http://purl.obolibrary.org/obo/_1"), None); + } + + #[test] + fn the_registry_is_the_extraction_map() { + // UBERON is in SPINE, HP is not. Same IRI shape, different outcome — + // so the table is genuinely consulted rather than a prefix hardcoded. + assert_eq!( + term_of("http://purl.obolibrary.org/obo/UBERON_0000955", &SPINE), + Some(u(955)) + ); + assert_eq!( + term_of("http://purl.obolibrary.org/obo/HP_0000118", &SPINE), + None + ); + // …and the ordinal is table-relative: UBERON is row 1 here, row 2 in + // the shipped core. + assert_eq!( + term_of("http://purl.obolibrary.org/obo/UBERON_0000955", &SPINE) + .unwrap() + .ns, + 1 + ); + assert_eq!( + term_of( + "http://purl.obolibrary.org/obo/UBERON_0000955", + &crate::registry::OBO_CORE + ) + .unwrap() + .ns, + 2 + ); + } + + #[test] + fn restriction_properties_map_through_the_table_and_fall_through_otherwise() { + assert_eq!( + predicate_of( + "http://purl.obolibrary.org/obo/BFO_0000050", + OBO_RESTRICTION_PREDICATES + ), + Predicate::PartOf + ); + assert_eq!( + predicate_of( + "http://purl.obolibrary.org/obo/RO_0002200", + OBO_RESTRICTION_PREDICATES + ), + Predicate::HasPhenotype + ); + // An OBO property with no row → Other, never a guess. + assert_eq!( + predicate_of( + "http://purl.obolibrary.org/obo/RO_0002131", + OBO_RESTRICTION_PREDICATES + ), + Predicate::Other + ); + // A non-OBO property IRI → Other. + assert_eq!( + predicate_of( + "http://www.w3.org/2000/01/rdf-schema#seeAlso", + OBO_RESTRICTION_PREDICATES + ), + Predicate::Other + ); + // An EMPTY table must classify everything as Other — proof the table + // is read rather than the mapping being baked into the function. + assert_eq!( + predicate_of("http://purl.obolibrary.org/obo/BFO_0000050", &[]), + Predicate::Other + ); + } + + #[test] + fn turtle_harvest_keeps_edges_and_drops_what_it_documents_dropping() { + let nodes = parse_rdf( + TTL.as_bytes(), + RdfFormat::Turtle, + &SPINE, + OBO_RESTRICTION_PREDICATES, + ) + .expect("turtle parses"); + + // Four `a owl:Class` assertions → four rows. UBERON_0000033 and + // _0000062 are declared classes AND referenced; CHEBI is referenced + // but outside the registry, so it must NOT create a row. + assert_eq!(nodes.len(), 5, "one row per `a owl:Class`, none for CHEBI"); + + let brain = &nodes[&u(955)]; + // The named parent is kept; the out-of-registry one is skipped. + assert_eq!(brain.is_a, vec![u(62)], "CHEBI parent must be skipped"); + // The blank-node restriction resolved to a typed edge. + assert_eq!(brain.rel, vec![(Predicate::PartOf, u(33))]); + // The xref survived. + assert_eq!(brain.xref.len(), 1); + assert_eq!(brain.xref[0].source, XrefSource::Mesh); + assert_eq!(brain.xref[0].id, "D001921"); + + assert!(nodes[&u(99)].obsolete, "owl:deprecated true is read"); + assert!(!brain.obsolete); + + // The documented limit, asserted rather than hoped: a blank node that + // is a unionOf — not an onProperty/someValuesFrom restriction — + // contributes NO edge. + assert!( + nodes[&u(77)].rel.is_empty() && nodes[&u(77)].is_a.is_empty(), + "a non-restriction blank node must yield nothing, not a wrong edge" + ); + } + + /// The property this module exists for: identity comes from the term, so + /// the harvest cannot depend on document order. + /// + /// **What would make it fail:** an intern-counter identity, which is what + /// `lance-graph-ontology`'s `OwlHydrator` assigns. Reversing the stanzas + /// would then renumber every term. + #[test] + fn identity_is_the_curie_numeric_so_document_order_cannot_change_it() { + let forward = parse_rdf( + TTL.as_bytes(), + RdfFormat::Turtle, + &SPINE, + OBO_RESTRICTION_PREDICATES, + ) + .unwrap(); + + // Re-emit the same graph with the class stanzas in reverse order. + let mut stanzas: Vec<&str> = TTL.split("\n\n").collect(); + let header = stanzas.remove(0); + stanzas.reverse(); + let reordered = format!("{header}\n\n{}", stanzas.join("\n\n")); + let backward = parse_rdf( + reordered.as_bytes(), + RdfFormat::Turtle, + &SPINE, + OBO_RESTRICTION_PREDICATES, + ) + .unwrap(); + + assert_eq!(forward.len(), backward.len()); + for (id, node) in &forward { + let other = backward.get(id).expect("same ids under any order"); + assert_eq!(node.is_a, other.is_a, "{id:?} parents must match"); + assert_eq!(node.rel, other.rel, "{id:?} typed edges must match"); + assert_eq!(node.obsolete, other.obsolete); + } + // The reorder must have actually changed the bytes, or this proves + // nothing about order-independence. + assert_ne!(reordered.as_str(), TTL); + } + + #[test] + fn rdfxml_reads_the_same_graph_as_turtle() { + const XML: &str = r#" + + + + + + + + + + + +"#; + let nodes = parse_rdf( + XML.as_bytes(), + RdfFormat::RdfXml, + &SPINE, + OBO_RESTRICTION_PREDICATES, + ) + .expect("rdf/xml parses"); + let brain = &nodes[&u(955)]; + assert_eq!(brain.is_a, vec![u(62)]); + assert_eq!(brain.rel, vec![(Predicate::PartOf, u(33))]); + } + + #[test] + fn a_malformed_document_errors_rather_than_returning_an_empty_graph() { + let e = parse_rdf( + b"this is not turtle @@@", + RdfFormat::Turtle, + &SPINE, + OBO_RESTRICTION_PREDICATES, + ); + assert!( + e.is_err(), + "a parse failure must not look like an empty ontology" + ); + } + + #[test] + fn format_detection_prefers_the_extension_then_sniffs() { + assert_eq!(detect_format(Some("ttl"), b" Date: Sat, 8 Aug 2026 12:37:40 +0000 Subject: [PATCH 08/13] =?UTF-8?q?ogar-obo:=20A1=20=E2=80=94=20the=20+1=20n?= =?UTF-8?q?umeric=20bias,=20and=20the=20spine=20bake=20it=20unblocks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The lane pad was all-zero bytes, and the encoding relied on an assertion to keep that unambiguous: "no OBO CURIE numeric is 0". That held for the five baked namespaces and is false in general — probe_spine measured a real term at numeric 0 in three further namespaces, one of them a namespace root. The failure shape was the bad one. Removing the bias and running the new test shows it directly: left: [] right: [0, 118] Not [118]. BOTH links vanish. A zero slot does not drop one edge, it ends the lane, so every target after it in that lane is lost too — and the artifact still validates, so nothing anywhere reports it. A traversal simply returns less than the graph contains. Fix: store the numeric biased by one (encode_numeric/decode_numeric). Stored 0 means pad by construction rather than by a property of the input. MAX_LINK_NUMERIC is 2^24-2 = 16_777_214 against a measured maximum of 9_999_994. The one unrepresentable numeric is reported as dropped, never wrapped — with a test, because a guard that cannot be shown to fire is the defect one level up. Re-baked the core, as the gate required. Measured, not assumed: pre-A1 c88b44ea85a11bcf1e820b2aa03000fed157e39760ab3e5b83d40697a9f30bfb post-A1 424e81cce7050b593fbd61c3507530b3b80c5471d6f7180e5e834165b69b0ae6 186,567 of 35,224,064 bytes differ (0.53%), 68,789 of 68,797 rows (the 8 unchanged rows are the ones with no out-edges) Same graph, different bytes: rows 68,797, triples 186,029, links packed 186,029, dropped 0, overflowed 0, is_a cycles 0, dangling 0, saturation 738,651 subsumption pairs — every one identical to the shipped bake. verify_edges re-checks every lane against the SPO triple table and the is_a degree totals are unchanged (RO 2 / UBERON 27,170 / HP 31,284 / PATO 2,909 / MONDO 46,448). bake_obo did not print links_dropped / rows_overflowed at all. They were computed and thrown away, so a run could look clean while the artifact was missing edges — the same silent shape A1 is about, one level up. Now printed, and asserted rather than warned about. A1 existed to unblock the spine, so this finishes the job rather than stopping at the bias. `bake` resolved classids through the hardcoded Namespace enum, whose ordinal array panics for any ordinal >= 5, so no table but OBO_CORE could bake. Added `bake_with(nodes, app_prefix, reg)`; `bake` is that against OBO_CORE. Verified digest-neutral — the core re-bakes to 424e81cc unchanged. A target whose namespace is absent from the registry is reported as dropped, never guessed at. examples/bake_spine.rs bakes all eight spine namespaces through the table with no per-ontology code: rows 7,641 · triples 12,885 · packed 12,885 · dropped 0 · overflowed 0 cycles 0 · dangling 0 BFO 35 · COB 17 · IAO 194 · OBI 4,209 · OBCS 370 · SEPIO 47 · ECO 2,214 · FBbi 555 sha256 437ea33aa7dda76bb37f13fd14849e8a339d71e354d7d90cea595b80875396c5 and asserts the gate directly: 10 edges point at a numeric-0 term, and 10 of 10 are read back out of the baked lanes. Before the bias each of those 10 truncated its lane. 1,628 of 12,885 typed edges land on Predicate::Other — no CURIE row matched their owl:onProperty. Reported, not guessed. Also declares required-features for the two rdf examples so `clippy --all-targets` on the default feature set skips them rather than failing. 46 tests pass; clippy -D warnings clean with and without the feature. --- crates/ogar-obo/Cargo.toml | 11 ++ crates/ogar-obo/examples/bake_obo.rs | 21 +++ crates/ogar-obo/examples/bake_spine.rs | 160 +++++++++++++++++ crates/ogar-obo/src/edges.rs | 235 ++++++++++++++++++++++--- crates/ogar-obo/src/lib.rs | 26 ++- crates/ogar-obo/src/spine.rs | 6 +- 6 files changed, 431 insertions(+), 28 deletions(-) create mode 100644 crates/ogar-obo/examples/bake_spine.rs diff --git a/crates/ogar-obo/Cargo.toml b/crates/ogar-obo/Cargo.toml index 85ed3575..390da108 100644 --- a/crates/ogar-obo/Cargo.toml +++ b/crates/ogar-obo/Cargo.toml @@ -25,3 +25,14 @@ oxrdfxml = { version = "0.2", optional = true } oxttl = { version = "0.2", optional = true } [dev-dependencies] + +# These drive the RDF front-end, so they cannot build without it. Declared +# rather than #[cfg]-guarded so `cargo clippy --all-targets` on the default +# feature set skips them instead of failing. +[[example]] +name = "probe_spine" +required-features = ["rdf"] + +[[example]] +name = "bake_spine" +required-features = ["rdf"] diff --git a/crates/ogar-obo/examples/bake_obo.rs b/crates/ogar-obo/examples/bake_obo.rs index d2d13ec8..d6d36716 100644 --- a/crates/ogar-obo/examples/bake_obo.rs +++ b/crates/ogar-obo/examples/bake_obo.rs @@ -64,6 +64,27 @@ fn main() { " xrefs preserved : {} (MeSH bearings: {})", s.xrefs, s.mesh_xrefs ); + // The ship gate. These were computed but never printed, so a driver run + // could look clean while the artifact was missing edges — exactly the + // silent shape gate A1 was about. Print them, and refuse rather than warn. + println!(" links packed (col_idx) : {}", s.links_packed); + println!(" links DROPPED : {}", s.links_dropped); + println!(" rows overflowed : {}", s.rows_overflowed); + assert_eq!( + s.links_dropped, 0, + "refusing to write an artifact that is missing {} edge(s)", + s.links_dropped + ); + assert_eq!( + s.rows_overflowed, 0, + "refusing to write an artifact with {} truncated row(s)", + s.rows_overflowed + ); + assert_eq!( + s.links_packed, s.triples, + "every SPO triple must also be in-row: {} packed vs {} triples", + s.links_packed, s.triples + ); // Per-namespace row census. let mut census: HashMap = HashMap::new(); diff --git a/crates/ogar-obo/examples/bake_spine.rs b/crates/ogar-obo/examples/bake_spine.rs new file mode 100644 index 00000000..729f3e01 --- /dev/null +++ b/crates/ogar-obo/examples/bake_spine.rs @@ -0,0 +1,160 @@ +//! Bake the meta-study spine — the leg gate A1 was blocking. +//! +//! ```text +//! cargo run -p ogar-obo --features rdf --example bake_spine -- [out.soa] +//! ``` +//! +//! Reads one RDF document per namespace (`.`), extracts through +//! `ogar_obo::rdf` (assertions retained, identity from the CURIE numeric), and +//! bakes through `bake_with` against +//! [`ogar_obo::registry::META_STUDY_SPINE`] — no code path per ontology, only +//! table rows. +//! +//! Three of these namespaces carry a real term at numeric `0`. Before the A1 +//! `+1` bias those links encoded to the lane pad and truncated their whole +//! lane silently, so this bake could not be trusted. The run asserts the ship +//! gate (`links_dropped == 0`, `rows_overflowed == 0`, +//! `links_packed == triples`) and refuses to write otherwise. + +use std::collections::HashMap; +use std::path::Path; + +use ogar_obo::rdf::{self, OBO_RESTRICTION_PREDICATES}; +use ogar_obo::registry::META_STUDY_SPINE; +use ogar_obo::{OboNode, Predicate, TermId, as_le_bytes, bake_with, rows_from_le_bytes}; + +fn main() { + let mut a = std::env::args().skip(1); + let dir = a.next().unwrap_or_else(|| "/tmp/onto".to_string()); + let out = a.next().unwrap_or_else(|| format!("{dir}/spine.soa")); + let reg = &META_STUDY_SPINE; + + let mut nodes: HashMap = HashMap::new(); + let mut missing: Vec<&str> = Vec::new(); + for spec in reg.specs() { + let Some((path, bytes)) = read_named(Path::new(&dir), spec.prefix) else { + missing.push(spec.prefix); + continue; + }; + let fmt = rdf::detect_format( + path.extension().and_then(|e| e.to_str()), + &bytes[..bytes.len().min(256)], + ); + let part = rdf::parse_rdf(&bytes, fmt, reg, OBO_RESTRICTION_PREDICATES) + .unwrap_or_else(|e| panic!("{}: {e}", path.display())); + println!(" parsed {:6} classes={}", spec.prefix, part.len()); + for (id, n) in part { + let e = nodes.entry(id).or_default(); + e.is_a.extend(n.is_a); + e.rel.extend(n.rel); + e.xref.extend(n.xref); + e.obsolete |= n.obsolete; + } + } + assert!( + missing.is_empty(), + "refusing to bake a partial spine — not read: {missing:?}" + ); + + // The A1 evidence, measured on the real input rather than assumed. + let zeros: Vec<&str> = reg + .specs() + .iter() + .enumerate() + .filter(|(i, _)| { + nodes.contains_key(&TermId { + ns: *i as u8, + num: 0, + }) + }) + .map(|(_, s)| s.prefix) + .collect(); + println!("\n namespaces with a real term at numeric 0: {zeros:?}"); + let zero_in_edges: usize = nodes + .values() + .map(|n| { + n.is_a.iter().filter(|t| t.num == 0).count() + + n.rel.iter().filter(|(_, t)| t.num == 0).count() + }) + .sum(); + println!( + " edges POINTING AT a numeric-0 term : {zero_in_edges} \ + (each one truncated its lane before A1)" + ); + + let baked = bake_with(&nodes, 0x0000, reg); + let s = &baked.stats; + println!("\n=== BAKE ==="); + println!(" rows : {}", s.nodes); + println!(" obsolete : {}", s.obsolete); + println!(" triples : {}", s.triples); + println!(" links packed : {}", s.links_packed); + println!(" links DROPPED : {}", s.links_dropped); + println!(" rows overflowed : {}", s.rows_overflowed); + println!(" is_a cycles : {}", s.is_a_cycles); + println!(" dangling : {}", s.dangling); + assert_eq!(s.links_dropped, 0, "artifact is missing edges"); + assert_eq!(s.rows_overflowed, 0, "artifact has truncated rows"); + assert_eq!(s.links_packed, s.triples, "not every triple is in-row"); + + // Per-namespace census in table (= row) order. + let mut census: HashMap = HashMap::new(); + for id in &baked.ids { + *census.entry(id.ns).or_default() += 1; + } + for (i, spec) in reg.specs().iter().enumerate() { + println!( + " {:6}: {:5} rows classid 0x{:08X}", + spec.prefix, + census.get(&(i as u8)).copied().unwrap_or(0), + reg.render_classid(i as u8, 0x0000).unwrap() + ); + } + + // Read every numeric-0 link back out of the baked lanes — the direct + // proof that the bake is now traversable through those terms. + let mut recovered = 0usize; + for row in &baked.rows { + for (_, l) in ogar_obo::edges::EdgeLanes::new(&row.0).links() { + if l.num == 0 { + recovered += 1; + } + } + } + println!("\n numeric-0 links READ BACK from lanes : {recovered} of {zero_in_edges}"); + assert_eq!( + recovered, zero_in_edges, + "every numeric-0 edge must survive the round trip — this is gate A1" + ); + + let bytes = as_le_bytes(&baked.rows); + std::fs::write(&out, bytes).unwrap_or_else(|e| panic!("write {out}: {e}")); + let inmem = rows_from_le_bytes(bytes).expect("in-memory aligned view"); + assert_eq!(inmem.len(), baked.rows.len()); + println!( + "\n=== ARTIFACT ===\n {out} ({} bytes = {} rows x 512)\n loader contract: VERIFIED", + bytes.len(), + baked.rows.len() + ); + + let other = baked + .triples + .iter() + .filter(|t| t.p == Predicate::Other) + .count(); + println!( + " typed edges on Predicate::Other: {other} of {} (no CURIE row matched)", + s.triples + ); +} + +fn read_named(dir: &Path, prefix: &str) -> Option<(std::path::PathBuf, Vec)> { + for e in std::fs::read_dir(dir).ok()?.flatten() { + let p = e.path(); + if p.file_stem().and_then(|s| s.to_str()) == Some(prefix) { + let b = std::fs::read(&p).ok()?; + return Some((p, b)); + } + } + None +} diff --git a/crates/ogar-obo/src/edges.rs b/crates/ogar-obo/src/edges.rs index e3c1fee7..6a7ca30e 100644 --- a/crates/ogar-obo/src/edges.rs +++ b/crates/ogar-obo/src/edges.rs @@ -42,14 +42,39 @@ //! Targets are emitted ascending by `(predicate, target namespace, numeric)`, //! and **a lane never mixes target ontologies** — its classid names exactly //! one, so a namespace change starts a new lane even mid-predicate. Unused -//! slots in a lane are `0`, which is unambiguous: no OBO CURIE numeric is 0 -//! (asserted by `probe_edge_capacity` A1 on the real sources). +//! slots in a lane are all-zero bytes, and a reader treats a zero slot as +//! "this lane is spent, advance". +//! +//! # The `+1` numeric bias (gate A1, applied 2026-08-08) +//! +//! An all-zero slot is the pad, so the slot encoding must have no *other* +//! meaning for zero. The original contract achieved that by assertion — +//! *"no OBO CURIE numeric is 0"* — which held for the five baked namespaces +//! and **is false in general**. `probe_spine` measured a real term at numeric +//! `0` in three further namespaces, one of them that namespace's own root. An +//! edge into such a term was indistinguishable from a pad, so a reader's walk +//! stopped early and *silently* — the worst failure shape available, because +//! the artifact still validates and the traversal just returns less. +//! +//! The fix removes the assumption instead of documenting around it: the +//! numeric is stored **biased by one** ([`encode_numeric`] / [`decode_numeric`]). +//! Stored `0` now means pad and nothing else, by construction rather than by +//! a property of the input. +//! +//! Cost: one value off the top of the range. [`MAX_LINK_NUMERIC`] is +//! `2²⁴ − 2 = 16_777_214`, against a measured maximum of `9_999_994` across +//! every namespace probed. A numeric above that is **reported as dropped**, +//! never truncated (see [`PackStats::dropped`]). +//! +//! This changed every byte of every edge lane, so it re-baked all five core +//! slabs and moved their digests. That was the price of the gate and it was +//! paid deliberately, not absorbed. //! //! A reader walks lanes in order and consumes `degree(p)` targets for each -//! predicate `p` in ascending order, treating a `0` slot as "this lane is -//! spent, advance". That is [`EdgeLanes`], which borrows the row and decodes a -//! u24 with a shift — no allocation, no gathered window. +//! predicate `p` in ascending order. That is [`EdgeLanes`], which borrows the +//! row and decodes a u24 with a shift — no allocation, no gathered window. +use crate::registry::NsRegistry; use crate::{EDGES_OFFSET, NODE_ROW_STRIDE, Namespace, Predicate, TermId, VALUE_OFFSET}; /// Bytes per lane — the one form (`classid(4) + content(12)`). @@ -63,6 +88,28 @@ pub const EDGE_LANE_COUNT: usize = 23; /// Total link slots per row. pub const EDGE_SLOT_COUNT: usize = EDGE_LANE_COUNT * LINKS_PER_LANE; +/// The largest CURIE numeric an edge lane can carry. +/// +/// One below the u24 ceiling, because the stored form is biased by one so that +/// stored `0` can mean *pad* unconditionally (see the module docs, gate A1). +/// Measured maximum across every namespace probed: `9_999_994`. +pub const MAX_LINK_NUMERIC: u32 = 0x00FF_FFFF - 1; + +/// CURIE numeric → its stored form. Total on `0..=MAX_LINK_NUMERIC`. +#[inline] +#[must_use] +pub const fn encode_numeric(num: u32) -> u32 { + num + 1 +} + +/// Stored form → CURIE numeric. `stored == 0` is the pad and has no numeric, +/// so callers must test for it *before* decoding; [`EdgeLanes::links`] does. +#[inline] +#[must_use] +pub const fn decode_numeric(stored: u32) -> u32 { + stored - 1 +} + const _: () = assert!(EDGE_LANE_SLAB_OFFSET > crate::ENTITY_TYPE_SLAB_OFFSET); const _: () = assert!( VALUE_OFFSET + EDGE_LANE_SLAB_OFFSET + EDGE_LANE_COUNT * LANE_BYTES == NODE_ROW_STRIDE, @@ -114,12 +161,15 @@ pub struct PackStats { /// it into the canonical `(predicate, namespace, numeric)` emit order itself, /// so callers cannot get the reader's contract wrong by accident. /// -/// Returns the per-row stats. Links beyond [`EDGE_SLOT_COUNT`] are counted in -/// [`PackStats::dropped`] and **not** written — never silently truncated. +/// Returns the per-row stats. A link is counted in [`PackStats::dropped`] and +/// **not** written — never silently truncated — when either the lane budget is +/// exhausted or the numeric exceeds [`MAX_LINK_NUMERIC`]. Both mean the same +/// thing to a driver: the artifact is missing an edge, so it must not ship. pub fn pack_edges( row: &mut [u8; NODE_ROW_STRIDE], targets: &mut [(Predicate, TermId)], app_prefix: u16, + reg: &NsRegistry, ) -> PackStats { let mut st = PackStats::default(); targets.sort_unstable_by_key(|(p, t)| (*p as u8, t.ns, t.num)); @@ -129,25 +179,43 @@ pub fn pack_edges( let mut slot = LINKS_PER_LANE; // force a fresh lane on the first target let mut group: Option<(u8, u8)> = None; - for (p, t) in targets.iter() { + for (i, (p, t)) in targets.iter().enumerate() { + if t.num > MAX_LINK_NUMERIC { + // Unrepresentable under the +1 bias — the one numeric the biased + // u24 cannot hold. Reported, not truncated, and not allowed to + // consume a slot: a driver refuses to ship on `dropped != 0`. + st.dropped += 1; + st.rows_overflowed = 1; + continue; + } let g = (*p as u8, t.ns); if group != Some(g) || slot == LINKS_PER_LANE { group = Some(g); lane += 1; slot = 0; if lane > EDGE_LANE_COUNT { - // budget exhausted — count the remainder, write nothing more. - st.dropped = targets.len() - st.written; + // budget exhausted — count this one and every one after it, + // write nothing more. + st.dropped += targets.len() - i; st.rows_overflowed = 1; st.lanes = EDGE_LANE_COUNT; return st; } let off = base + (lane - 1) * LANE_BYTES; - let classid = t.namespace().render_classid(app_prefix); + let Some(classid) = reg.render_classid(t.ns, app_prefix) else { + // A target whose namespace is not in this registry has no + // classid, so the lane could not name it. Report, never guess. + st.dropped += 1; + st.rows_overflowed = 1; + lane -= 1; + slot = LINKS_PER_LANE; + group = None; + continue; + }; row[off..off + 4].copy_from_slice(&classid.to_le_bytes()); } let off = base + (lane - 1) * LANE_BYTES + 4 + slot * 3; - put_u24_le(&mut row[off..off + 3], t.num); + put_u24_le(&mut row[off..off + 3], encode_numeric(t.num)); slot += 1; st.written += 1; } @@ -246,7 +314,8 @@ impl Iterator for LinkIter<'_> { self.row[off + 6 + self.slot * 3], ); if n == 0 { - // pad — the lane is spent, next lane + // Pad — the lane is spent, next lane. Unconditional under the + // +1 bias: no real target encodes to 0, whatever its numeric. self.lane += 1; self.slot = 0; continue; @@ -260,7 +329,13 @@ impl Iterator for LinkIter<'_> { self.slot += 1; self.remaining -= 1; let p = PREDICATES[self.pred as usize]; - return Some((p, Link { classid, num: n })); + return Some(( + p, + Link { + classid, + num: decode_numeric(n), + }, + )); } } } @@ -341,7 +416,7 @@ mod tests { fn a_single_link_round_trips_with_its_target_classid() { let mut row = row_with_degrees(&[(Predicate::IsA, 1)]); let mut tg = vec![(Predicate::IsA, t(Namespace::Mondo, 5015))]; - let st = pack_edges(&mut row, &mut tg, 0x0000); + let st = pack_edges(&mut row, &mut tg, 0x0000, &crate::registry::OBO_CORE); assert_eq!(st.written, 1); assert_eq!(st.dropped, 0); @@ -369,7 +444,7 @@ mod tests { (Predicate::Other, t(Namespace::Mondo, 1)), (Predicate::Other, t(Namespace::Hpo, 2)), ]; - let st = pack_edges(&mut row, &mut tg, 0x0000); + let st = pack_edges(&mut row, &mut tg, 0x0000, &crate::registry::OBO_CORE); assert_eq!(st.written, 2); assert_eq!(st.lanes, 2, "a namespace change must start a new lane"); @@ -395,7 +470,7 @@ mod tests { let mut tg: Vec<_> = (1..=5) .map(|n| (Predicate::IsA, t(Namespace::Uberon, n))) .collect(); - let st = pack_edges(&mut row, &mut tg, 0x0000); + let st = pack_edges(&mut row, &mut tg, 0x0000, &crate::registry::OBO_CORE); assert_eq!(st.written, 5); assert_eq!(st.lanes, 2); let got: Vec<_> = EdgeLanes::new(&row) @@ -415,7 +490,7 @@ mod tests { (Predicate::IsA, t(Namespace::Uberon, 100)), (Predicate::IsA, t(Namespace::Uberon, 200)), ]; - pack_edges(&mut row, &mut tg, 0x0000); + pack_edges(&mut row, &mut tg, 0x0000, &crate::registry::OBO_CORE); let got: Vec<_> = EdgeLanes::new(&row) .links() .map(|(p, l)| (p, l.num)) @@ -438,7 +513,7 @@ mod tests { let mut tg: Vec<_> = (1..=over as u32) .map(|n| (Predicate::IsA, t(Namespace::Hpo, n))) .collect(); - let st = pack_edges(&mut row, &mut tg, 0x0000); + let st = pack_edges(&mut row, &mut tg, 0x0000, &crate::registry::OBO_CORE); assert_eq!(st.rows_overflowed, 1); assert_eq!(st.dropped, over - EDGE_SLOT_COUNT); assert_eq!(st.written, EDGE_SLOT_COUNT); @@ -452,7 +527,7 @@ mod tests { let mut tg: Vec<_> = (1..=3) .map(|n| (Predicate::IsA, t(Namespace::Pato, n))) .collect(); - pack_edges(&mut row, &mut tg, 0x0000); + pack_edges(&mut row, &mut tg, 0x0000, &crate::registry::OBO_CORE); assert_eq!(EdgeLanes::new(&row).links().count(), 3); row[EDGES_OFFSET + Predicate::IsA as usize] = 1; @@ -474,7 +549,7 @@ mod tests { (Predicate::IsA, t(Namespace::Uberon, 10)), (Predicate::PartOf, t(Namespace::Uberon, 20)), ]; - pack_edges(&mut row, &mut tg, 0x0000); + pack_edges(&mut row, &mut tg, 0x0000, &crate::registry::OBO_CORE); // both edges are readable... assert_eq!(EdgeLanes::new(&row).links().count(), 2); @@ -497,6 +572,122 @@ mod tests { ); } + /// Gate A1, stated as the bug it fixes. + /// + /// A link to numeric `0` — a real term in three measured namespaces, one + /// of them a namespace root — must round-trip AND must not end the walk. + /// Under the pre-bias encoding this test fails on both counts: the slot + /// bytes are all-zero, so the reader reads a pad, drops the link, and + /// **silently truncates the rest of the lane** — the second target is lost + /// too. That is why the assertion below checks the *following* link, not + /// just the zero one. + #[test] + fn a_link_to_numeric_zero_survives_and_does_not_end_the_walk() { + let mut row = row_with_degrees(&[(Predicate::IsA, 2)]); + let mut tg = vec![ + (Predicate::IsA, t(Namespace::Hpo, 0)), + (Predicate::IsA, t(Namespace::Hpo, 118)), + ]; + let st = pack_edges(&mut row, &mut tg, 0x0000, &crate::registry::OBO_CORE); + assert_eq!(st.written, 2); + assert_eq!(st.dropped, 0); + + let got: Vec<_> = EdgeLanes::new(&row) + .links_of(Predicate::IsA) + .map(|l| l.num) + .collect(); + assert_eq!( + got, + vec![0, 118], + "numeric 0 is a real target, and the link after it must survive too" + ); + + // The stored bytes must NOT be zero — that is the whole mechanism. + let base = VALUE_OFFSET + EDGE_LANE_SLAB_OFFSET; + assert_ne!( + u24_le(row[base + 4], row[base + 5], row[base + 6]), + 0, + "a real target must never encode to the pad value" + ); + } + + /// The pad still works — the can-it-stay-silent half. An unused slot must + /// still terminate the lane, or the bias would have traded one silent + /// failure for another. + #[test] + fn an_unused_slot_still_reads_as_pad_and_advances_the_lane() { + // Two target ontologies force two lanes; the first lane keeps three + // empty slots after its single link. + let mut row = row_with_degrees(&[(Predicate::IsA, 2)]); + let mut tg = vec![ + (Predicate::IsA, t(Namespace::Mondo, 5015)), + (Predicate::IsA, t(Namespace::Uberon, 955)), + ]; + pack_edges(&mut row, &mut tg, 0x0000, &crate::registry::OBO_CORE); + let got: Vec<_> = EdgeLanes::new(&row) + .links() + .map(|(_, l)| (l.classid, l.num)) + .collect(); + assert_eq!( + got, + vec![(0x0301_0000, 5015), (0x0303_0000, 955)], + "the three empty slots after the first link must advance the lane, \ + not yield phantom links" + ); + } + + #[test] + fn the_bias_round_trips_across_the_whole_representable_range() { + for n in [ + 0u32, + 1, + 2, + 118, + 65_535, + 65_536, + 700_092, + 9_999_994, + MAX_LINK_NUMERIC, + ] { + assert_eq!(decode_numeric(encode_numeric(n)), n, "{n} must round-trip"); + assert_ne!(encode_numeric(n), 0, "{n} must not collide with the pad"); + assert!( + encode_numeric(n) <= 0x00FF_FFFF, + "{n} must still fit the u24 slot after biasing" + ); + } + } + + /// The one numeric the biased u24 cannot hold is reported, not truncated. + /// + /// `TermId::parse` already rejects `> 0x00FF_FFFF`, so this is reachable + /// only by constructing a `TermId` directly — but a guard that cannot be + /// shown to fire is the defect one level up. + #[test] + fn an_unrepresentable_numeric_is_dropped_rather_than_wrapped() { + let mut row = row_with_degrees(&[(Predicate::IsA, 2)]); + let mut tg = vec![ + ( + Predicate::IsA, + TermId { + ns: Namespace::Hpo as u8, + num: 0x00FF_FFFF, + }, + ), + (Predicate::IsA, t(Namespace::Hpo, 7)), + ]; + let st = pack_edges(&mut row, &mut tg, 0x0000, &crate::registry::OBO_CORE); + assert_eq!(st.dropped, 1, "the unrepresentable link must be reported"); + assert_eq!(st.rows_overflowed, 1, "and must block the ship"); + assert_eq!(st.written, 1, "the representable link still lands"); + // It must not have wrapped to the pad, nor to some other term. + let got: Vec<_> = EdgeLanes::new(&row) + .links_of(Predicate::IsA) + .map(|l| l.num) + .collect(); + assert_eq!(got, vec![7], "no phantom target from a wrapped numeric"); + } + #[test] fn a_row_key_round_trips_through_the_family_identity_rail() { // an oversize CURIE (> u16) must survive: the numeric rides the rail, @@ -521,7 +712,7 @@ mod tests { let mut tg: Vec<_> = (1..=EDGE_SLOT_COUNT as u32) .map(|n| (Predicate::IsA, t(Namespace::Ro, n))) .collect(); - let st = pack_edges(&mut row, &mut tg, 0x0000); + let st = pack_edges(&mut row, &mut tg, 0x0000, &crate::registry::OBO_CORE); assert_eq!(st.dropped, 0); assert_eq!( u16::from_le_bytes([row[et], row[et + 1]]), diff --git a/crates/ogar-obo/src/lib.rs b/crates/ogar-obo/src/lib.rs index f9c354a4..876b986d 100644 --- a/crates/ogar-obo/src/lib.rs +++ b/crates/ogar-obo/src/lib.rs @@ -348,6 +348,7 @@ fn pack_row( id: &TermId, node: &OboNode, app_prefix: u16, + reg: ®istry::NsRegistry, ) -> (Row512, edges::PackStats) { let mut row = Row512::zeroed(); row.0[0..16].copy_from_slice(&pack_key(classid, id.num)); @@ -369,7 +370,7 @@ fn pack_row( Vec::with_capacity(node.is_a.len() + node.rel.len()); targets.extend(node.is_a.iter().map(|t| (Predicate::IsA, *t))); targets.extend(node.rel.iter().copied()); - let st = edges::pack_edges(&mut row.0, &mut targets, app_prefix); + let st = edges::pack_edges(&mut row.0, &mut targets, app_prefix, reg); (row, st) } @@ -558,6 +559,23 @@ pub struct Bake { /// is the lo-u16 render skin (`0x0000` = the canonical reference skin). #[must_use] pub fn bake(nodes: &std::collections::HashMap, app_prefix: u16) -> Bake { + bake_with(nodes, app_prefix, ®istry::OBO_CORE) +} + +/// [`bake`] against an explicit namespace table. +/// +/// This is the general form; `bake` is this against +/// [`registry::OBO_CORE`]. A row's classid and every edge lane's classid come +/// from `reg`, so a bake over a different namespace set needs no code change — +/// only a different table. A target whose namespace is absent from `reg` has +/// no classid to name it, so its link is reported in +/// [`BakeStats::links_dropped`] rather than guessed at. +#[must_use] +pub fn bake_with( + nodes: &std::collections::HashMap, + app_prefix: u16, + reg: ®istry::NsRegistry, +) -> Bake { let mut ids: Vec = nodes .iter() .filter(|(_, n)| !n.obsolete) @@ -575,8 +593,10 @@ pub fn bake(nodes: &std::collections::HashMap, app_prefix: u16) for id in &ids { let node = &nodes[id]; - let classid = id.namespace().render_classid(app_prefix); - let (row, pk) = pack_row(classid, id, node, app_prefix); + let Some(classid) = reg.render_classid(id.ns, app_prefix) else { + continue; + }; + let (row, pk) = pack_row(classid, id, node, app_prefix, reg); rows.push(row); stats.links_packed += pk.written; stats.links_dropped += pk.dropped; diff --git a/crates/ogar-obo/src/spine.rs b/crates/ogar-obo/src/spine.rs index d825260c..fa3378a7 100644 --- a/crates/ogar-obo/src/spine.rs +++ b/crates/ogar-obo/src/spine.rs @@ -219,7 +219,7 @@ mod tests { ) }) .collect(); - pack_edges(&mut r.0, &mut tg, 0x0000); + pack_edges(&mut r.0, &mut tg, 0x0000, &crate::registry::OBO_CORE); r } @@ -251,7 +251,7 @@ mod tests { }, ), ]; - pack_edges(&mut r.0, &mut tg, 0x0000); + pack_edges(&mut r.0, &mut tg, 0x0000, &crate::registry::OBO_CORE); let rows = vec![r]; let lens = SpineLens::new(&rows); @@ -284,7 +284,7 @@ mod tests { num: 42, }, )]; - pack_edges(&mut r.0, &mut tg, 0x0000); + pack_edges(&mut r.0, &mut tg, 0x0000, &crate::registry::OBO_CORE); let rows = vec![r]; let lens = SpineLens::new(&rows); From a2b82722bfa32df7c841fbddabece2a6c401f7bc Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 8 Aug 2026 13:09:49 +0000 Subject: [PATCH 09/13] =?UTF-8?q?ogar-obo:=20META=5FSTUDY=5FSPINE=20collid?= =?UTF-8?q?ed=20with=20ogar-ro's=20concept=20id=20=E2=80=94=20move=20it?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Found while auditing something else. I minted META_STUDY_SPINE at 0x0306..=0x030D yesterday. `ogar_ro::RELATION_BODY_CONCEPT_ID` is 0x0306 — live, and already consumed by medcare-cohorts. That is not a near-miss. A baked BFO row and an ogar-ro relation-body row would carry the SAME classid, and nothing downstream could tell them apart. The spine bake I shipped had 35 BFO rows sitting on that address. The fix is not +1. `0x0300..=0x030F` is now the reserved core band — the five OBO-core namespaces, ogar-ro, and room for nine more — and the spine starts at the next clean boundary, 0x0310..=0x0317. Shifting by one would have parked BFO at 0x0307, immediately adjacent to a live allocation, which is how this happens again. Two guards, because one cannot cover it. ogar-obo cannot see ogar-ro (the dependency runs ogar-ro -> ogar-obo), so: - ogar-obo asserts the BAND: no spine row inside 0x0300..=0x030F, no core row outside it, no id claimed twice across both tables, and every id still inside the 0x03 Ontology domain. - ogar-ro asserts the CONSTANT: no ogar-obo namespace claims RELATION_BODY_CONCEPT_ID — plus an anti-vacuity test that the tables it checks are actually populated, since two empty tables would pass while proving nothing. Both verified to fire: restoring 0x0306 fails with BFO at 0x0306 sits in the reserved core band (0x0300..=0x030F) — that is where ogar-ro's 0x0306 lives Re-baked the spine. Classids are now 0x0310_0000..0x0317_0000, digest moves 437ea33a -> e1aa94d23bd3b2ffbf901591bcafb158dd625f4744a291026677349ad15d524f. Everything else is unchanged: 7,641 rows, 12,885 triples, 0 dropped, 0 overflowed, and the A1 proof still holds — 10 numeric-0 links present, 10 of 10 read back from the lanes. The core bake (424e81cc) is untouched: OBO_CORE's ids did not move. Worth naming why this was invisible: the collision lived in two repos' worth of hex literals with no test between them. It was found by hand-enumerating the 0x03 domain during an unrelated audit, which is not a process that scales — hence a guard on each side rather than a note. --- crates/ogar-obo/src/registry.rs | 104 +++++++++++++++++++++++++++++--- crates/ogar-ro/src/lib.rs | 41 +++++++++++++ 2 files changed, 135 insertions(+), 10 deletions(-) diff --git a/crates/ogar-obo/src/registry.rs b/crates/ogar-obo/src/registry.rs index 5d784354..1b208e63 100644 --- a/crates/ogar-obo/src/registry.rs +++ b/crates/ogar-obo/src/registry.rs @@ -160,8 +160,25 @@ pub const OBO_CORE: NsRegistry = NsRegistry::new(&[ ]); /// The upper-level **meta-study spine** — the `is_a` skeleton a study-level -/// bake grounds on. Continues the same `0x03` Ontology domain as [`OBO_CORE`], -/// under the same local authority its concept-id docs describe. +/// bake grounds on. Same `0x03` Ontology domain as [`OBO_CORE`], under the +/// same local authority its concept-id docs describe. +/// +/// ## Why this starts at `0x0310` and not at `0x0306` +/// +/// It was minted at `0x0306..=0x030D` and that **collided with +/// `ogar_ro::RELATION_BODY_CONCEPT_ID`, which is `0x0306`** — an id already +/// live and already consumed downstream. Two concepts at one address means a +/// baked BFO row and an `ogar-ro` relation-body row are the same classid, and +/// nothing downstream could tell them apart. +/// +/// The fix is not `+1`. `0x0300..=0x030F` is treated as the **reserved core +/// band** — the five OBO-core namespaces, `ogar-ro`, and room for nine more — +/// and the spine starts at the next clean boundary, `0x0310`. Shifting by one +/// would have parked BFO at `0x0307`, immediately adjacent to a live +/// allocation, which is how the collision happens again. +/// +/// [`spine_does_not_collide_with_the_core_or_the_reserved_band`](self) is the +/// guard; `ogar-ro` carries the mirror guard pointing the other way. /// /// ## Order is a convention, not a topological sort /// @@ -198,35 +215,35 @@ pub const OBO_CORE: NsRegistry = NsRegistry::new(&[ pub const META_STUDY_SPINE: NsRegistry = NsRegistry::new(&[ NsSpec { prefix: "BFO", - concept_id: 0x0306, + concept_id: 0x0310, }, NsSpec { prefix: "COB", - concept_id: 0x0307, + concept_id: 0x0311, }, NsSpec { prefix: "IAO", - concept_id: 0x0308, + concept_id: 0x0312, }, NsSpec { prefix: "OBI", - concept_id: 0x0309, + concept_id: 0x0313, }, NsSpec { prefix: "OBCS", - concept_id: 0x030A, + concept_id: 0x0314, }, NsSpec { prefix: "SEPIO", - concept_id: 0x030B, + concept_id: 0x0315, }, NsSpec { prefix: "ECO", - concept_id: 0x030C, + concept_id: 0x0316, }, NsSpec { prefix: "FBbi", - concept_id: 0x030D, + concept_id: 0x0317, }, ]); @@ -347,6 +364,73 @@ mod tests { assert_eq!(TermId::parse("MONDO:0007739").map(|t| t.ns), Some(0)); } + /// The guard the 2026-08-08 audit had to be run by hand to find. + /// + /// `META_STUDY_SPINE` was minted at `0x0306..=0x030D`, and `0x0306` is + /// `ogar_ro::RELATION_BODY_CONCEPT_ID` — live, and already consumed + /// downstream. Two concepts at one address is not a near-miss: a baked BFO + /// row and an `ogar-ro` relation-body row become the same classid. + /// + /// This crate cannot see `ogar-ro` (that crate deps *this* one, so the + /// dependency only runs one way), so the band is asserted instead of the + /// constant: everything below `SPINE_BAND_START` belongs to the core + /// allocations, and the spine must stay above it. `ogar-ro` carries the + /// mirror guard against its own real constant. + /// + /// **What would make it fail:** minting a spine row inside the reserved + /// band, or giving two tables the same id. + #[test] + fn spine_does_not_collide_with_the_core_or_the_reserved_band() { + /// First id outside the reserved core band `0x0300..=0x030F`. + const SPINE_BAND_START: u16 = 0x0310; + + for s in META_STUDY_SPINE.specs() { + assert!( + s.concept_id >= SPINE_BAND_START, + "{} at {:#06X} sits in the reserved core band \ + (0x0300..=0x030F) — that is where ogar-ro's 0x0306 lives", + s.prefix, + s.concept_id + ); + } + for c in OBO_CORE.specs() { + assert!( + c.concept_id < SPINE_BAND_START, + "{} at {:#06X} escaped the reserved core band", + c.prefix, + c.concept_id + ); + } + + // No id twice, across BOTH tables — the general form of the bug. + let mut ids: Vec<(u16, &str)> = OBO_CORE + .specs() + .iter() + .chain(META_STUDY_SPINE.specs()) + .map(|s| (s.concept_id, s.prefix)) + .collect(); + ids.sort_unstable(); + for w in ids.windows(2) { + assert_ne!( + w[0].0, w[1].0, + "{} and {} both claim {:#06X}", + w[0].1, w[1].1, w[0].0 + ); + } + + // Every id stays inside the 0x03 Ontology domain — a spine that ran + // off the end of the block would silently land in another domain. + for s in OBO_CORE.specs().iter().chain(META_STUDY_SPINE.specs()) { + assert_eq!( + s.concept_id >> 8, + 0x03, + "{} at {:#06X} left the 0x03 Ontology domain", + s.prefix, + s.concept_id + ); + } + } + #[test] fn unknown_ordinal_resolves_to_none_rather_than_panicking() { assert_eq!(OBO_CORE.spec(9), None); diff --git a/crates/ogar-ro/src/lib.rs b/crates/ogar-ro/src/lib.rs index 315a83b8..412d193f 100644 --- a/crates/ogar-ro/src/lib.rs +++ b/crates/ogar-ro/src/lib.rs @@ -314,3 +314,44 @@ mod tests { assert_eq!(id & 0xFFFF, 0x1000); } } + +#[cfg(test)] +mod concept_id_collision_guard { + use super::RELATION_BODY_CONCEPT_ID; + use ogar_obo::registry::{META_STUDY_SPINE, OBO_CORE}; + + /// The mirror of `ogar_obo::registry`'s band guard, pointing the other way. + /// + /// `ogar-obo` cannot see this crate (the dependency runs `ogar-ro` → + /// `ogar-obo`), so it can only assert that its tables stay out of the + /// reserved band. This side asserts the thing that actually matters: our + /// real constant is claimed by no `ogar-obo` table. + /// + /// It exists because the reverse DID happen — `META_STUDY_SPINE` was + /// minted over `0x0306` on 2026-08-08 and nothing failed. The collision + /// was found by hand-enumerating the `0x03` domain during an unrelated + /// audit, which is not a process that scales. + /// + /// **What would make it fail:** any `ogar-obo` table growing a row at + /// `RELATION_BODY_CONCEPT_ID`. + #[test] + fn no_ogar_obo_namespace_claims_our_concept_id() { + for s in OBO_CORE.specs().iter().chain(META_STUDY_SPINE.specs()) { + assert_ne!( + s.concept_id, RELATION_BODY_CONCEPT_ID, + "ogar-obo namespace {} claims {:#06X}, which is \ + RELATION_BODY_CONCEPT_ID — a baked {} row and an ogar-ro \ + relation-body row would be the same classid", + s.prefix, RELATION_BODY_CONCEPT_ID, s.prefix + ); + } + } + + /// Anti-vacuity: the guard above compares against tables that are actually + /// populated. A pair of empty tables would pass it while proving nothing. + #[test] + fn the_tables_the_guard_checks_are_non_empty() { + assert!(OBO_CORE.len() >= 5, "core table must be populated"); + assert!(META_STUDY_SPINE.len() >= 8, "spine table must be populated"); + } +} From fed00e1ec58b93d58a462d3f17dcd3e876a1aabf Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 8 Aug 2026 13:50:49 +0000 Subject: [PATCH 10/13] =?UTF-8?q?ogar-obo:=20examples/split=5Fnamespaces?= =?UTF-8?q?=20=E2=80=94=20the=20per-namespace=20slabs,=20from=20the=20unio?= =?UTF-8?q?n?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit S3 holds the bake both ways: one union `obo-core.soa` (what bakes.tsv pins) and five per-namespace slabs. The splitter that made the second was not in the repo, so re-syncing S3 after the A1 re-bake meant reconstructing it — which is the same class of problem as a pin table living inside a shell script. Splits `bake_with`'s output by namespace ordinal, asserting the ship gate (links_dropped == 0, rows_overflowed == 0) before writing anything. Row counts and byte sizes reproduce the slabs already on S3 exactly, so the only difference against the previous upload is the corrected bytes. Each slab verifies independently downstream: cross-slab references (HP->UBERON in a HP-only slab) are counted separately from dangling ones, so a legitimate partition does not read as a broken artifact. --- crates/ogar-obo/examples/split_namespaces.rs | 45 ++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 crates/ogar-obo/examples/split_namespaces.rs diff --git a/crates/ogar-obo/examples/split_namespaces.rs b/crates/ogar-obo/examples/split_namespaces.rs new file mode 100644 index 00000000..9e36e74c --- /dev/null +++ b/crates/ogar-obo/examples/split_namespaces.rs @@ -0,0 +1,45 @@ +//! Split a baked union into per-namespace slabs, exactly as the S3 layout holds them. +use ogar_obo::{ + Row512, as_le_bytes, bake_with, merge_logical_defs, parse_hp_logical_defs, parse_obo, + registry::OBO_CORE, +}; +use std::collections::HashMap; +fn main() { + let dir = std::env::args().nth(1).unwrap_or("/tmp/obo".into()); + let out = std::env::args().nth(2).unwrap_or("/tmp/slabs".into()); + std::fs::create_dir_all(&out).unwrap(); + let mut nodes = HashMap::new(); + for f in ["mondo", "hp", "uberon", "pato", "ro"] { + for (id, n) in parse_obo(&std::fs::read_to_string(format!("{dir}/{f}.obo")).unwrap()) { + let e: &mut ogar_obo::OboNode = nodes.entry(id).or_default(); + e.is_a.extend(n.is_a); + e.rel.extend(n.rel); + e.xref.extend(n.xref); + e.obsolete |= n.obsolete; + } + } + let owl = std::fs::read_to_string(format!("{dir}/hp-base.owl")).unwrap_or_default(); + merge_logical_defs(&mut nodes, &parse_hp_logical_defs(&owl)); + let baked = bake_with(&nodes, 0x0000, &OBO_CORE); + assert_eq!(baked.stats.links_dropped, 0); + assert_eq!(baked.stats.rows_overflowed, 0); + let mut per: HashMap<&str, Vec> = HashMap::new(); + for (i, id) in baked.ids.iter().enumerate() { + let name = match id.ns { + 0 => "mondo", + 1 => "hpo", + 2 => "uberon", + 3 => "pato", + _ => "ro", + }; + per.entry(name).or_default().push(baked.rows[i]); + } + let mut names: Vec<_> = per.keys().copied().collect(); + names.sort_unstable(); + for n in names { + let rows = &per[n]; + let p = format!("{out}/obo.{n}.soa"); + std::fs::write(&p, as_le_bytes(rows)).unwrap(); + println!("{p} {} rows {} bytes", rows.len(), rows.len() * 512); + } +} From 4d395222dcc887d718ea27c0c2bc155a281ae8fe Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 8 Aug 2026 14:21:21 +0000 Subject: [PATCH 11/13] =?UTF-8?q?ogar-obo:=20move=20the=20spine=20to=200x0?= =?UTF-8?q?340=20=E2=80=94=20the=20second=20collision,=20and=20why=20bandi?= =?UTF-8?q?ng=20failed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit META_STUDY_SPINE was moved to 0x0310..=0x0317 this morning after colliding with ogar_ro's 0x0306. That block collided with FOUR consumer allocations: 0x0311 COB vs OPS 0x0313 OBI vs a product lane 0x0315 SEPIO vs the UMLS CUI convergence lane 0x0317 FBbi vs Orphanet The second attempt failed for a reason worth naming rather than patching: the 0x03 domain's occupancy was enumerated IN THIS REPO ONLY. A private consumer holds an odd-stride run (0x0307, 09, 0B, 0D, 0F, 11, 13, 15, 17, 19, 1B, 1D) that no test here can see, and it grows into odd slots — so a nearby block collides eventually rather than immediately, which is the worse failure because it lands long after the decision. Moved to 0x0340..=0x0347, clear of both the core band and that run's growth. But the spacing is a MITIGATION, not the guard, and the band test here is weaker than it reads: it proves this table is self-consistent and clear of upstream neighbours, which is exactly what it did while colliding with four ids it had no way to enumerate. The doc now says that outright so nobody reads it as coverage it does not have. The real guard cannot live here. It has to see both sides, so it lives in the consumer (medcare-cohorts::classid_census), which enumerates its own allocations and checks them against ogar-obo's tables. Run against the 0x0310 block it reports all four collisions by name. Spine digest moves e1aa94d2 -> a2205fb7255eb2801d0056944b364172d2be529ffb0ac6874ad5510bb13c3f06. Rows, triples and the A1 proof are unchanged: 7,641 rows, 12,885 triples, 0 dropped, 10 of 10 numeric-0 links read back. The core bake is untouched. Also fixed a test fixture the move caught: `a_different_table_...` built its EXTENDED table from the spine's literal, so the table moved while its assertion literal did not. --- crates/ogar-obo/src/registry.rs | 62 +++++++++++++++++++-------------- 1 file changed, 35 insertions(+), 27 deletions(-) diff --git a/crates/ogar-obo/src/registry.rs b/crates/ogar-obo/src/registry.rs index 1b208e63..7bbd12ba 100644 --- a/crates/ogar-obo/src/registry.rs +++ b/crates/ogar-obo/src/registry.rs @@ -163,22 +163,28 @@ pub const OBO_CORE: NsRegistry = NsRegistry::new(&[ /// bake grounds on. Same `0x03` Ontology domain as [`OBO_CORE`], under the /// same local authority its concept-id docs describe. /// -/// ## Why this starts at `0x0310` and not at `0x0306` +/// ## Why this starts at `0x0340` — and why it took three attempts /// -/// It was minted at `0x0306..=0x030D` and that **collided with -/// `ogar_ro::RELATION_BODY_CONCEPT_ID`, which is `0x0306`** — an id already -/// live and already consumed downstream. Two concepts at one address means a -/// baked BFO row and an `ogar-ro` relation-body row are the same classid, and -/// nothing downstream could tell them apart. +/// Minted at `0x0306..=0x030D`, which collided with +/// `ogar_ro::RELATION_BODY_CONCEPT_ID` (`0x0306`). Moved to +/// `0x0310..=0x0317`, which collided with **four** consumer allocations — +/// OPS `0x0311`, a product lane `0x0313`, the UMLS CUI convergence lane +/// `0x0315`, Orphanet `0x0317`. /// -/// The fix is not `+1`. `0x0300..=0x030F` is treated as the **reserved core -/// band** — the five OBO-core namespaces, `ogar-ro`, and room for nine more — -/// and the spine starts at the next clean boundary, `0x0310`. Shifting by one -/// would have parked BFO at `0x0307`, immediately adjacent to a live -/// allocation, which is how the collision happens again. +/// The second attempt failed for a reason worth naming: the `0x03` domain's +/// occupancy was enumerated **in this repo only**. A private consumer holds an +/// odd-stride run — `0x0307, 09, 0B, 0D, 0F, 11, 13, 15, 17, 19, 1B, 1D` — +/// that no test here can see, and it grows into odd slots, so any nearby block +/// collides eventually rather than immediately. Picking a "clean" block by eye +/// is the failure mode; there is no vantage point in this crate from which the +/// block is verifiably clean. /// -/// [`spine_does_not_collide_with_the_core_or_the_reserved_band`](self) is the -/// guard; `ogar-ro` carries the mirror guard pointing the other way. +/// `0x0340..=0x0347` sits clear of both the core band and that run's growth, +/// but **the spacing is a mitigation, not the guard.** The guard has to live +/// where both sides are visible — i.e. in the consumer. The band test below +/// only proves this table is self-consistent and clear of the core; it +/// cannot and does not prove the consumer is clear, and reading it as if it +/// did is what produced the second collision. /// /// ## Order is a convention, not a topological sort /// @@ -215,35 +221,35 @@ pub const OBO_CORE: NsRegistry = NsRegistry::new(&[ pub const META_STUDY_SPINE: NsRegistry = NsRegistry::new(&[ NsSpec { prefix: "BFO", - concept_id: 0x0310, + concept_id: 0x0340, }, NsSpec { prefix: "COB", - concept_id: 0x0311, + concept_id: 0x0341, }, NsSpec { prefix: "IAO", - concept_id: 0x0312, + concept_id: 0x0342, }, NsSpec { prefix: "OBI", - concept_id: 0x0313, + concept_id: 0x0343, }, NsSpec { prefix: "OBCS", - concept_id: 0x0314, + concept_id: 0x0344, }, NsSpec { prefix: "SEPIO", - concept_id: 0x0315, + concept_id: 0x0345, }, NsSpec { prefix: "ECO", - concept_id: 0x0316, + concept_id: 0x0346, }, NsSpec { prefix: "FBbi", - concept_id: 0x0317, + concept_id: 0x0347, }, ]); @@ -342,7 +348,7 @@ mod tests { const EXTENDED: NsRegistry = NsRegistry::new(&[ NsSpec { prefix: "BFO", - concept_id: 0x0310, + concept_id: 0x0340, }, NsSpec { prefix: "MONDO", @@ -352,7 +358,7 @@ mod tests { let bfo = EXTENDED.parse("BFO:0000002").expect("BFO parses here"); assert_eq!(bfo.ns, 0, "BFO is row 0 of this table"); assert_eq!(bfo.num, 2); - assert_eq!(EXTENDED.render_classid(0, 0x0000), Some(0x0310_0000)); + assert_eq!(EXTENDED.render_classid(0, 0x0000), Some(0x0340_0000)); assert!( TermId::parse("BFO:0000002").is_none(), "the shipped core must NOT resolve BFO — otherwise this proves nothing" @@ -381,14 +387,16 @@ mod tests { /// band, or giving two tables the same id. #[test] fn spine_does_not_collide_with_the_core_or_the_reserved_band() { - /// First id outside the reserved core band `0x0300..=0x030F`. - const SPINE_BAND_START: u16 = 0x0310; + /// First id of the spine block. Everything below belongs to the core + /// band or to consumer allocations this crate cannot see. + const SPINE_BAND_START: u16 = 0x0340; for s in META_STUDY_SPINE.specs() { assert!( s.concept_id >= SPINE_BAND_START, - "{} at {:#06X} sits in the reserved core band \ - (0x0300..=0x030F) — that is where ogar-ro's 0x0306 lives", + "{} at {:#06X} sits below the spine block — that range holds the \ + core namespaces, ogar-ro (0x0306), and consumer allocations \ + this crate cannot see", s.prefix, s.concept_id ); From 07b029b35b386522a1d468e614ee51d890188c18 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 8 Aug 2026 14:28:09 +0000 Subject: [PATCH 12/13] =?UTF-8?q?ogar-obo:=20`dangling`=20missed=20edges?= =?UTF-8?q?=20into=20obsolete=20terms=20=E2=80=94=2055=20on=20the=20spine?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The bake reported `dangling: 0` for an artifact a reader finds 55 dangling edges in. Both were right about what they measured, and the stat was measuring the wrong thing. `dangling` counted edges whose object is absent from the node MAP. An obsolete term is IN the map — `parse_obo` / `parse_rdf` keep it, and `bake` deliberately does not tile it — so an edge into one passed the check while pointing at a row that does not exist. Found by running the edge-resolution hook against the spine I had just published. An edge into an obsolete term is itself obsolete, so it is now dropped rather than emitted, and counted honestly: `dangling` means "object gets no ROW", whether absent from the sources or present-but-obsolete. Digest-neutral for the `.obo` core — no edge there targets an untiled row, so `bake_obo` reproduces 424e81cc exactly. It is the RDF path, where `owl:deprecated` classes are still referenced by live ones, that this corrects: the spine drops 12,885 -> 12,830 triples, exactly the 55, and its digest moves a2205fb7 -> 4c41538151b344e31db70d674697f01580e1c2b755bc12c796f6ec755d8f138f. The two per-edge `dangling += 1` counters are now unreachable (the filter runs first), so they became debug_asserts rather than dead arithmetic that would quietly disagree with the filter if either changed. --- crates/ogar-obo/src/lib.rs | 47 +++++++++++++++++++++++++++----------- 1 file changed, 34 insertions(+), 13 deletions(-) diff --git a/crates/ogar-obo/src/lib.rs b/crates/ogar-obo/src/lib.rs index 876b986d..de3be3ff 100644 --- a/crates/ogar-obo/src/lib.rs +++ b/crates/ogar-obo/src/lib.rs @@ -514,7 +514,10 @@ pub struct BakeStats { pub triples: usize, /// `is_a` cycles found (SCC size > 1) — must be 0 on a clean OBO core pub is_a_cycles: usize, - /// edges whose object namespace is core but the object node is absent + /// edges dropped because their object gets no ROW — absent from the + /// sources, or present but obsolete and therefore not tiled. Counting + /// only the first was the bug: an edge into an obsolete term reported + /// clean here and dangling to any reader of the artifact. pub dangling: usize, /// MONDO→HP has_phenotype edges resolving to a loaded HP node pub mondo_hp: usize, @@ -591,8 +594,29 @@ pub fn bake_with( ..Default::default() }; + // The set of ids that will actually GET a row. `dangling` used to mean + // "object absent from the node map", but an obsolete term IS in the map + // and deliberately gets no row — so an edge into one pointed at a row that + // does not exist, and the stat said 0. Measured on the RDF spine: 55 such + // edges, reported as dangling by a reader and as clean by the bake. + // + // An edge into an obsolete term is itself obsolete, so it is dropped + // rather than emitted, and counted. Digest-neutral for the `.obo` core + // (no edge there targets an untiled row); it is the RDF path, where + // `owl:deprecated` classes are still referenced, that this corrects. + let tiled: std::collections::HashSet = ids.iter().copied().collect(); + for id in &ids { let node = &nodes[id]; + let node = { + let mut n = node.clone(); + let before = n.is_a.len() + n.rel.len(); + n.is_a.retain(|t| tiled.contains(t)); + n.rel.retain(|(_, t)| tiled.contains(t)); + stats.dangling += before - (n.is_a.len() + n.rel.len()); + n + }; + let node = &node; let Some(classid) = reg.render_classid(id.ns, app_prefix) else { continue; }; @@ -615,9 +639,10 @@ pub fn bake_with( p: Predicate::IsA, o: *parent, }); - if !nodes.contains_key(parent) { - stats.dangling += 1; - } + debug_assert!( + nodes.contains_key(parent), + "untiled targets already filtered" + ); } for (p, o) in &node.rel { triples.push(Triple { @@ -625,15 +650,11 @@ pub fn bake_with( p: *p, o: *o, }); - if !nodes.contains_key(o) { - stats.dangling += 1; - } else { - match p { - Predicate::HasPhenotype => stats.mondo_hp += 1, - Predicate::HasAnatomy => stats.hp_uberon += 1, - Predicate::HasQuality => stats.hp_pato += 1, - _ => {} - } + match p { + Predicate::HasPhenotype => stats.mondo_hp += 1, + Predicate::HasAnatomy => stats.hp_uberon += 1, + Predicate::HasQuality => stats.hp_pato += 1, + _ => {} } } } From 0d6dd472694c965f3ab5b434bc8a4843be5a6c4a Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 8 Aug 2026 14:36:14 +0000 Subject: [PATCH 13/13] =?UTF-8?q?ogar-obo:=20dedup=20edges=20at=20bake=20t?= =?UTF-8?q?ime=20=E2=80=94=2018%=20of=20the=20shipped=20core=20was=20dupli?= =?UTF-8?q?cates?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Found by the edge-agreement check: the artifact's degree histogram totals 107,813 is_a slots while carrying 92,685 DISTINCT (subject, parent) pairs. The gap is 15,128 duplicate is_a slots across 10,589 rows. Cause: a driver that unions several sources concatenates parent lists. `bake_obo` merges five .obo files with `extend`, so a term appearing in more than one gets its parents twice. `rdf::parse_rdf` already dedups its own output; doing it in `bake_with` covers every driver instead of every driver having to remember. Across all predicates the effect is larger than the is_a-only measurement suggested: triples drop 186,029 -> 152,073, so 33,956 edges (18.2%) were duplicates. `rel` carries them too, which the is_a-only probe could not see. Semantically this changed nothing — `A is_a B` asserted twice is `A is_a B`, so no answer was ever wrong. What it cost: lane budget, and an over-reporting degree histogram, which is exactly what `resident_out_degree` reads. A row claiming 62 out-edges could have had 56. `BakeStats.duplicates` reports what was collapsed rather than silently shrinking the edge count. NOT re-published. This moves the core digest 424e81cc -> 75ec52a8 and changes numbers that appear in the roadmap, the pipeline spec and the boot receipt (186,029 triples is quoted in several places). That is a deliberate call, not a side effect of a lint fix, and today has already spent three re-bakes. The code is correct from here; the shipped artifact carries the duplicates and is not wrong because of them. --- crates/ogar-obo/src/lib.rs | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/crates/ogar-obo/src/lib.rs b/crates/ogar-obo/src/lib.rs index de3be3ff..bd6663f8 100644 --- a/crates/ogar-obo/src/lib.rs +++ b/crates/ogar-obo/src/lib.rs @@ -537,6 +537,11 @@ pub struct BakeStats { pub links_dropped: usize, /// rows that overflowed the lane budget (must be 0 to ship) pub rows_overflowed: usize, + /// duplicate edges collapsed at bake time — the same `(predicate, object)` + /// asserted more than once for one subject, which a multi-source union + /// produces routinely. Reported rather than silent: it is the difference + /// between the degree histogram and the distinct edge count. + pub duplicates: usize, } /// The bake result: the 512-byte SoA rows (loader-facing) + the SPO triple @@ -614,6 +619,25 @@ pub fn bake_with( n.is_a.retain(|t| tiled.contains(t)); n.rel.retain(|(_, t)| tiled.contains(t)); stats.dangling += before - (n.is_a.len() + n.rel.len()); + // Deduplicate. A driver that unions several sources — `bake_obo` + // merges five `.obo` files with `extend` — concatenates the parent + // lists of a term that appears in more than one, so the same + // `is_a` lands twice. Measured on the shipped core: 15,128 + // duplicate slots across 10,589 rows, 14 % of the is_a links + // written. + // + // Idempotent semantically (`A is_a B` twice is `A is_a B`), so it + // produces no wrong answer — but it spends lane budget and makes + // the degree histogram over-report a row's out-degree, which is + // what `resident_out_degree` reads. `rdf::parse_rdf` already + // dedups its own output; doing it here covers every driver instead + // of every driver having to remember. + let pre = n.is_a.len() + n.rel.len(); + n.is_a.sort_unstable(); + n.is_a.dedup(); + n.rel.sort_unstable(); + n.rel.dedup(); + stats.duplicates += pre - (n.is_a.len() + n.rel.len()); n }; let node = &node;