From 76c63324c2ddb4175c0ad07e2fc3009057003b0b Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 10 Aug 2026 14:04:51 +0000 Subject: [PATCH] =?UTF-8?q?obo:=20the=20RF2=20snapshot=20reader=20?= =?UTF-8?q?=E2=80=94=20one=20parser,=20three=20refusals?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The reader every SNOMED-package consumer would otherwise duplicate wrongly, deliberately column-blind past the shared five (the file family is in the FILENAME; the consumer knows which file it opened). What it owns is the part that goes wrong silently: - the active filter, with the excluded count REPORTED so a bake census can say what it left out instead of looking short; - snapshot uniqueness — a duplicate id means Full export or corrupt concatenation, and the whole file is refused rather than keeping whichever row the file order favored; - column integrity — a ragged row refuses with its line number, because a permissive reader mis-aligns every cell after a stray tab and each still LOOKS like a value. SCTIDs read as u64 (a real 54-bit id is in the fixtures); non-numeric or absent cells are refused, never truncated. No meaning is interpreted: no concept model, no map semantics — types exist only before the bake, and this is the join stage's input surface. Generated by [Claude Code](https://claude.ai/code) --- crates/ogar-obo/src/lib.rs | 4 + crates/ogar-obo/src/rf2.rs | 179 +++++++++++++++++++++++++++++++++++++ 2 files changed, 183 insertions(+) create mode 100644 crates/ogar-obo/src/rf2.rs diff --git a/crates/ogar-obo/src/lib.rs b/crates/ogar-obo/src/lib.rs index e9391ddb..a5f54450 100644 --- a/crates/ogar-obo/src/lib.rs +++ b/crates/ogar-obo/src/lib.rs @@ -42,6 +42,10 @@ pub mod crosswalk; pub mod reason; +/// RF2 snapshot reader — the one parser for SNOMED release-format files +/// (active filter, snapshot uniqueness, column integrity). Consumers never +/// hand-roll this. +pub mod rf2; /// Row stride of the canonical SoA node — `key(16) + edges(16) + value(480)`. /// Mirrors `lance_graph_contract::canonical_node::NODE_ROW_STRIDE`. diff --git a/crates/ogar-obo/src/rf2.rs b/crates/ogar-obo/src/rf2.rs new file mode 100644 index 00000000..2c9e9ac4 --- /dev/null +++ b/crates/ogar-obo/src/rf2.rs @@ -0,0 +1,179 @@ +//! **The RF2 snapshot reader** — one parser for every SNOMED release-format +//! file, so no consumer hand-rolls its own. +//! +//! RF2 snapshot files are TSV with a header row; the reader is deliberately +//! COLUMN-BLIND past the five columns every `der2_*` refset shares +//! (`id effectiveTime active moduleId refsetId`), because the refset family +//! is encoded in the FILENAME's pattern (`sRefset` = one string column, +//! `iissscc` = the extended-map shape, …) and a consumer knows which file it +//! opened. The reader's job is the part every consumer would otherwise +//! duplicate wrongly: +//! +//! - **the `active` filter** — an inactive row is history, not content, and +//! a consumer that forgets the filter bakes retired mappings; +//! - **snapshot uniqueness** — a snapshot carries at most one row per `id`; +//! a duplicate means the file is not a snapshot (a Full file, or a corrupt +//! concatenation), and the reader REFUSES it rather than keeping either row; +//! - **column integrity** — every row must have exactly the header's arity +//! (a permissive reader silently mis-aligns every column after a stray tab). +//! +//! Nothing here interprets meaning: no concept model, no map semantics, no +//! vocabulary. Types exist only before the bake; this is the join stage's +//! input surface. + +use std::collections::HashSet; + +/// One parsed snapshot: the header's column names and the ACTIVE rows, each +/// exactly `columns.len()` wide, borrowed from the input text. +#[derive(Debug, PartialEq, Eq)] +pub struct Rf2Snapshot<'a> { + /// Column names, verbatim from the header row. + pub columns: Vec<&'a str>, + /// Active rows only (`active == "1"`), in file order. + pub rows: Vec>, + /// How many rows the file carried that were NOT kept because they are + /// inactive — reported so a bake's census can state what it excluded + /// rather than looking mysteriously short. + pub inactive: usize, +} + +/// Why a file was refused. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Rf2Error { + /// No header row. + Empty, + /// The header lacks one of the five columns every refset shares. + MissingColumn(&'static str), + /// A row's arity differs from the header's (line number, got, want). + Ragged(usize, usize, usize), + /// Two rows share an `id` — not a snapshot. Carries the duplicated id. + DuplicateId(String), +} + +/// Parse one RF2 snapshot file. Borrows from `text`; allocates only the +/// row/column vectors. +pub fn parse_snapshot(text: &str) -> Result, Rf2Error> { + let mut lines = text.lines().filter(|l| !l.is_empty()); + let header = lines.next().ok_or(Rf2Error::Empty)?; + let columns: Vec<&str> = header.split('\t').collect(); + for want in ["id", "effectiveTime", "active", "moduleId", "refsetId"] { + if !columns.contains(&want) { + // sct2_* core files carry the first four but not refsetId; they + // are still readable — only der2_* demand all five. + if want == "refsetId" && columns.contains(&"definitionStatusId") + || want == "refsetId" && columns.contains(&"typeId") + || want == "refsetId" && columns.contains(&"conceptId") + { + continue; + } + return Err(Rf2Error::MissingColumn(match want { + "id" => "id", + "effectiveTime" => "effectiveTime", + "active" => "active", + "moduleId" => "moduleId", + _ => "refsetId", + })); + } + } + let active_ix = columns.iter().position(|c| *c == "active").unwrap(); + let id_ix = columns.iter().position(|c| *c == "id").unwrap(); + + let mut rows = Vec::new(); + let mut inactive = 0usize; + let mut seen: HashSet<&str> = HashSet::new(); + for (n, line) in lines.enumerate() { + let cells: Vec<&str> = line.split('\t').collect(); + if cells.len() != columns.len() { + return Err(Rf2Error::Ragged(n + 2, cells.len(), columns.len())); + } + if !seen.insert(cells[id_ix]) { + return Err(Rf2Error::DuplicateId(cells[id_ix].to_string())); + } + if cells[active_ix] == "1" { + rows.push(cells); + } else { + inactive += 1; + } + } + Ok(Rf2Snapshot { + columns, + rows, + inactive, + }) +} + +impl<'a> Rf2Snapshot<'a> { + /// Index of a named column, or `None` — the caller decides whether a + /// missing map column is an error for ITS file family. + #[must_use] + pub fn col(&self, name: &str) -> Option { + self.columns.iter().position(|c| *c == name) + } + + /// A row's cell in a named column, parsed as an SCTID (`u64` — SNOMED ids + /// run to 60 bits and never fit smaller). `None` for an absent column or + /// a non-numeric cell — refused, never truncated. + #[must_use] + pub fn sctid(&self, row: usize, name: &str) -> Option { + self.rows.get(row)?.get(self.col(name)?)?.parse().ok() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const OK: &str = "id\teffectiveTime\tactive\tmoduleId\trefsetId\treferencedComponentId\tmapTarget\n\ + a1\t20250101\t1\t900\t447562003\t73211009\tE11\n\ + a2\t20250101\t0\t900\t447562003\t44054006\tE10\n\ + a3\t20250101\t1\t900\t447562003\t16898231000119107\tD66\n"; + + /// **The active filter fires and is COUNTED.** An inactive row is + /// excluded from `rows` and reported in `inactive`, so a bake census can + /// say what it left out instead of looking short. + #[test] + fn inactive_rows_are_excluded_and_counted() { + let s = parse_snapshot(OK).expect("parses"); + assert_eq!(s.rows.len(), 2); + assert_eq!(s.inactive, 1); + assert!(s.rows.iter().all(|r| r[2] == "1")); + } + + /// **SCTIDs read as u64, including past 2^53.** The 54-bit id in the + /// fixture is a real measured shape (`orphanet.rs` carries the same one); + /// a reader that parsed into f64 or u32 corrupts it silently. + #[test] + fn sctids_are_u64_and_refuse_rather_than_truncate() { + let s = parse_snapshot(OK).expect("parses"); + assert_eq!(s.sctid(1, "referencedComponentId"), Some(16898231000119107)); + assert_eq!(s.sctid(0, "mapTarget"), None, "E11 is not numeric — refused"); + assert_eq!(s.sctid(0, "nope"), None, "absent column — refused"); + } + + /// **A duplicate id is REFUSED, not resolved.** A snapshot carries one + /// row per id; two rows means the file is a Full export or a corrupt + /// concatenation, and keeping either row silently would bake whichever + /// the file order favored. + #[test] + fn a_duplicate_id_refuses_the_whole_file() { + let dup = OK.replace("a3\t", "a1\t"); + assert_eq!( + parse_snapshot(&dup), + Err(Rf2Error::DuplicateId("a1".into())) + ); + } + + /// **A ragged row refuses the file with its line number.** A permissive + /// reader mis-aligns every column after a stray tab — the worst kind of + /// silent, because every cell still LOOKS like a value. + #[test] + fn a_ragged_row_refuses_with_the_line_number() { + let ragged = OK.replace("\tD66", ""); + assert_eq!(parse_snapshot(&ragged), Err(Rf2Error::Ragged(4, 6, 7))); + assert_eq!(parse_snapshot(""), Err(Rf2Error::Empty)); + assert_eq!( + parse_snapshot("id\tactive\tmoduleId\trefsetId\n"), + Err(Rf2Error::MissingColumn("effectiveTime")) + ); + } +}