diff --git a/Cargo.toml b/Cargo.toml index 27d173b2..b8549076 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -41,6 +41,7 @@ members = [ "crates/ogar-ad", "crates/ogar-az", "crates/ogar-dir-sim", + "crates/ogar-genom", ] [workspace.package] diff --git a/crates/ogar-genom/Cargo.toml b/crates/ogar-genom/Cargo.toml new file mode 100644 index 00000000..a383cba0 --- /dev/null +++ b/crates/ogar-genom/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "ogar-genom" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +authors.workspace = true +rust-version.workspace = true +description = "Packed genomic sequence substrate (first proof): DNA2 resident sequence (2 bits/base) with exact-round-trip ambiguity and soft-mask interval sidecars, the V4 base-set alphabet for IUPAC ambiguity, half-open coordinates with strand, zero-copy reverse-complement views, and the codon translation fold over the standard genetic code. Zero dependencies; mints no classids (the Genetics domain 0x0E is operator-gated)." + +[dependencies] diff --git a/crates/ogar-genom/lab/genom_probe.rs b/crates/ogar-genom/lab/genom_probe.rs new file mode 100644 index 00000000..b4b1b689 --- /dev/null +++ b/crates/ogar-genom/lab/genom_probe.rs @@ -0,0 +1,332 @@ +// LAB PROBE — not built by the workspace. Reproduce the OGAR-GENOM-CAPSTONE §3-§10 numbers: +// Cargo.toml: [dependencies] ndarray = { path = "", default-features = false, features = ["std"] } +// RUSTFLAGS="-C target-cpu=native" cargo run --release -- chr21.seq (chr21.seq = FASTA body, no header/newlines) +// SIMD comes from the ndarray::simd facade only. + +//! OGAR-GENOM feasibility probe. LAB CODE — never shipped. +//! All SIMD from `ndarray::simd` (facade), per the workspace invariant. +//! +//! Measures, on real GRCh38 chr21 (UCSC soft-masked): +//! E1 encode u8->DNA2 + interval sidecars, exact round-trip +//! E2 reverse complement: materialized (u8, DNA2) vs streaming view +//! E3 codon translation fold: u8 baseline vs DNA2 + permute_bytes LUT +//! E4 motif scan: exact k-mer (u8 vs DNA2 rolling) and IUPAC motif (V4 query over DNA2) +//! E5 allele-frequency fold over a synthetic genotype bitplane population +//! E6 sparse variant overlay read + +use ndarray::simd::U8x64; +use std::hint::black_box; +use std::time::Instant; + +// ---------- alphabet ---------- +const A: u8 = 0; +const C: u8 = 1; +const G: u8 = 2; +const T: u8 = 3; + +fn base2(b: u8) -> u8 { + match b { + b'A' | b'a' => A, + b'C' | b'c' => C, + b'G' | b'g' => G, + b'T' | b't' => T, + _ => 0, // N etc. -> placeholder, recorded in sidecar + } +} + +/// Standard genetic code (NCBI table 1) indexed by DNA2 codon b0<<4|b1<<2|b2, A=0 C=1 G=2 T=3. +fn codon_lut() -> [u8; 64] { + // canonical TCAG-ordered table + let tcag = b"FFLLSSSSYY**CC*WLLLLPPPPHHQQRRRRIIIMTTTTNNKKSSRRVVVVAAAADDEEGGGG"; + let ord = |x: u8| match x { T => 0, C => 1, A => 2, G => 3, _ => unreachable!() }; + let mut lut = [0u8; 64]; + for c in 0..64u8 { + let (b0, b1, b2) = (c >> 4, (c >> 2) & 3, c & 3); + lut[c as usize] = tcag[(ord(b0) * 16 + ord(b1) * 4 + ord(b2)) as usize]; + } + lut +} + +// ---------- packed resident sequence ---------- +struct Dna2 { + words: Vec, // 32 bases / u64, base i at bits 2*(i%32) + len: usize, +} +impl Dna2 { + #[inline(always)] + fn get(&self, i: usize) -> u8 { + ((self.words[i >> 5] >> ((i & 31) * 2)) & 3) as u8 + } +} + +/// Half-open intervals (start,end) — sidecar for N runs / soft-mask. +fn runs(seq: &[u8], pred: impl Fn(u8) -> bool) -> Vec<(u32, u32)> { + let mut v = Vec::new(); + let mut i = 0; + while i < seq.len() { + if pred(seq[i]) { + let s = i; + while i < seq.len() && pred(seq[i]) { i += 1; } + v.push((s as u32, i as u32)); + } else { i += 1; } + } + v +} + +fn encode(seq: &[u8]) -> Dna2 { + let mut words = vec![0u64; seq.len().div_ceil(32)]; + for (wi, chunk) in seq.chunks(32).enumerate() { + let mut w = 0u64; + for (j, &b) in chunk.iter().enumerate() { w |= (base2(b) as u64) << (2 * j); } + words[wi] = w; + } + Dna2 { words, len: seq.len() } +} + +fn decode(d: &Dna2, n_runs: &[(u32, u32)], soft: &[(u32, u32)]) -> Vec { + const UP: [u8; 4] = *b"ACGT"; + let mut out: Vec = (0..d.len).map(|i| UP[d.get(i) as usize]).collect(); + for &(s, e) in soft { for x in &mut out[s as usize..e as usize] { *x = x.to_ascii_lowercase(); } } + for &(s, e) in n_runs { for x in &mut out[s as usize..e as usize] { *x = if x.is_ascii_lowercase() { b'n' } else { b'N' }; } } + out +} + +/// Unpack 64 bases starting at base index `i` (must be multiple of 32) into one byte each. +#[inline(always)] +fn unpack64(d: &Dna2, wi: usize, buf: &mut [u8; 64]) { + let w0 = d.words[wi]; + let w1 = *d.words.get(wi + 1).unwrap_or(&0); + for j in 0..32 { buf[j] = ((w0 >> (2 * j)) & 3) as u8; buf[32 + j] = ((w1 >> (2 * j)) & 3) as u8; } +} + +fn time(name: &str, bytes: usize, reps: u32, mut f: impl FnMut() -> R) -> f64 { + black_box(f()); + let t = Instant::now(); + for _ in 0..reps { black_box(f()); } + let s = t.elapsed().as_secs_f64() / reps as f64; + println!(" {name:<46} {:>9.3} ms {:>8.2} GB/s(src-eq)", s * 1e3, bytes as f64 / s / 1e9); + s +} + +fn main() { + let path = std::env::args().nth(1).expect("chr21.seq"); + let seq = std::fs::read(&path).unwrap(); + let n = seq.len(); + println!("sequence: {n} bases from {path}"); + + // ---------------- E1 ---------------- + println!("\nE1 encode + sidecars"); + let n_runs = runs(&seq, |b| b == b'N' || b == b'n'); + let soft = runs(&seq, |b| b.is_ascii_lowercase()); + let other_iupac = seq.iter().filter(|&&b| !b"ACGTNacgtn".contains(&b)).count(); + let d = encode(&seq); + time("encode u8 -> DNA2", n, 5, || encode(&seq)); + let rt = decode(&d, &n_runs, &soft); + assert!(rt == seq, "round-trip FAILED"); + println!(" round-trip exact: OK other-IUPAC={other_iupac} N-runs={} softmask-runs={}", n_runs.len(), soft.len()); + let dna2_b = d.words.len() * 8; + let side_b = (n_runs.len() + soft.len()) * 8; + println!(" bytes: u8={n} DNA2={dna2_b} ({:.3} b/base) sidecars={side_b} ({:.4} b/base) V4(nibble)={}", + 8.0 * dna2_b as f64 / n as f64, 8.0 * side_b as f64 / n as f64, n.div_ceil(2)); + println!(" N-only sidecar: {} bytes", n_runs.len() * 8); + + // ---------------- E2 ---------------- + println!("\nE2 reverse complement (what bytes move?)"); + let comp = |b: u8| match b { b'A' => b'T', b'C' => b'G', b'G' => b'C', b'T' => b'A', b'a' => b't', b'c' => b'g', b'g' => b'c', b't' => b'a', x => x }; + time("u8 materialized revcomp (writes n B)", n, 5, || seq.iter().rev().map(|&b| comp(b)).collect::>()); + time("DNA2 materialized revcomp (writes n/4 B)", n, 5, || { + let nw = d.words.len(); + let mut out = vec![0u64; nw]; + let pad = nw * 32 - d.len; // shift to re-align after reversal + for (i, &w) in d.words.iter().enumerate() { + // reverse 2-bit groups within the word, then complement (xor 0b11 per base) + let mut x = w; + x = ((x >> 2) & 0x3333_3333_3333_3333) | ((x & 0x3333_3333_3333_3333) << 2); + x = ((x >> 4) & 0x0F0F_0F0F_0F0F_0F0F) | ((x & 0x0F0F_0F0F_0F0F_0F0F) << 4); + x = x.swap_bytes(); + out[nw - 1 - i] = !x; + } + // realign by `pad` bases (2*pad bits) — a funnel shift over the word stream + if pad > 0 { + let s = 2 * pad as u32; + for i in 0..nw { let hi = if i + 1 < nw { out[i + 1] } else { 0 }; out[i] = (out[i] >> s) | (hi << (64 - s)); } + } + out + }); + // streaming view: no buffer; a fold (GC count) through the revcomp coordinate transform + time("revcomp VIEW fold: GC count via i->len-1-i", n, 3, || { + let mut gc = 0u64; + for i in 0..d.len { let b = 3 - d.get(d.len - 1 - i); gc += (b == C || b == G) as u64; } + gc + }); + // word-granular streaming VIEW: produce revcomp words on the fly, no buffer; checksum vs materialized + let rc_word = |w: u64| -> u64 { let mut x = w; + x = ((x >> 2) & 0x3333_3333_3333_3333) | ((x & 0x3333_3333_3333_3333) << 2); + x = ((x >> 4) & 0x0F0F_0F0F_0F0F_0F0F) | ((x & 0x0F0F_0F0F_0F0F_0F0F) << 4); + !x.swap_bytes() }; + let nw = d.words.len(); let pad = (nw * 32 - d.len) as u32 * 2; + time("revcomp word-VIEW fold (checksum, 0 B written)", n, 5, || { + let mut acc = 0u64; + for i in 0..nw { let lo = rc_word(d.words[nw - 1 - i]); let hi = if i + 1 < nw { rc_word(d.words[nw - 2 - i]) } else { 0 }; + let w = if pad > 0 { (lo >> pad) | (hi << (64 - pad)) } else { lo }; acc = acc.rotate_left(7) ^ w; } + acc + }); + // GC count is strand-invariant: proves the fold needs no transform at all here. + time("direct GC fold, DNA2 popcount (strand-invariant)", n, 5, || { + // C=01,G=10: GC iff the two bits differ -> xor of the pair + let mut gc = 0u64; + for &w in &d.words { gc += (((w >> 1) ^ w) & 0x5555_5555_5555_5555).count_ones() as u64; } + gc // NB: tail/N-placeholder bases counted as A; corrected via sidecar in real code + }); + + // ---------------- E3 ---------------- + println!("\nE3 translation fold (frame 0, all 64 codons)"); + let lut = codon_lut(); + assert_eq!(lut[(A << 4 | T << 2 | G) as usize], b'M'); + assert_eq!(lut[(T << 4 | A << 2 | A) as usize], b'*'); + let ncod = n / 3; + let t_u8 = time("u8 baseline: 3 table lookups + LUT -> protein", n, 5, || { + let mut p = Vec::with_capacity(ncod); + for k in 0..ncod { let i = 3 * k; let c = base2(seq[i]) << 4 | base2(seq[i + 1]) << 2 | base2(seq[i + 2]); p.push(lut[c as usize]); } + p + }); + let t_d2 = time("DNA2 scalar: 6-bit extract (u128 window) + LUT", n, 5, || { + let mut p = Vec::with_capacity(ncod); + for k in 0..ncod { + let i = 3 * k; let (wi, off) = (i >> 5, (i & 31) * 2); + let w = d.words[wi] as u128 | (*d.words.get(wi + 1).unwrap_or(&0) as u128) << 64; + let raw = (w >> off) as u8 & 63; // b0 in low bits + let c = (raw & 3) << 4 | ((raw >> 2) & 3) << 2 | (raw >> 4); // reorder to b0<<4|b1<<2|b2 + p.push(lut[c as usize]); + } + p + }); + // all-offset codon fold: codon at EVERY base position (frames 0,1,2 at once) via permute_bytes + let lut_v = U8x64::from_array(lut); + let t_all = time("DNA2 SIMD: codon@every pos (3 frames), vpermb", n, 5, || { + let mut out = vec![0u8; n]; + let mut buf = [0u8; 64 + 2]; + let mut i = 0; + while i + 64 <= n { + let mut b64 = [0u8; 64]; + unpack64(&d, i >> 5, &mut b64); + buf[..64].copy_from_slice(&b64); + buf[64] = d.get((i + 64).min(n - 1)); buf[65] = d.get((i + 65).min(n - 1)); + let b0 = U8x64::from_slice(&buf[0..64]); + let b1 = U8x64::from_slice(&buf[1..65]); + let b2 = U8x64::from_slice(&buf[2..66]); + let idx = or3(b0.shl_epi16(4), b1.shl_epi16(2), b2); // byte-safe: values <4, no cross-byte carry + lut_v.permute_bytes(idx).copy_to_slice(&mut out[i..i + 64]); + i += 64; + } + out + }); + println!(" -> per-codon: u8 {:.2} ns, DNA2 scalar {:.2} ns; all-offset SIMD {:.3} ns/codon-slot ({}x codons for 3 frames)", + t_u8 / ncod as f64 * 1e9, t_d2 / ncod as f64 * 1e9, t_all / n as f64 * 1e9, 3); + // cross-check: SIMD all-offset frame 0 == scalar + { + let mut buf = [0u8; 66]; + let mut b64 = [0u8; 64]; + unpack64(&d, 0, &mut b64); + buf[..64].copy_from_slice(&b64); + buf[64] = d.get(64); buf[65] = d.get(65); + for k in 0..20 { let i = 3 * k; let c = buf[i] << 4 | buf[i + 1] << 2 | buf[i + 2]; assert_eq!(lut[c as usize], { let c2 = base2(seq[i]) << 4 | base2(seq[i + 1]) << 2 | base2(seq[i + 2]); lut[c2 as usize] }); } + println!(" frame-0 cross-check SIMD vs u8: OK"); + } + + // ---------------- E4 ---------------- + println!("\nE4 motif scan"); + let kmer = b"GGCTCACGCCTG"; // Alu-ish 12-mer + let km2: u32 = kmer.iter().enumerate().fold(0, |a, (j, &b)| a | (base2(b) as u32) << (2 * j)); + let up: Vec = seq.iter().map(|b| b.to_ascii_uppercase()).collect(); + let c_u8 = { let mut c = 0; time("u8 exact 12-mer (window compare)", n, 3, || { c = up.windows(12).filter(|w| *w == kmer).count(); c }); c }; + let c_d2 = { let mut c = 0; time("DNA2 rolling 24-bit compare", n, 3, || { + let mut h: u32 = 0; let mut cnt = 0usize; let mask = (1u32 << 24) - 1; + for i in 0..n { h = (h >> 2) | (d.get(i) as u32) << 22; if i >= 11 && (h & mask) == km2 { cnt += 1; } } + c = cnt; cnt }); c }; + println!(" hits u8={c_u8} DNA2={c_d2} (DNA2 also matches inside N runs as AAAA..: verify via sidecar)"); + // IUPAC motif TATAWAWR as V4 query over DNA2: onehot(base) & pat[j] != 0 + let v4 = |c: u8| -> u8 { match c { b'A'=>1,b'C'=>2,b'G'=>4,b'T'=>8,b'R'=>1|4,b'Y'=>2|8,b'W'=>1|8,b'S'=>2|4,b'N'=>15,_=>0 } }; + let pat: Vec = b"TATAWAWR".iter().map(|&c| v4(c)).collect(); + let pv: Vec = pat.iter().map(|&p| and_table(p)).collect(); + let mut onehot_lut = [0u8; 64]; for j in 0..4 { for l in 0..4 { onehot_lut[16 * l + j] = 1 << j; } } + let oh = U8x64::from_array(onehot_lut); + let zero = U8x64::splat(0); + let c_v4 = { let mut c = 0; time("V4 query over DNA2 (vpshufb onehot, AND, cmpeq)", n, 3, || { + let mut cnt = 0u64; + let mut ohs = vec![0u8; n + 64]; + let mut b64 = [0u8; 64]; + let mut i = 0; while i + 64 <= n { unpack64(&d, i >> 5, &mut b64); oh.shuffle_bytes(U8x64::from_array(b64)).copy_to_slice(&mut ohs[i..i + 64]); i += 64; } + let mut i = 0; while i + 64 + 8 <= n { + let mut hit = !0u64; + for (j, p) in pv.iter().enumerate() { let s = U8x64::from_slice(&ohs[i + j..i + j + 64]); hit &= !p.shuffle_bytes(s).cmpeq_mask(zero); } + cnt += hit.count_ones() as u64; i += 64; + } + c = cnt; cnt }); c }; + let definite = |b: u8| if b == b'N' { 0 } else { v4(b) }; + let re_def = up.windows(8).filter(|w| w.iter().zip(&pat).all(|(&b, &p)| definite(b) & p != 0)).count(); + let re_pos = up.windows(8).filter(|w| w.iter().zip(&pat).all(|(&b, &p)| v4(b) & p != 0)).count(); + // DNA2 placeholder for N is A; correct V4 hits by subtracting windows touching an N run + let in_n = { let mut c = 0usize; for i in 0..n.saturating_sub(8) { let touches = n_runs.iter().any(|&(s, e)| (i + 8) as u32 > s && (i as u32) < e); if touches { let w = &up[i..i+8]; if w.iter().zip(&pat).all(|(&b,&p)| { let x = if b==b'N' {1} else {v4(b)}; x & p != 0 }) { c += 1; } } } c }; + println!(" IUPAC TATAWAWR: V4/DNA2={c_v4} scalar definite={re_def} scalar possible(N=any)={re_pos} DNA2 hits inside N-runs={in_n}"); + + // ---------------- E5 ---------------- + println!("\nE5 allele-frequency fold, genotype bitplanes (site-major)"); + for &ind in &[100_000usize, 1_000_000] { + let sites = if ind == 1_000_000 { 200 } else { 2000 }; + let wpp = ind.div_ceil(64); // words per plane + // two planes per site: carries_alt, hom_alt; dosage = popcnt(p0)+popcnt(p1) + let mut rng = 0x9E3779B97F4A7C15u64; + let mut planes = vec![0u64; sites * 2 * wpp]; + for w in planes.iter_mut() { rng ^= rng << 13; rng ^= rng >> 7; rng ^= rng << 17; *w = rng & (rng >> 3) & (rng >> 5); } + let bytes = planes.len() * 8; + let s = time(&format!("{ind} indiv x {sites} sites: AF fold (popcnt)"), bytes, 3, || { + let mut af = vec![0u64; sites]; + for (si, a) in af.iter_mut().enumerate() { + let base = si * 2 * wpp; + *a = ndarray::simd::popcount_batch_u64(&planes[base..base + 2 * wpp]); + } + af + }); + println!(" -> {:.1} us/site, {:.2} ns/individual-site", s / sites as f64 * 1e6, s / (sites * ind) as f64 * 1e9); + // selection mask fold: fitness mask AND site plane -> AF among selected parents + let sel: Vec = (0..wpp).map(|i| (i as u64).wrapping_mul(0x9E37_79B9_7F4A_7C15)).collect(); + time(&format!("{ind} indiv: AF among selected (mask AND+popcnt)"), bytes, 3, || { + let mut tot = 0u64; + for si in 0..sites { let base = si * 2 * wpp; + for p in 0..2 { for (a, b) in planes[base + p * wpp..base + (p + 1) * wpp].iter().zip(&sel) { tot += (a & b).count_ones() as u64; } } } + tot + }); + } + + // ---------------- E6 ---------------- + println!("\nE6 sparse SNP overlay read (1 SNV / 1000 bp, human-like density)"); + let snvs: Vec<(u32, u8)> = (0..n as u32).step_by(1000).map(|p| (p, ((p >> 3) & 3) as u8)).collect(); + println!(" overlay: {} SNVs, {} B (pos u32 + alt u8, unpacked {}B/var)", snvs.len(), snvs.len() * 8, 8); + time("window read 1kb x 10k random windows w/ overlay", 10_000 * 1000, 3, || { + let mut acc = 0u64; let mut r = 12345u64; + for _ in 0..10_000 { + r = r.wrapping_mul(6364136223846793005).wrapping_add(1); + let s = (r >> 20) as usize % (n - 1000); + let mut k = snvs.partition_point(|&(p, _)| (p as usize) < s); + for i in s..s + 1000 { + let b = if k < snvs.len() && snvs[k].0 as usize == i { k += 1; snvs[k - 1].1 } else { d.get(i) }; + acc += b as u64; + } + } + acc + }); + time("window read 1kb x 10k random windows, no overlay", 10_000 * 1000, 3, || { + let mut acc = 0u64; let mut r = 12345u64; + for _ in 0..10_000 { r = r.wrapping_mul(6364136223846793005).wrapping_add(1); let s = (r >> 20) as usize % (n - 1000); + for i in s..s + 1000 { acc += d.get(i) as u64; } } + acc + }); +} + +// OR of disjoint bit fields (b0<<4 | b1<<2 | b2) via saturating_add — exact, no carries possible. +#[inline(always)] +fn or3(a: U8x64, b: U8x64, c: U8x64) -> U8x64 { a.saturating_add(b).saturating_add(c) } + +/// Per-pattern-position AND table: shuffle_bytes(T_p, onehot) = onehot & p (16-entry per 128-bit lane). +fn and_table(p: u8) -> U8x64 { let mut t = [0u8; 64]; for l in 0..4 { for x in 0..16 { t[16 * l + x] = (x as u8) & p; } } U8x64::from_array(t) } diff --git a/crates/ogar-genom/src/alphabet.rs b/crates/ogar-genom/src/alphabet.rs new file mode 100644 index 00000000..d4147b73 --- /dev/null +++ b/crates/ogar-genom/src/alphabet.rs @@ -0,0 +1,167 @@ +//! The two alphabets: DNA2 [`Base`] (resident) and V4 [`BaseSet`] (ambiguity algebra). + +/// One unambiguous nucleotide, in DNA2 order. Complement is `3 - b` (= `b ^ 0b11`). +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +#[repr(u8)] +pub enum Base { + A = 0, + C = 1, + G = 2, + T = 3, +} + +impl Base { + pub const ALL: [Base; 4] = [Base::A, Base::C, Base::G, Base::T]; + + /// From the 2-bit code; only the low two bits are read. + #[inline] + pub const fn from_code(code: u8) -> Base { + match code & 3 { + 0 => Base::A, + 1 => Base::C, + 2 => Base::G, + _ => Base::T, + } + } + + #[inline] + pub const fn code(self) -> u8 { + self as u8 + } + + #[inline] + pub const fn complement(self) -> Base { + Base::from_code(3 - self as u8) + } + + /// Upper-case ASCII letter. + #[inline] + pub const fn to_ascii(self) -> u8 { + b"ACGT"[self as usize] + } + + /// From an unambiguous ASCII base (either case); `None` for anything else. + #[inline] + pub const fn from_ascii(c: u8) -> Option { + match c { + b'A' | b'a' => Some(Base::A), + b'C' | b'c' => Some(Base::C), + b'G' | b'g' => Some(Base::G), + b'T' | b't' => Some(Base::T), + _ => None, + } + } +} + +/// V4: a nibble naming the set of possible bases. Bit `b` set ⇔ `Base` with code `b` is possible. +/// +/// `A=0001 C=0010 G=0100 T=1000`; `R=A|G=0101`, `Y=C|T=1010`, `N=1111`. +/// The empty set (`0000`) is representable and means "contradiction"; it is +/// never produced by IUPAC decoding. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct BaseSet(u8); + +impl BaseSet { + pub const EMPTY: BaseSet = BaseSet(0); + pub const ANY: BaseSet = BaseSet(0b1111); + + /// From the raw nibble; bits above 3 are discarded. + #[inline] + pub const fn from_bits(bits: u8) -> BaseSet { + BaseSet(bits & 0b1111) + } + + #[inline] + pub const fn bits(self) -> u8 { + self.0 + } + + #[inline] + pub const fn single(b: Base) -> BaseSet { + BaseSet(1 << b as u8) + } + + #[inline] + pub const fn union(self, o: BaseSet) -> BaseSet { + BaseSet(self.0 | o.0) + } + + #[inline] + pub const fn intersection(self, o: BaseSet) -> BaseSet { + BaseSet(self.0 & o.0) + } + + #[inline] + pub const fn is_subset_of(self, o: BaseSet) -> bool { + self.0 & !o.0 == 0 + } + + #[inline] + pub const fn contains(self, b: Base) -> bool { + self.0 & (1 << b as u8) != 0 + } + + #[inline] + pub const fn is_empty(self) -> bool { + self.0 == 0 + } + + #[inline] + pub const fn len(self) -> u32 { + self.0.count_ones() + } + + /// The single base, if the set has exactly one member. + #[inline] + pub const fn as_single(self) -> Option { + match self.0 { + 0b0001 => Some(Base::A), + 0b0010 => Some(Base::C), + 0b0100 => Some(Base::G), + 0b1000 => Some(Base::T), + _ => None, + } + } + + /// Complement of every member (A↔T, C↔G) — the set on the other strand. + /// Reverses the nibble's bit order, because complement is `3 - code`. + #[inline] + pub const fn complement(self) -> BaseSet { + let b = self.0; + BaseSet(((b & 1) << 3) | ((b & 2) << 1) | ((b & 4) >> 1) | ((b & 8) >> 3)) + } + + /// Iterate the members in code order. + pub fn iter(self) -> impl Iterator { + Base::ALL.into_iter().filter(move |&b| self.contains(b)) + } + + /// From an IUPAC nucleotide letter (either case; `U` reads as `T`). + pub const fn from_iupac(c: u8) -> Option { + let bits = match c.to_ascii_uppercase() { + b'A' => 0b0001, + b'C' => 0b0010, + b'G' => 0b0100, + b'T' | b'U' => 0b1000, + b'R' => 0b0101, + b'Y' => 0b1010, + b'S' => 0b0110, + b'W' => 0b1001, + b'K' => 0b1100, + b'M' => 0b0011, + b'B' => 0b1110, + b'D' => 0b1101, + b'H' => 0b1011, + b'V' => 0b0111, + b'N' => 0b1111, + _ => return None, + }; + Some(BaseSet(bits)) + } + + /// Upper-case IUPAC letter. The empty set has no letter and returns `b'-'` + /// (IUPAC's gap symbol); callers that can produce it should check first. + pub const fn to_iupac(self) -> u8 { + b"-ACMGRSVTWYHKDBN"[self.0 as usize] + } +} diff --git a/crates/ogar-genom/src/coord.rs b/crates/ogar-genom/src/coord.rs new file mode 100644 index 00000000..fe778404 --- /dev/null +++ b/crates/ogar-genom/src/coord.rs @@ -0,0 +1,108 @@ +//! Coordinates: half-open intervals, strand, reading frame. +//! +//! These are **reference coordinates** — positions in one packed sequence. A +//! coordinate on a variant-bearing haplotype is a different space (indels +//! shift it), and is deliberately not representable here: the liftover between +//! the two belongs to the variant layer, not to this type. + +/// Half-open `[start, end)` interval in base positions. +/// +/// The fields are private so `start <= end` holds for every value: the only +/// public constructor is [`Interval::new`], which refuses inverted bounds. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct Interval { + start: u64, + end: u64, +} + +impl Interval { + /// `None` when `start > end`. + pub const fn new(start: u64, end: u64) -> Option { + if start <= end { + Some(Interval { start, end }) + } else { + None + } + } + + /// Crate-internal constructor for bounds the caller has already ordered. + #[inline] + pub(crate) const fn ordered(start: u64, end: u64) -> Interval { + debug_assert!(start <= end); + Interval { start, end } + } + + /// Crate-internal: grow a run by one base at its end. + #[inline] + pub(crate) fn extend_end(&mut self) { + self.end += 1; + } + + #[inline] + pub const fn start(self) -> u64 { + self.start + } + + #[inline] + pub const fn end(self) -> u64 { + self.end + } + + #[inline] + pub const fn len(self) -> u64 { + self.end - self.start + } + + #[inline] + pub const fn is_empty(self) -> bool { + self.start == self.end + } + + #[inline] + pub const fn contains(self, pos: u64) -> bool { + self.start <= pos && pos < self.end + } + + #[inline] + pub const fn contains_interval(self, o: Interval) -> bool { + self.start <= o.start && o.end <= self.end + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum Strand { + Forward, + Reverse, +} + +impl Strand { + #[inline] + pub const fn flip(self) -> Strand { + match self { + Strand::Forward => Strand::Reverse, + Strand::Reverse => Strand::Forward, + } + } +} + +/// Reading frame offset `0..3` within a view (after strand is applied). +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct Frame(u8); + +impl Frame { + pub const ZERO: Frame = Frame(0); + + /// `None` for offsets ≥ 3. + pub const fn new(offset: u8) -> Option { + if offset < 3 { + Some(Frame(offset)) + } else { + None + } + } + + #[inline] + pub const fn offset(self) -> u8 { + self.0 + } +} diff --git a/crates/ogar-genom/src/lib.rs b/crates/ogar-genom/src/lib.rs new file mode 100644 index 00000000..6fb32184 --- /dev/null +++ b/crates/ogar-genom/src/lib.rs @@ -0,0 +1,42 @@ +//! # ogar-genom — packed genomic sequence substrate (first proof) +//! +//! The law this crate tests: **the sequence stays resident; another biological +//! interpretation is a new coordinate, a mask, or a fold — not a copy.** +//! +//! | operation | what bytes move | +//! |---|---| +//! | [`encode`](sequence::encode) | ASCII in → 2 bits/base + interval sidecars (a durable write, once) | +//! | [`SeqView::sub`](view::SeqView::sub) / [`SeqView::reverse_complement`](view::SeqView::reverse_complement) | none — a view is a borrow plus an interval and a strand | +//! | [`SeqView::base`](view::SeqView::base), [`SeqView::base_set`](view::SeqView::base_set) | one word read, plus a binary search of the ambiguity sidecar | +//! | [`translate`](translate::translate) | streaming fold: yields amino acids, allocates nothing | +//! | [`decode`](sequence::decode) | full materialization, by name, O(n) | +//! +//! ## Representation (measured, not chosen by elegance) +//! +//! Resident sequence is **DNA2** (`A=00 C=01 G=10 T=11`, 32 bases per `u64`). +//! Ambiguity (IUPAC `N`, `R`, `Y`, …) lives in a **sparse run sidecar**, not in +//! the resident bits. On GRCh38 chr21 (46.7 Mbp) the reference has **52** `N` +//! runs, **zero** non-`N` IUPAC symbols, and **48,677** soft-mask runs: the +//! sidecars cost 0.067 bits/base against V4's 4 bits/base everywhere. **V4** +//! ([`BaseSet`](alphabet::BaseSet)) is kept as the *query/answer* algebra — +//! union, intersection and subset over possible bases — not as storage. +//! See `docs/OGAR-GENOM-CAPSTONE.md` for the measurements and the falsifiers. +//! +//! ## What this crate does NOT do +//! +//! No variants, transcripts, populations, lineage or ontology addresses (those +//! are later layers), no file-format parsers (adapters at the boundary), no +//! classid mints, and no SIMD: kernels here are word-level bit operations. SIMD +//! fast paths, when they land, come from `ndarray::simd` only. + +pub mod alphabet; +pub mod coord; +pub mod sequence; +pub mod translate; +pub mod view; + +pub use alphabet::{Base, BaseSet}; +pub use coord::{Frame, Interval, Strand}; +pub use sequence::{AmbiguityRun, EncodeError, PackedSeq, decode, encode}; +pub use translate::{AminoAcids, CODON_TABLE, codon_index, translate}; +pub use view::SeqView; diff --git a/crates/ogar-genom/src/sequence.rs b/crates/ogar-genom/src/sequence.rs new file mode 100644 index 00000000..3912a501 --- /dev/null +++ b/crates/ogar-genom/src/sequence.rs @@ -0,0 +1,184 @@ +//! The resident sequence: DNA2 words plus sparse interval sidecars. +//! +//! Exact round trip is part of the contract: [`decode`]`(`[`encode`]`(x)) == x` +//! for every accepted input, case included. + +use crate::alphabet::{Base, BaseSet}; +use crate::coord::Interval; + +/// A maximal run of one ambiguous IUPAC symbol, e.g. 50,000 × `N`. +/// +/// Resident DNA2 bits inside the run are **placeholder zeros** and carry no +/// meaning; every read that may land in a run must go through +/// [`PackedSeq::base_set`]. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct AmbiguityRun { + pub interval: Interval, + pub set: BaseSet, +} + +/// DNA2-packed sequence: 32 bases per `u64`, base `i` at bits `2*(i%32)..+2`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PackedSeq { + words: Box<[u64]>, + len: u64, + /// Sorted, non-overlapping, maximal (adjacent runs differ in `set`). + ambiguity: Box<[AmbiguityRun]>, + /// Sorted, non-overlapping, maximal lower-case runs (soft-masked repeats). + soft_mask: Box<[Interval]>, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum EncodeError { + /// A byte that is not an IUPAC nucleotide letter. `U`/`u` is refused on + /// purpose: RNA is a different layer, and accepting it as `T` would make + /// the round trip lossy. + InvalidSymbol { pos: u64, byte: u8 }, +} + +impl core::fmt::Display for EncodeError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + EncodeError::InvalidSymbol { pos, byte } => { + write!( + f, + "invalid nucleotide symbol 0x{byte:02x} at position {pos}" + ) + } + } + } +} + +impl std::error::Error for EncodeError {} + +/// Encode an ASCII nucleotide sequence (no FASTA headers or newlines). +pub fn encode(ascii: &[u8]) -> Result { + let len = ascii.len() as u64; + let mut words = vec![0u64; ascii.len().div_ceil(32)].into_boxed_slice(); + let mut ambiguity: Vec = Vec::new(); + let mut soft_mask: Vec = Vec::new(); + + for (i, &c) in ascii.iter().enumerate() { + let pos = i as u64; + if c.is_ascii_lowercase() { + match soft_mask.last_mut() { + Some(iv) if iv.end() == pos => iv.extend_end(), + _ => soft_mask.push(Interval::ordered(pos, pos + 1)), + } + } + if let Some(b) = Base::from_ascii(c) { + words[i >> 5] |= (b.code() as u64) << ((i & 31) * 2); + continue; + } + let set = match BaseSet::from_iupac(c) { + Some(s) if !matches!(c, b'U' | b'u') => s, + _ => return Err(EncodeError::InvalidSymbol { pos, byte: c }), + }; + match ambiguity.last_mut() { + Some(r) if r.interval.end() == pos && r.set == set => r.interval.extend_end(), + _ => ambiguity.push(AmbiguityRun { + interval: Interval::ordered(pos, pos + 1), + set, + }), + } + } + Ok(PackedSeq { + words, + len, + ambiguity: ambiguity.into_boxed_slice(), + soft_mask: soft_mask.into_boxed_slice(), + }) +} + +/// Full materialization back to ASCII. O(n) bytes written — the one named +/// materializer of this layer. +pub fn decode(seq: &PackedSeq) -> Vec { + let mut out: Vec = (0..seq.len) + .map(|p| seq.resident_base(p).to_ascii()) + .collect(); + for r in seq.ambiguity.iter() { + let c = r.set.to_iupac(); + out[r.interval.start() as usize..r.interval.end() as usize].fill(c); + } + for iv in seq.soft_mask.iter() { + for x in &mut out[iv.start() as usize..iv.end() as usize] { + *x = x.to_ascii_lowercase(); + } + } + out +} + +impl PackedSeq { + #[inline] + pub fn len(&self) -> u64 { + self.len + } + + #[inline] + pub fn is_empty(&self) -> bool { + self.len == 0 + } + + /// The packed words. Bits past `len` in the last word are zero. + #[inline] + pub fn words(&self) -> &[u64] { + &self.words + } + + pub fn ambiguity_runs(&self) -> &[AmbiguityRun] { + &self.ambiguity + } + + pub fn soft_mask(&self) -> &[Interval] { + &self.soft_mask + } + + /// Resident bytes: words + sidecars (excluding the struct header). + pub fn resident_bytes(&self) -> usize { + self.words.len() * 8 + + self.ambiguity.len() * core::mem::size_of::() + + self.soft_mask.len() * core::mem::size_of::() + } + + /// The raw resident base — **ignores the ambiguity overlay**. Correct only + /// where the caller already knows `pos` lies outside every ambiguity run. + #[inline] + pub fn resident_base(&self, pos: u64) -> Base { + debug_assert!(pos < self.len); + let i = pos as usize; + Base::from_code((self.words[i >> 5] >> ((i & 31) * 2)) as u8) + } + + /// The ambiguity run covering `pos`, if any. O(log runs). + pub fn ambiguity_at(&self, pos: u64) -> Option { + let k = self.ambiguity.partition_point(|r| r.interval.end() <= pos); + self.ambiguity + .get(k) + .copied() + .filter(|r| r.interval.contains(pos)) + } + + /// The set of possible bases at `pos` on the forward strand. Panics if out of range. + pub fn base_set(&self, pos: u64) -> BaseSet { + assert!( + pos < self.len, + "position {pos} out of range (len {})", + self.len + ); + match self.ambiguity_at(pos) { + Some(r) => r.set, + None => BaseSet::single(self.resident_base(pos)), + } + } + + /// Whether `pos` is soft-masked (lower-case in the source). O(log runs). + pub fn is_soft_masked(&self, pos: u64) -> bool { + let k = self.soft_mask.partition_point(|iv| iv.end() <= pos); + self.soft_mask.get(k).is_some_and(|iv| iv.contains(pos)) + } + + /// Whole-sequence interval `[0, len)`. + pub fn full(&self) -> Interval { + Interval::ordered(0, self.len) + } +} diff --git a/crates/ogar-genom/src/translate.rs b/crates/ogar-genom/src/translate.rs new file mode 100644 index 00000000..376f3667 --- /dev/null +++ b/crates/ogar-genom/src/translate.rs @@ -0,0 +1,92 @@ +//! The codon fold: DNA2 codon (6 bits) → amino acid, standard genetic code. +//! +//! The 64-entry table is the whole "ISA". It is biology's table, not ours, and +//! it is kept apart from every other 0..63 vocabulary in the workspace. + +use crate::alphabet::{Base, BaseSet}; +use crate::coord::Frame; +use crate::view::SeqView; + +/// Codon index `b0<<4 | b1<<2 | b2` in DNA2 codes (A=0 C=1 G=2 T=3). +#[inline] +pub const fn codon_index(b0: Base, b1: Base, b2: Base) -> u8 { + (b0.code() << 4) | (b1.code() << 2) | b2.code() +} + +/// NCBI translation table 1, indexed by [`codon_index`]; `*` is stop. +pub const CODON_TABLE: [u8; 64] = build_table(); + +const fn build_table() -> [u8; 64] { + // The classic TCAG-ordered table, re-indexed into DNA2 (ACGT) order. + const TCAG: &[u8; 64] = b"FFLLSSSSYY**CC*WLLLLPPPPHHQQRRRRIIIMTTTTNNKKSSRRVVVVAAAADDEEGGGG"; + const ORD: [usize; 4] = [2, 1, 3, 0]; // DNA2 code → TCAG rank (A→2, C→1, G→3, T→0) + let mut out = [0u8; 64]; + let mut c = 0; + while c < 64 { + out[c] = TCAG[ORD[c >> 4] * 16 + ORD[(c >> 2) & 3] * 4 + ORD[c & 3]]; + c += 1; + } + out +} + +/// Translate one codon of possible-base sets. If every expansion yields the +/// same residue (e.g. `GCN` → `A`), that residue; otherwise `X`. An empty set +/// anywhere yields `X`. +pub fn translate_codon_sets(s0: BaseSet, s1: BaseSet, s2: BaseSet) -> u8 { + if let (Some(a), Some(b), Some(c)) = (s0.as_single(), s1.as_single(), s2.as_single()) { + return CODON_TABLE[codon_index(a, b, c) as usize]; + } + let mut seen: Option = None; + for a in s0.iter() { + for b in s1.iter() { + for c in s2.iter() { + let aa = CODON_TABLE[codon_index(a, b, c) as usize]; + match seen { + None => seen = Some(aa), + Some(x) if x == aa => {} + Some(_) => return b'X', + } + } + } + } + seen.unwrap_or(b'X') +} + +/// Streaming translation of a view in a reading frame. Yields one residue per +/// complete codon; a trailing partial codon is dropped. Allocates nothing. +pub fn translate<'a>(view: SeqView<'a>, frame: Frame) -> AminoAcids<'a> { + AminoAcids { + view, + pos: frame.offset() as u64, + } +} + +pub struct AminoAcids<'a> { + view: SeqView<'a>, + pos: u64, +} + +impl Iterator for AminoAcids<'_> { + type Item = u8; + + fn next(&mut self) -> Option { + if self.pos + 3 > self.view.len() { + return None; + } + let p = self.pos; + self.pos += 3; + let v = &self.view; + Some(translate_codon_sets( + v.base_set(p), + v.base_set(p + 1), + v.base_set(p + 2), + )) + } + + fn size_hint(&self) -> (usize, Option) { + let n = (self.view.len().saturating_sub(self.pos) / 3) as usize; + (n, Some(n)) + } +} + +impl ExactSizeIterator for AminoAcids<'_> {} diff --git a/crates/ogar-genom/src/view.rs b/crates/ogar-genom/src/view.rs new file mode 100644 index 00000000..8542641d --- /dev/null +++ b/crates/ogar-genom/src/view.rs @@ -0,0 +1,133 @@ +//! Zero-copy views: a borrow of the resident sequence plus an interval and a strand. +//! +//! A view owns nothing. `sub` and `reverse_complement` return new views over the +//! same words; no base is copied until a caller asks for one. + +use crate::alphabet::{Base, BaseSet}; +use crate::coord::{Interval, Strand}; +use crate::sequence::PackedSeq; + +#[derive(Clone, Copy, Debug)] +pub struct SeqView<'a> { + seq: &'a PackedSeq, + /// Reference interval, always forward-strand coordinates. + interval: Interval, + strand: Strand, +} + +impl<'a> SeqView<'a> { + /// Forward-strand view of the whole sequence. + pub fn new(seq: &'a PackedSeq) -> SeqView<'a> { + SeqView { + seq, + interval: seq.full(), + strand: Strand::Forward, + } + } + + /// View of a reference interval on a strand; `None` if out of range. + pub fn of(seq: &'a PackedSeq, interval: Interval, strand: Strand) -> Option> { + (interval.end() <= seq.len()).then_some(SeqView { + seq, + interval, + strand, + }) + } + + #[inline] + pub fn sequence(&self) -> &'a PackedSeq { + self.seq + } + + #[inline] + pub fn reference_interval(&self) -> Interval { + self.interval + } + + #[inline] + pub fn strand(&self) -> Strand { + self.strand + } + + #[inline] + pub fn len(&self) -> u64 { + self.interval.len() + } + + #[inline] + pub fn is_empty(&self) -> bool { + self.interval.is_empty() + } + + /// View position `i` → forward reference position. Panics when out of range. + #[inline] + pub fn reference_pos(&self, i: u64) -> u64 { + assert!( + i < self.len(), + "view index {i} out of range (len {})", + self.len() + ); + match self.strand { + Strand::Forward => self.interval.start() + i, + Strand::Reverse => self.interval.end() - 1 - i, + } + } + + /// Possible bases at view position `i`, read on this view's strand. + pub fn base_set(&self, i: u64) -> BaseSet { + let s = self.seq.base_set(self.reference_pos(i)); + match self.strand { + Strand::Forward => s, + Strand::Reverse => s.complement(), + } + } + + /// The base at `i` if unambiguous. + #[inline] + pub fn base(&self, i: u64) -> Option { + self.base_set(i).as_single() + } + + pub fn is_soft_masked(&self, i: u64) -> bool { + self.seq.is_soft_masked(self.reference_pos(i)) + } + + /// Sub-view by **view** coordinates (so on the reverse strand, `[0, k)` is + /// the first `k` bases read 5'→3' on that strand). `None` if out of range. + pub fn sub(&self, view_interval: Interval) -> Option> { + if view_interval.end() > self.len() { + return None; + } + let interval = match self.strand { + Strand::Forward => Interval::ordered( + self.interval.start() + view_interval.start(), + self.interval.start() + view_interval.end(), + ), + Strand::Reverse => Interval::ordered( + self.interval.end() - view_interval.end(), + self.interval.end() - view_interval.start(), + ), + }; + Some(SeqView { interval, ..*self }) + } + + /// Same reference interval, read on the other strand. No bytes move. + #[inline] + pub fn reverse_complement(&self) -> SeqView<'a> { + SeqView { + strand: self.strand.flip(), + ..*self + } + } + + /// Iterate possible-base sets along the view. Allocates nothing. + pub fn base_sets(&self) -> impl Iterator + 'a { + let v = *self; + (0..v.len()).map(move |i| v.base_set(i)) + } + + /// Materialize as IUPAC ASCII (upper-case). O(len) — by name. + pub fn materialize_iupac(&self) -> Vec { + self.base_sets().map(BaseSet::to_iupac).collect() + } +} diff --git a/crates/ogar-genom/tests/no_alloc.rs b/crates/ogar-genom/tests/no_alloc.rs new file mode 100644 index 00000000..bce0a239 --- /dev/null +++ b/crates/ogar-genom/tests/no_alloc.rs @@ -0,0 +1,62 @@ +//! "A view does not secretly allocate a population": count heap allocations +//! while building views and running folds over a resident sequence. + +use ogar_genom::*; +use std::alloc::{GlobalAlloc, Layout, System}; +use std::sync::atomic::{AtomicUsize, Ordering}; + +struct Counting; +static ALLOCS: AtomicUsize = AtomicUsize::new(0); +unsafe impl GlobalAlloc for Counting { + unsafe fn alloc(&self, l: Layout) -> *mut u8 { + ALLOCS.fetch_add(1, Ordering::SeqCst); + unsafe { System.alloc(l) } + } + unsafe fn dealloc(&self, p: *mut u8, l: Layout) { + unsafe { System.dealloc(p, l) } + } +} +#[global_allocator] +static A: Counting = Counting; + +#[test] +fn views_and_folds_allocate_nothing() { + let src: Vec = (0..100_000u32) + .map(|i| b"ACGTNacgt"[(i.wrapping_mul(2654435761) >> 29) as usize % 9]) + .collect(); + let p = encode(&src).unwrap(); + let before = ALLOCS.load(Ordering::SeqCst); + + let v = SeqView::new(&p); + let rc = v.reverse_complement(); + let sub = rc.sub(Interval::new(1_000, 90_000).unwrap()).unwrap(); + let mut residues = 0usize; + for f in 0..3 { + residues += translate(sub, Frame::new(f).unwrap()) + .filter(|&a| a != b'X') + .count(); + } + let gc = sub + .base_sets() + .filter(|s| s.is_subset_of(BaseSet::from_iupac(b'S').unwrap())) + .count(); + + let after = ALLOCS.load(Ordering::SeqCst); + assert_eq!( + after - before, + 0, + "views/folds allocated {} times", + after - before + ); + // anti-vacuity: the folds really ran over real content + assert!( + residues > 1_000 && gc > 1_000, + "residues={residues} gc={gc}" + ); + // ...and the counter can fire: a materialization is observed + let _m = sub.materialize_iupac(); + assert!( + ALLOCS.load(Ordering::SeqCst) > after, + "counting allocator is live" + ); +} diff --git a/crates/ogar-genom/tests/substrate.rs b/crates/ogar-genom/tests/substrate.rs new file mode 100644 index 00000000..a2af93d4 --- /dev/null +++ b/crates/ogar-genom/tests/substrate.rs @@ -0,0 +1,263 @@ +//! Falsifiers for the first ogar-genom proof. Every check compares against an +//! independent derivation (a naive u8 implementation or a hand-written table), +//! never against the code under test. + +use ogar_genom::*; + +/// Independent naive reference: u8 reverse complement over IUPAC letters. +fn naive_revcomp(s: &[u8]) -> Vec { + s.iter() + .rev() + .map(|&c| { + BaseSet::from_iupac(c) + .expect("fixture is IUPAC") + .complement() + .to_iupac() + }) + .collect() +} + +/// Independent naive translation over unambiguous upper-case ACGT. +fn naive_translate(s: &[u8], frame: usize) -> Vec { + // hand-written ACGT-ordered standard code (not derived from CODON_TABLE) + const T: &[u8; 64] = b"KNKNTTTTRSRSIIMIQHQHPPPPRRRRLLLLEDEDAAAAGGGGVVVV*Y*YSSSS*CWCLFLF"; + let code = |c: u8| b"ACGT".iter().position(|&x| x == c).unwrap(); + s[frame..] + .as_chunks::<3>() + .0 + .iter() + .map(|w| T[code(w[0]) * 16 + code(w[1]) * 4 + code(w[2])]) + .collect() +} + +fn lcg_seq(n: usize, seed: u64) -> Vec { + let mut x = seed; + (0..n) + .map(|_| { + x = x + .wrapping_mul(6364136223846793005) + .wrapping_add(1442695040888963407); + b"ACGT"[(x >> 62) as usize] + }) + .collect() +} + +#[test] +fn round_trip_is_exact_including_case_and_ambiguity() { + let fixtures: &[&[u8]] = &[ + b"", + b"A", + b"ACGTacgt", + b"NNNNNNNNNNacgtNNNNnnnnACGTRYKMSWBDHVrykmswbdhvN", + b"NNNRRRNNN", // adjacent runs of different symbols + b"ACGTACGTACGTACGTACGTACGTACGTACGTA", // 33: crosses the first word boundary + ]; + for &f in fixtures { + let p = encode(f).unwrap(); + assert_eq!(decode(&p), f, "fixture {:?}", String::from_utf8_lossy(f)); + assert_eq!(p.len(), f.len() as u64); + } + let long = lcg_seq(10_007, 7); + assert_eq!(decode(&encode(&long).unwrap()), long); +} + +#[test] +fn sidecars_are_maximal_runs() { + let p = encode(b"ACNNNNRRgtNN").unwrap(); + let runs: Vec<_> = p + .ambiguity_runs() + .iter() + .map(|r| (r.interval.start(), r.interval.end(), r.set.to_iupac())) + .collect(); + assert_eq!(runs, vec![(2, 6, b'N'), (6, 8, b'R'), (10, 12, b'N')]); + let soft: Vec<_> = p.soft_mask().iter().map(|i| (i.start(), i.end())).collect(); + assert_eq!(soft, vec![(8, 10)]); +} + +#[test] +fn invalid_symbols_are_refused_with_position() { + for (s, pos, byte) in [ + (&b"ACGU"[..], 3u64, b'U'), + (b"AC-G", 2, b'-'), + (b"AC\nG", 2, b'\n'), + (b"AXG", 1, b'X'), + ] { + assert_eq!(encode(s), Err(EncodeError::InvalidSymbol { pos, byte })); + } +} + +#[test] +fn v4_algebra_over_all_iupac_letters() { + let letters = b"ACGTRYSWKMBDHVN"; + for &c in letters { + let s = BaseSet::from_iupac(c).unwrap(); + assert_eq!(s.to_iupac(), c, "letter round trip"); + assert_eq!( + s.complement().complement(), + s, + "complement is an involution" + ); + // complement maps each member to its complement base + let expect = s.iter().fold(BaseSet::EMPTY, |acc, b| { + acc.union(BaseSet::single(b.complement())) + }); + assert_eq!(s.complement(), expect); + } + let r = BaseSet::from_iupac(b'R').unwrap(); // A|G + let y = BaseSet::from_iupac(b'Y').unwrap(); // C|T + assert!(r.intersection(y).is_empty()); + assert_eq!(r.union(y), BaseSet::ANY); + assert!(BaseSet::single(Base::A).is_subset_of(r)); + assert!(!BaseSet::single(Base::C).is_subset_of(r)); + assert_eq!(r.complement(), y); + assert_eq!(BaseSet::EMPTY.to_iupac(), b'-'); +} + +#[test] +fn codon_table_matches_an_independent_hand_written_table() { + const HAND: &[u8; 64] = b"KNKNTTTTRSRSIIMIQHQHPPPPRRRRLLLLEDEDAAAAGGGGVVVV*Y*YSSSS*CWCLFLF"; + assert_eq!(&CODON_TABLE, HAND); + for a in Base::ALL { + for b in Base::ALL { + for c in Base::ALL { + let s = [a.to_ascii(), b.to_ascii(), c.to_ascii()]; + assert_eq!( + CODON_TABLE[codon_index(a, b, c) as usize], + naive_translate(&s, 0)[0] + ); + } + } + } + assert_eq!(CODON_TABLE.iter().filter(|&&x| x == b'*').count(), 3); + assert_eq!( + CODON_TABLE[codon_index(Base::A, Base::T, Base::G) as usize], + b'M' + ); +} + +#[test] +fn reverse_complement_is_a_view() { + let src = b"AACGTTTGCAnRYacg"; + let p = encode(src).unwrap(); + let v = SeqView::new(&p); + let rc = v.reverse_complement(); + assert_eq!(rc.materialize_iupac(), naive_revcomp(src)); + assert_eq!( + rc.reverse_complement().materialize_iupac(), + v.materialize_iupac() + ); + assert!( + core::ptr::eq(rc.sequence(), &p), + "same resident sequence, nothing copied" + ); + // soft mask follows the reference position, not the view position + assert!(rc.is_soft_masked(0) && !rc.is_soft_masked(3)); +} + +#[test] +fn sub_views_compose_on_both_strands() { + let src = lcg_seq(200, 3); + let p = encode(&src).unwrap(); + let v = SeqView::new(&p); + let iv = Interval::new(37, 151).unwrap(); + let fwd = v.sub(iv).unwrap(); + assert_eq!(fwd.materialize_iupac(), &src[37..151]); + let rc_sub = v.reverse_complement().sub(iv).unwrap(); + let full_rc = naive_revcomp(&src); + assert_eq!(rc_sub.materialize_iupac(), &full_rc[37..151]); + // sub of a reverse view == reverse of the matching forward sub + let inner = Interval::new(10, 40).unwrap(); + assert_eq!( + rc_sub.sub(inner).unwrap().materialize_iupac(), + &full_rc[47..77] + ); + assert!(v.sub(Interval::new(0, 201).unwrap()).is_none()); +} + +#[test] +fn all_frames_both_strands_across_word_boundaries_match_naive() { + for (n, seed) in [ + (5usize, 1u64), + (31, 2), + (32, 3), + (33, 4), + (64, 5), + (97, 6), + (1000, 7), + (4099, 8), + ] { + let src = lcg_seq(n, seed); + let p = encode(&src).unwrap(); + let v = SeqView::new(&p); + let rc = naive_revcomp(&src); + for f in 0..3u8 { + let frame = Frame::new(f).unwrap(); + if n >= f as usize { + assert_eq!( + translate(v, frame).collect::>(), + naive_translate(&src, f as usize), + "fwd n={n} f={f}" + ); + assert_eq!( + translate(v.reverse_complement(), frame).collect::>(), + naive_translate(&rc, f as usize), + "rev n={n} f={f}" + ); + } + } + } +} + +#[test] +fn known_fixtures_translate() { + // Human preproinsulin (INS) CDS, first 10 codons: MALWMRLLPL + let ins = encode(b"ATGGCCCTGTGGATGCGCCTCCTGCCCCTG").unwrap(); + assert_eq!( + translate(SeqView::new(&ins), Frame::ZERO).collect::>(), + b"MALWMRLLPL" + ); + let t = encode(b"ATGGCCATTGTAATGGGCCGCTGAAAGGGTGCCCGATAG").unwrap(); + assert_eq!( + translate(SeqView::new(&t), Frame::ZERO).collect::>(), + b"MAIVMGR*KGAR*" + ); +} + +#[test] +fn ambiguity_translates_only_when_determinate() { + let cases: &[(&[u8], &[u8])] = &[ + (b"GCN", b"A"), // fourfold degenerate site + (b"GAR", b"E"), // GAA, GAG + (b"GAN", b"X"), // D or E + (b"NNN", b"X"), + (b"TRA", b"X"), // TAA stop or TGA stop? both '*' -> determinate stop + ]; + for &(s, want) in &cases[..4] { + let p = encode(s).unwrap(); + assert_eq!( + translate(SeqView::new(&p), Frame::ZERO).collect::>(), + want, + "{}", + String::from_utf8_lossy(s) + ); + } + // TRA = TAA | TGA: both stop -> '*', not 'X' + let p = encode(cases[4].0).unwrap(); + assert_eq!( + translate(SeqView::new(&p), Frame::ZERO).collect::>(), + b"*" + ); +} + +#[test] +fn inverted_intervals_cannot_be_constructed() { + // Fields are private: `Interval::new` is the only public constructor. + assert!(Interval::new(5, 3).is_none()); + let empty = Interval::new(4, 4).unwrap(); + assert!(empty.is_empty()); + assert_eq!(empty.len(), 0); + let p = encode(b"ACGTACGT").unwrap(); + let v = SeqView::of(&p, Interval::new(2, 6).unwrap(), Strand::Reverse).unwrap(); + assert_eq!(v.len(), 4); + assert_eq!(v.sub(Interval::new(0, 4).unwrap()).unwrap().len(), 4); +} diff --git a/docs/DISCOVERY-MAP.md b/docs/DISCOVERY-MAP.md index 1de32756..97fa8968 100644 --- a/docs/DISCOVERY-MAP.md +++ b/docs/DISCOVERY-MAP.md @@ -253,6 +253,15 @@ two halves of a cell. ADR‑026 names the cascade that ties them. | **D‑VALFACET** | **value‑slab homogeneous closure = the contained 16‑byte facet** `facet_classid(4)+helix‑place(6)+CAM‑PQ(6)` — identity⊥search⊥schema; the value‑side restatement of canon (key path = 6‑byte CAM‑PQ **address** D‑TILE256; this facet's CAM‑PQ = the **content/search** code; helix = place/residue D‑PHASE); lance‑graph value‑tenant harvest CONFIRMS — 9/10 slab tenants don't homogenize (KEEP/DEFER, only HelixResidue matches) + the slab↔parallel‑MailboxSoA two‑world seam | G (harvest facts) / H (facet closure, gated F‑1+F‑code) | EPIPHANY | `.claude/board/EPIPHANIES.md` E‑VALUE‑SLAB‑FACET; lance‑graph `soa-value-tenant-migration-v1-harvest.md` | D‑TILE256, D‑PHASE, D‑KEYKV, D‑CANON‑GUID | | **D‑ENVPARSE** | **classid defines V2/V3 + value‑schema + edge‑codec per file+consumer; ONE reusable envelope parser reads `classid → registry → parse`** (operator `0x1007` — a leading‑`1` generation marker before the domain self‑identifies new‑gen envelopes; versioning in the classid/schema‑pointer, never a GUID‑tail nibble) — no per‑file format byte; the single classid‑late‑bound read path that also reconciles the two‑world seam; the registry entry gains a **`tail_variant` (V2/V3)** axis beside `ReadMode {value_schema, edge_codec}`; same shape as D‑ACTIONHANDLER‑RESOLVER (renderer over the classid keyspace) | G (pieces: classid_read_mode / ClassView / new_v2 / cascade_key‑V3 / node_rows_from_le_bytes) / H (the composed parser, to‑wire) | EPIPHANY→to‑wire | `.claude/board/EPIPHANIES.md` E‑CLASSID‑ENVELOPE‑PARSER; lance‑graph `canonical_node.rs` | D‑VALFACET, D‑IDENTITY‑PIN, D‑KEYKV, D‑TILE256 | +### 2.11 OGAR-GENOM (2026‑10‑04) — packed genomics feasibility; full text `docs/OGAR-GENOM-CAPSTONE.md` + +| ID | Shape | Grade | Status | Home | Deps | +|---|---|:--:|:--:|---|---| +| D‑GENOM‑DNA2 | resident sequence = DNA2 (2 b/base) + sparse ambiguity/soft‑mask run sidecars; V4 `BaseSet` = query/answer algebra, not storage (chr21: 52 N runs, 0 other IUPAC) | G (measured) | CODED | `crates/ogar-genom` | `canonical_node` out‑of‑line genome doctrine | +| D‑GENOM‑VIEW | sub‑region / strand / frame = views (0 B moved); translation = allocation‑free fold; views must be word‑granular to compete with materialization (4.0 vs 42.8 ms) | G (measured) | CODED (per‑base) / IDEA (word kernel) | `crates/ogar-genom` | `ndarray::simd` for kernels | +| D‑GENOM‑LINEAGE | ancestry ≠ storage version: two‑parent descent needs interval‑labelled edges (tree sequence / ARG); `LanceVersion` and `ScenarioBranch` are single‑parent — H7 falsified | H | EPIPHANY | capstone §11 | — | +| D‑GENOM‑FITCH | Fitch parsimony state sets ARE V4 `BaseSet`s (∩ else ∪); candidate missing‑link reconstruction as lane‑parallel nibble algebra | H | IDEA | capstone §12 | D‑GENOM‑DNA2 | + --- ## 3. The materialization pipeline — what's ready, what's blocked diff --git a/docs/OGAR-GENOM-CAPSTONE.md b/docs/OGAR-GENOM-CAPSTONE.md new file mode 100644 index 00000000..b268bae3 --- /dev/null +++ b/docs/OGAR-GENOM-CAPSTONE.md @@ -0,0 +1,759 @@ +# OGAR-GENOM — packed genomics, expression folds, evolutionary replay + +> **Status:** architecture and feasibility capstone (2026-10-04), plus a first +> proof crate, `crates/ogar-genom`. Measured where it says "measured"; +> everything else is labelled. No classid is minted here: the Genetics domain +> `0x0E` holds zero concepts by design (`ogar-vocab` lib.rs:1383-1389), and +> minting stays with the operator. +> +> **The question this answers:** does the existing substrate support +> +> ``` +> genotype ─fold under environment→ phenotype/fitness ─select→ sparse write → next generation +> ``` +> +> without copying genomes to look at them? **Short answer:** the *expression* +> half (`resident sequence → coordinates + masks → folds`) holds and is +> measured. Two of the shortcuts in the hypothesis do not hold: +> +> - The *heredity* half does not work as "per-individual delta streams on a +> version line". Version lines are linear, and delta streams duplicate +> shared ancestry. +> - The model that survives for ancestry is the **tree sequence / ancestral +> recombination graph**: interval-labelled parent→child edges. It is prior +> art, and the plan adopts it rather than reinventing it. + +--- + +## 0. What bytes move — the rule this document is judged by + +Each operation is classified as one of four kinds: + +| class | meaning | +|---|---| +| **VIEW** | a borrow plus coordinates; zero bytes of sequence move | +| **FOLD** | streams over resident bytes and emits a small result (count, residue stream, mask) | +| **MATERIALIZE** | writes a derived object proportional to its length; must be named `materialize*`/`decode` | +| **WRITE** | durable state change (new reference, new variant row, new lineage edge) | + +--- + +## 1. Inventory — what already exists (reuse, do not re-mint) + +Each entry was read from source. "Zero hits" lines name the closed search space. + +### 1.1 ndarray (`AdaWorldAPI/ndarray`, `src/simd.rs` facade) + +- `U8x64::{permute_bytes, shuffle_bytes, cmpeq_mask, shl_epi16, saturating_add, movemask, popcnt}`. + - `permute_bytes` is `vpermb` when VBMI is present. It is a full 64-entry + byte LUT in one instruction, which is the exact shape of `Codon6 → amino acid`. + - **Gotcha:** it is `#[inline]` and does a `simd_caps()` check on every call, + so a hot loop pays one branch per 64 codons. + - `shuffle_bytes` (`vpshufb`) is a 16-entry LUT per 128-bit lane, which is + the exact shape of the V4 nibble algebra. +- `simd_masking_ops::{eq_u8_to_mask, mask_and/or/xor/andnot, mask_ternlog*, mask_set_range, masked_group_*}`. +- `popcount_batch_u64`. + +### 1.2 lance-graph + +- **`lance-graph-mask-risc`** (engine side). Masks are `u64` bit-planes over + rows. + - `ir::Program{ops: Vec, terminal}` with `MaskOp::{Pred, And, Or, + Xor, AndNot, Not, Ternlog, Gather}`. + - `Pred::Range{lo,hi}` covers row-index ranges. + - Terminals include `Count`, `Any`, `Keep` and the masked/grouped reductions. + - "Fold" here means a reduction terminal that never materializes the mask + (`FusedFold`, `Tern2Fold`, tiled `TILE_WORDS=256`). + - **This is the population-fold engine** for genotype bitplanes (§10). + - It has no notion of value intervals or coordinate transforms. +- **Contract `class_view.rs`.** `FieldMask(u64)` is presence-only, at most 64 + fields; `WideFieldMask` widens it. `trait ClassView` covers field selection + and byte reading. + - **It does not describe coordinate transforms.** + - So `GenomeView` is *not* a ClassView: a ClassView selects fields of a + row, and a genome view remaps positions in one sequence. These are + different signatures, and merging them would distort both (§5). +- **Contract `alpha.rs::AlphaMask`.** A bitmap over base ordinals, with + `materialize_ordinals` as its one named materializer. It is the shape for + "which positions" masks. +- **Contract `temporal_pov.rs`.** `LanceVersion = u64`, `VersionRange`, and + `TemporalPov` form a **linear** version line. + - `scenario.rs::ScenarioBranch` has **one** parent (`forked_from: u64`). + - There is no two-parent merge anywhere in the repo. + - **Consequence: H7 is falsified** (§11). +- **Contract `canonical_node.rs`.** + - `NodeRow` = key 16 | edges 16 | value 480. Its doc already says that bulk + raw data — *"a ~3.2 Gbp genome"* — is **a separate Lance table addressed + by key/classid, never inlined**. + - `CLASSID_CPIC_V3 = 0x0E01_1000`. Its doc rules that the V3 basins are + **genomic mereology** (genome → chromosome → region → locus → gene as + HHTL coordinates), not labels. + - **This is H6, already decided upstream.** +- **Counterfactuals.** + - `counterfactual.rs` keeps counterfactuals in their own lane, never as + observed SPO. + - `planner/dismech_counterfactual.rs::counterfactual_replay` cuts one edge + and replays. + - The cognitive `world/counterfactual.rs` explicitly says **"NOT + do-calculus"**. + - There is no general SCM. +- **Provenance.** + - `causal_audit.rs::SupportBasis` has `TextAttested`, `DirectlyObserved`, + `InterventionBacked`, `SimulationOnly`, … with `EvidenceSourceId`, + `SupportReceipt` and `SupportLedger`. + - `dismech_evidence.rs::EvidenceSource` has `HumanClinical`, + `ModelOrganism`, `InVitro`, `Computational`. + - The INDRA harvest (`.claude/harvest/indra-reference-wiring.md`) records + these as *shipped types, unwired*. +- **Prior genetics synthesis** (`.claude/handovers/2026-06-16-genetic-research-headstone-exploration.md`, + `docs/GENETIC_RESEARCH_VIA_STACK.md`): synthesis only, no code. + - It sets standing invariants: no `ValueSchema::Genetic`; counterfactuals in + their own lane; htslib stays upstream; research tooling, not diagnostics. + - Its proposed `adapter-genetics-experimental` crate does not exist. + - This document keeps all of those invariants. + +### 1.3 OGAR + +- **`ogar-obo`.** `parse_obo`, `bake`, `NsRegistry`/`NsSpec` (namespaces as + data), `Crosswalk`, `Predicate` edge byte, `spine::SpineLens`, + `reason::saturate`. + - Baked today: MONDO, HPO, UBERON, PATO, RO. + - `META_STUDY_SPINE` (BFO, IAO, **OBI**, OBCS, SEPIO, **ECO**, COB, FBbi) is + registered but bake-gated. +- **`ogar-ro`.** The `RELATIONS` palette has `IS_A`, `PART_OF` + (`BFO:0000050`), `REGULATES` (`RO:0002211`), `HAS_PHENOTYPE` + (`RO:0002200`), … + - **Absent:** transcribed-from / gene-product / translated-from relations. + These are candidates from RO, to be checked against an RO release when + minted. +- **`ogar-vocab`.** `ConceptDomain::Genetics = 0x0E`, with **zero** concepts + (a test asserts the count is 0). `0x96` BiologicalProcess and `0x97` + Substance are reserved homes for GO and ChEBI. +- **`ogar-cpic`.** A gene → phenotype-axis → drug recommendation table (7 + rows). No alleles, no genotypes, by design. +- **`ogar-dismech`.** Contains a `VARIANT_OF` predicate (`0xA2`). +- **`ogar-dir-sim/provenance.rs`.** `Origin{Observed, Simulated{rule, evidence}}`. + +### 1.4 MedCare-rs (private) + +- CPIC wiring: `medcare-cohorts` `Pgx`, `AXIS_PGX`. +- `medcare-dismech` holds Genetic nodes with raw HGNC and GO CURIEs, never + addresses. +- GO is unbaked (slot `0x96`); ChEBI is unbaked (slot `0x97`). +- SO, PRO, HGNC, ClinVar and dbSNP are docs-only or absent. +- The doc layer records the chain `SO → VARIO → HGNC → HP → MONDO` as + architecture only. +- Commitments #9 (no sourcing or licensing talk in public repos) and #10 + (types exist only before the bake) **bind this design**. Nothing in this + document discusses corpus terms. + +### 1.5 Zero hits — closed search space + +No type named `GenomeId`, `VariantId`, `GeneId`, `SequenceId`, `LineageId`, +`PopulationId`, `Strand`, `Allele*`, `Haplotype*`, `Genotype*` or +`GenomicInterval` exists in lance-graph, OGAR or MedCare-rs Rust (non-`target/`). +The same space has no FASTA, VCF, GFF, BAM, codon or nucleotide code. +(`indra` is the upstream Python INDRA; its `MutCondition.to_hgvs` is protein-level only.) + +**Do not overload:** + +- `NodeGuid` / `ClassId` (classes, not positions); +- `CausalEdge64` (causal claims, not ancestry); +- HHTL `NiblePath` (a tree; one parent per node); +- `LanceVersion` (storage time, not biological time). + +--- + +## 2. Real genomics requirements that break the simple analogy + +1. **Coordinates are not stable across individuals.** An insertion upstream + shifts every downstream position, so "patch the bytes" is not a variant + model. Two coordinate spaces are needed: + - **reference** coordinates (stable, shared); + - **haplotype** coordinates (per individual, derived); + - plus a liftover map between them, which is piecewise-affine over indel + breakpoints. +2. **Structural variants are not local.** Inversion, translocation and + duplication rearrange intervals across contigs. In an SV-dense region the + faithful representation is a sequence *graph* (GFA, variation graphs). + Even then it is a graph of **segments**, never of bases. +3. **Recombination writes ancestry, not sequence.** A crossover produces no + new bases. It changes *which parent each interval descends from*. +4. **Two parents.** Sexual reproduction is a DAG over intervals: an ancestral + recombination graph (ARG). A linear version chain cannot hold it. +5. **Ploidy and phase.** Humans are diploid. A genotype at a site is two + alleles, possibly unphased. +6. **Splicing is ordered and strand-specific.** Exons are concatenated in + transcript order. On the minus strand that is *descending* genomic order, + and CDS phase crosses exon junctions. +7. **Expression is not a function of sequence alone.** It depends on + chromatin, methylation, cell type, environment and time. Sequence identity + and regulatory state are separate axes. Nothing epigenetic goes in the + nucleotide bits. +8. **Ambiguity has three readings.** Matching a motif against `N` is + *possible*, not *definite*. Measured on chr21: + - `TATAWAWR` definite hits: **30,430**; + - "N matches anything" hits: **6,651,513**. + + An API that collapses the two produces garbage. `BaseSet` keeps them + apart. + +--- + +## 3. V4 vs DNA2 — measured, not chosen by elegance + +**Data.** GRCh38 chr21 (UCSC soft-masked FASTA; the Ensembl copy agrees on +counts), 46,709,983 bases. + +| measurement | value | +|---|---| +| `N` | 6,621,364 bases (14.2%) in **52 runs** (largest 5.01 Mbp) | +| non-`N` IUPAC symbols (`R`,`Y`,…) | **0** | +| soft-masked (lower-case) | 20,718,742 bases in **48,677 runs** | +| u8 | 46.7 MB | +| V4 everywhere | 23.4 MB (4 b/base) | +| **DNA2** | **11.68 MB (2.000 b/base)** | +| sidecars, 8 B/run (probe) | 0.39 MB (0.067 b/base) | +| sidecars, 16 B/run (crate: `u64` intervals) | ~0.78 MB (0.13 b/base) | +| `N`-only sidecar | **416 B** | +| encode u8→DNA2 | 14.1 ms (3.3 GB/s) | +| exact round trip (case + `N` restored) | OK | + +**Break-even.** With 16-byte runs, an ambiguity sidecar beats V4-everywhere +while it holds fewer than ~1 run per 64 bases. References sit five orders of +magnitude below that. Consensus or ancient-DNA sequences with dense ambiguity +could approach it. That is the one case where V4 as resident storage could +win, and it is not measured here. + +**Verdict.** + +- DNA2 is resident. +- Ambiguity and soft-mask live in sparse interval sidecars. +- V4 is the **query and answer algebra**: query masks, possible-base sets, + and Fitch state sets (§12). + +V4 is not storage. + +**Prior art, honestly.** This is UCSC **`.2bit`**'s design: 2 bits/base plus +N blocks plus mask blocks, with `T=0 C=1 A=2 G=3` there. The ACGT order chosen +here makes complement `3 − b`. Import and export should be byte-compatible +adapters, and this document claims no novelty for the encoding. + +--- + +## 4. Codon ISA — measured + +Translation is NCBI table 1, frame 0, chr21, single core. The machine has +AVX-512 VBMI. The probe uses the `ndarray::simd` facade only. + +| path | time | per codon | +|---|---|---| +| naive u8 (3 table lookups + LUT, push to Vec) | 294 ms | 18.9 ns | +| DNA2 scalar (6-bit extract from a `u128` window + LUT) | 32.8 ms | 2.11 ns | +| DNA2 + `vpermb`: codon at **every** position (all 3 frames at once) | 30.1 ms | 0.64 ns per position = 1.93 ns per frame-codon | + +What this shows: + +- **The LUT is not the bottleneck; codon extraction is.** The 64-entry table + is one `vpermb` per 64 codons. Building the 6-bit index (unpack DNA2, shift, + combine three shifted windows) dominates. +- The SIMD path's real win is that **three frames cost the same as one**: it + computes a codon at every offset and lets a frame be a stride. +- Six-frame translation = this fold plus the same fold over a reverse-complement view. +- Lane and word boundaries are handled by reading two bytes past each + 64-base block. Correctness is pinned against a naive `u8` translator across + lengths 5..4099, all frames, both strands + (`all_frames_both_strands_across_word_boundaries_match_naive`). +- **Ambiguous codons.** A codon translates only if **every expansion agrees** + (`GCN→A`, `GAR→E`, `TRA→*`); otherwise it yields `X`. +- **H5: confirmed with the nuance above.** The table is biology's NCBI + table 1 and stays separate from every R2IL / 0..63 cognitive vocabulary. + +--- + +## 5. The central law — what bytes move, per operation + +Reverse complement on chr21: + +| path | time | bytes written | +|---|---|---| +| u8 materialized | 23–26 ms | 46.7 MB | +| DNA2 materialized (word bit-reverse + NOT + funnel shift) | 2.8 ms | 11.7 MB | +| **DNA2 word-streaming view** (reversed words produced on the fly into a fold) | **4.0 ms** | **0** | +| DNA2 per-base view (`len−1−i` per base) | 42.8 ms | 0 | +| strand-invariant fold (GC via `popcount(((w>>1)^w) & 0x5555_5555_5555_5555)`) | 0.46 ms | 0 | + +What this shows: + +- **A view is only competitive when it is word-granular.** A per-base + coordinate transform is ~10× slower than materializing. +- The first-PR crate ships the per-base path as the *correctness reference* + (`SeqView::base_set`). The word-streaming kernel is follow-up #1. +- Some folds are strand-invariant and need no transform at all. + +| operation | class | notes | +|---|---|---| +| chromosome → region | **VIEW** | `SeqView::sub` | +| reverse complement | **VIEW** | strand flip; fold via word stream | +| reading frame | **VIEW** | an offset | +| region → motif hits | **FOLD** → mask | `AlphaMask`/bitplane output; never a hit list unless materialized | +| CDS → codons → amino acids | **FOLD** | `translate()` is an iterator, allocation-free (pinned by `no_alloc.rs`) | +| gene → exon subset | **VIEW** (scatter list) | ordered interval recipe | +| exons → transcript | **VIEW**, *materialize on demand* | concatenation is a coordinate recipe (§7) | +| transcript → protein sequence | **MATERIALIZE** when stored; **FOLD** when consumed | proteins outlive their transcript in most consumers (structure, alignment) | +| ingest reference | **WRITE** once | DNA2 + sidecars, out-of-line Lance table | +| SNV/indel per haplotype | **WRITE** (sparse) | §6 | +| recombination | **WRITE** of ancestry edges | not sequence (§10–11) | +| phenotype / fitness | **FOLD** | model output; written only if kept as evidence (§14) | +| selection | **FOLD** → mask | AND + popcount, §10 | + +**H2 is partly supported.** Region, strand, frame and translation are views +and folds, measured. Splicing is a view in principle (§7, not built). +Indel-bearing haplotypes need a liftover map, so they are *not* a pure view +of the reference. Downstream consumers of proteins usually need the +materialized sequence. + +--- + +## 6. Variants are sparse writes — and coordinates are the hard part + +- **Density.** A human genome differs from the reference at ~4–5 M sites + (published population-scale figures; not measured here). That is ~0.15% of + 3.1 Gbp, so per-haplotype variants are sparse. **H3: partly supported.** + Large SVs and CNVs are not sparse in bytes. +- **Overlay read cost** (measured on chr21; 1 SNV/kbp; 10k random 1-kb windows): + - with the overlay merged on the fly by `partition_point` + walk: 13.0 ms; + - without: 9.3 ms; + - **+40%**, without rewriting the reference. +- **Coordinate model** (proposed): + - Internal coordinates are 0-based half-open on the **reference** + (`coord::Interval`). VCF's 1-based positions are converted by the + adapter, never inside. + - Variant identity is the **normalized** allele: left-aligned, parsimonious + `(contig, start, end, alt)`. External identifiers are computed by GA4GH + VRS where interop needs one. + - A haplotype view is `reference + sorted overlay`. Its coordinate space is + derived through a piecewise-affine **liftover** built from the indel + breakpoints. This is a small sorted array, O(#indels). + - Liftover is a *view* when consumed and a *materialization* when the + haplotype is stored. + - **SVs** are interval rearrangements over segments: + - inversion = strand flip of a segment view; + - translocation = re-ordered segment recipe; + - duplication = repeated segment reference. + + Past a density threshold the honest form is a segment graph (GFA), + adapted at the boundary and not native here. +- **Compaction.** When an overlay outgrows its reference (diverged lineages, + other species), it is promoted to a new reference: a new WRITE with new + identity. Overlays never chain without bound (see H9). + +--- + +## 7. Transcripts and alternative splicing + +A transcript is `(reference, strand, ordered exon intervals)`. The exon list +is the **coordinate recipe**, typically 2–100 intervals of 16 B each. + +``` +genomic: A B C D E (resident, unchanged) +isoform 1: A B C E recipe = [A,B,C,E] +isoform 2: A C D E recipe = [A,C,D,E] +``` + +- **Extra memory per isoform** = its recipe. Per-isoform memory is constant + in transcript length: ~10 exons × 16 B = 160 B against kilobases of RNA. +- On the minus strand, the recipe lists exons in descending genomic order and + each segment is read as a reverse-complement view. +- Translation over the recipe needs a **cross-junction codon window**: the + codon fold must carry up to two bases across a segment boundary. This is + the one kernel the splice view adds, and it is the *same* carry problem the + SIMD path already solves at 64-base block edges. +- **When materialization is needed:** feeding external tools (aligners, + folding models), exporting FASTA, and repeated random access into one + isoform where the recipe walk dominates. +- This is the place to prove a real advantage, and it is **PR #2**. + +--- + +## 8. Regulation and epigenetic state + +These are separate axes, each its own sidecar or table keyed by reference +coordinates: + +- accessibility (interval masks); +- methylation (per-CpG values, sparse); +- expression (per gene × sample × condition matrix — a dense numeric table, + not a sequence property). + +``` +expression_fold(gene, sample, environment) = + f( sequence view , regulatory masks(sample) , environment ) +``` + +Nothing in this document claims heritable epigenetic transmission. If a +model needs it, it is a separately versioned regulatory-state table with its +own provenance, never bits in the nucleotide words. + +--- + +## 9. Ontology layer — addresses over regions, never bases + +**H6 holds, and it was already decided upstream:** + +- `canonical_node.rs` says the genome is out-of-line and that basins are + genomic mereology. +- OGAR addresses `Chromosome`, `GenomicRegion`, `Gene`, `Exon`, + `Transcript`, `Protein`, `Variant`, `RegulatoryRegion` and `Phenotype` as + *classes*. +- Instances carry coordinates (`contig, Interval, strand`) pointing into the + resident sequence. One graph node per base is never needed. + +**Use standards, do not mint lookalikes:** + +| concept | source | today in the stack | +|---|---|---| +| feature types (gene, exon, transcript, SNV, insertion, …) | Sequence Ontology (SO) | docs only | +| gene identity | HGNC | raw CURIEs only | +| proteins | PRO / UniProt | docs only | +| functions and processes | GO | slot `0x96` reserved, unbaked | +| disease | MONDO | **baked** | +| phenotype | HPO | **baked** | +| relations | RO / BFO (`ogar-ro`) | **baked**; missing transcribed-from, gene-product, translation relations | +| evidence and assay | ECO, OBI | registered, bake-gated | + +- The integration path is the existing one: `NsSpec` rows in `ogar-obo`, the + `Crosswalk` for xrefs, and RO predicates in `ogar-ro`. +- **Every new concept or namespace allocation is an operator mint** + (`0x0E` has zero concepts by ruling). +- Per MedCare-rs commitment #10, relation kinds resolve before the bake. + Nothing here proposes a post-bake type that describes a relation. + +--- + +## 10. Population model — measured + +**Layout.** Genotypes are site-major **bitplanes**: per site, two planes over +individuals (`carries_alt`, `hom_alt`), and dosage = popcount(p0) + popcount(p1). + +| fold | 100k individuals × 2,000 sites | 1M individuals × 200 sites | +|---|---|---| +| allele-frequency (popcount) | 2.48 ms — **1.2 µs/site** | 1.98 ms — **9.9 µs/site** | +| AF among selected parents (mask AND + popcount) | 1.81 ms | 2.10 ms | + +- Both folds run at **20–28 GB/s: memory-bound**. That is ~0.01 ns per + individual-site. +- **H8 confirmed:** selection is a mask; AF-among-selected is AND + popcount. + In production both are `lance-graph-mask-risc` Programs (`And` then `Count`). +- Recombination and mutation are the only writes. + +**Storage per individual, human-scale estimates** (not measured unless marked): + +| representation | per diploid individual | 1,000 | 1,000,000 | +|---|---|---|---| +| full copies, u8 | 6.2 GB | 6.2 TB | 6.2 PB | +| full copies, DNA2 (2 haplotypes) | 1.55 GB | 1.55 TB | 1.55 PB | +| reference + sparse variants (~4.5 M/haplotype × 8 B) | ~72 MB | ~72 GB | ~72 TB | +| site-major bitplanes (dense, 2 bits per individual-site) | — | 0.25 KB/site | 250 KB/site | +| shared ancestry (tree sequence / ARG) | sublinear | literature: whole-chromosome histories for millions of samples in GB-scale files | — | + +- Bitplanes are dense for common variants. Rare variants want carrier lists: + a sorted row-id list, i.e. a sparse mask. The hybrid is what PGEN/BGEN-class + formats do. +- Per-individual sparse deltas are **~21× smaller than copies but still + linear** in population × variants. They repeat every allele each carrier + shares by descent. +- **Shared-ancestry representations scale with the number of distinct + ancestral events, not carriers.** That is the decisive scaling fact for long + runs (H9). + +--- + +## 11. Evolutionary time ≠ storage version + +Four kinds of time stay distinct: + +1. **storage version** — `LanceVersion`; linear; when bytes were written; +2. **generation** — integer, biological; +3. **ancestry** — a DAG over intervals; +4. **sample time** — when an organism was observed or collected. + +**H7 is falsified.** The temporal machinery is linear and `ScenarioBranch` +has one parent. Overloading either for sexual reproduction would be the +`CausalEdge64`-for-everything mistake. + +**What survives — the tree-sequence / ARG table model** (msprime, tskit, +SLiM). Proposed as the lineage layer, adopted, not reinvented: + +``` +nodes (id, time/generation, flags, individual) +edges (left, right, parent, child) ← interval-labelled; 2 parents fall out naturally +sites (position, ancestral_state) +mutations (site, node, derived_state, parent_mutation) +``` + +- A crossover is two edges with complementary intervals and different + parents. A mutation is a row on the node where it arose. +- A genome is **never stored**. It is a *replay*: walk the edges covering an + interval up to the root and apply the mutations on that path. This is + exactly "evolution = fold → select → sparse write → repeat", with the + writes being edges and mutations. +- Simplification (dropping extinct lineages) is the compaction step that + keeps it sublinear. +- Edges are interval-keyed, so they reuse `coord::Interval`. The tables are + ordinary columnar rows and fit Lance natively. + +--- + +## 12. Missing links — constraint-based candidate reconstruction + +This is framed as **candidate reconstruction**, never history. + +There is a genuine mechanism match, graded **[H]**: **Fitch parsimony's +state sets are exactly V4 `BaseSet`s.** + +- Per site, bottom-up: a node's set = intersection of its children's sets if + non-empty, else the union. +- Top-down resolution then picks members. +- Over many sites at once this is lane-parallel `AND` / `OR` / empty-test on + nibbles: the same `vpand` / `vpor` / `cmpeq_mask` shapes measured in §3–4. + +Given genomes A and C plus a topology, candidate intermediates B are the +per-site resolutions consistent with the constraints. + +Kept separate: + +- maximum-likelihood or Bayesian ancestral reconstruction (substitution + models, branch lengths) — statistical, not mask algebra; +- indel-aware reconstruction, which needs alignment first. + +Output is a candidate set with its constraint receipts. It is written, if at +all, as `SupportBasis::SimulationOnly`. + +--- + +## 13. Counterfactual genomics + +A model intervention, "same ancestral state without V", is cheap here: + +- drop one mutation row in the tree sequence, or one overlay entry; +- replay the phenotype or fitness fold. + +This matches the shape of `dismech_counterfactual::counterfactual_replay`: +cut one edge, replay, return a thresholded verdict. + +It is **not** biological causality: + +| | model intervention | experimental intervention | +|---|---|---| +| what is changed | a row in our model | the organism (CRISPR, knockout) | +| provenance | `SimulationOnly` | `InterventionBacked` | +| licenses a causal claim? | no | yes, within its design | + +Counterfactual results stay in their own lane (the standing +`counterfactual.rs` rule). The repo's own counterfactual module disclaims +do-calculus; this design does not re-claim it. + +--- + +## 14. Evidence and provenance + +- A **variant** is not evidence. +- An **observation of a variant in a sample by an assay** is. + +| claim | basis (`SupportBasis`) | source class (`EvidenceSource`) | +|---|---|---| +| paper reports gene X associated with phenotype Y | `TextAttested` | — | +| cohort observes variant V with phenotype Y | `DirectlyObserved` | `HumanClinical` | +| simulation predicts V changes fitness | `SimulationOnly` | `Computational` | +| CRISPR perturbation of V changes Y | `InterventionBacked` | `InVitro` / `ModelOrganism` | + +**H10: partly supported.** The four witness kinds already exist as distinct +variants. The missing record is the **observation row**: `(sample, assay, +variant, call quality, …)` joined to a receipt. The INDRA harvest names the +same gap: evidence types shipped, unwired. Assays map to OBI and evidence +codes to ECO, both registered and bake-gated. + +--- + +## 15. Interop boundary + +| format | role | where | +|---|---|---| +| FASTA / **2bit** | reference identity and content | ingest/export adapter → DNA2 + sidecars (2bit is near byte-compatible) | +| refget / seqcol digests | external sequence identity | carried as data on the reference row (an external standard id — not an internal pin) | +| FASTQ | reads + qualities | boundary only; not resident | +| VCF / BCF | variants + genotypes | adapter → normalized variant table + genotype bitplanes / carrier lists | +| GFF3 / GTF | annotation | adapter → interval tables + transcript recipes | +| BED | intervals | 1:1 with `coord::Interval` (both 0-based half-open) | +| SAM / BAM / CRAM | alignments | out of scope. CRAM is the same law (resident reference + per-read deltas) — prior art, not a target | +| tskit `.trees` | ancestry | lineage-layer import/export | +| GFA | SV-dense graphs | boundary adapter only | + +Parsers come from existing crates (the 2026-06-16 handover named `noodles`). +htslib stays upstream. The internal canonical forms are: + +- DNA2 + sidecars; +- interval tables; +- variant rows; +- bitplanes; +- tree-sequence tables. + +--- + +## 16. Performance — where the time goes + +Per human haploid reference (3.1 Gbp): + +| form | size | +|---|---| +| u8 | 3.1 GB | +| V4 | 1.55 GB | +| DNA2 | 775 MB | +| DNA2 + sidecars | ~780–830 MB | + +Soft-mask runs scale from chr21 to ~3.2 M genome-wide, so 26–52 MB at 8–16 B/run. + +Throughput, single core, chr21-measured rates extrapolated to 3.1 Gbp: + +| fold | rate | bound by | +|---|---|---| +| encode | 3.3 GB/s → ~1 s/genome | branchy per-byte classification | +| strand-invariant composition (popcount) | ~25 GB/s of packed bytes → ~30 ms/genome | **memory** | +| reverse-complement word view | ~3 GB/s of packed bytes | **compute** (bit reversal) | +| translation, DNA2, 3 frames at once | ~0.64 ns per output codon (one per position) → ~2 s per 3-frame genome pass | **compute** (codon extraction) | +| exact 12-mer, u8 `windows ==` | 1.4–1.8 GB/s | memcmp, vectorized | +| exact 12-mer, DNA2 rolling scalar | 0.6–0.7 GB/s | **branch/serial dependency** | +| V4 IUPAC motif over DNA2 (`vpshufb` onehot + AND-table + `cmpeq`) | 1.25–1.4 GB/s | compute | +| AF fold over bitplanes | 20–28 GB/s | **memory** | +| overlay read | +40% vs plain at 1 SNV/kbp | binary-search + branch | + +**Honest negatives:** + +- The scalar rolling-hash k-mer over DNA2 is ~2.5× *slower* than u8 window + compare. Packed k-mer search needs a SIMD shift-and-compare kernel to pay + off. Not built. +- The per-base view path is 10× slower than word-granular. + +Probe source: `crates/ogar-genom/lab/genom_probe.rs`. It is lab code, not +built by the workspace, and its SIMD comes from `ndarray::simd` only. + +--- + +## 17. Falsification table + +| # | hypothesis | verdict | why | +|---|---|---|---| +| H1 | DNA sequence is a good resident immutable population | **CONFIRMED** | exact round trip; 2 b/base; one named materializer; matches 2bit / CRAM practice | +| H2 | gene/transcript/protein interpretation is mostly views + folds | **PARTLY SUPPORTED** | region / strand / frame / translation measured as views/folds; splicing is a view in principle (PR #2); indel haplotypes need liftover; proteins often must materialize | +| H3 | durable genomic change is sparse | **PARTLY SUPPORTED** | SNVs/indels ~0.15%/haplotype; recombination writes ancestry, not sequence; SVs/CNVs not sparse in bytes | +| H4 | DNA2 + sparse ambiguity beats V4 resident | **CONFIRMED** (reference) / **NEEDS BENCHMARK** (dense-ambiguity sequences) | chr21: 52 N runs, 0 other IUPAC → 0.067–0.13 b/base vs +2 b/base; V4 kept as query algebra, exact agreement with scalar on `TATAWAWR` | +| H5 | codon translation benefits from a 6-bit LUT | **CONFIRMED, nuanced** | LUT = one `vpermb`/64 codons; extraction dominates; DNA2 9× faster than naive u8; SIMD gives 3 frames for the price of 1 | +| H6 | OGAR addresses regions, not bases | **CONFIRMED** | already ruled upstream (out-of-line genome; mereology basins); standards (SO, HGNC, GO, RO) supply semantics; mints operator-gated | +| H7 | generations fit the existing temporal infrastructure | **FALSIFIED** | versions linear; `ScenarioBranch` single-parent; no two-parent merge — needs an ARG/tree-sequence lineage layer | +| H8 | selection is mainly masks/folds | **CONFIRMED** (bench) | AF and AF-among-selected = AND + popcount at memory bandwidth (9.9 µs/site at 1M individuals) | +| H9 | lineage delta storage stays efficient under long runs | **FALSIFIED** for per-individual ordered delta streams / **NEEDS BENCHMARK** for tree sequences | deltas grow with carriers × generations and duplicate shared descent; tree sequences scale with distinct events (literature-reported, not measured here) | +| H10 | provenance distinguishes observed / literature / experimental / simulated | **PARTLY SUPPORTED** | `SupportBasis` + `EvidenceSource` carry the distinction; the per-sample observation record is missing and evidence types are unwired | + +--- + +## 18. Minimal architecture + +**Not one mega crate.** Each layer lands where its kind already lives. + +| layer | home | depends on | status | +|---|---|---|---| +| alphabet, sequence, coord, view, translate | `OGAR/crates/ogar-genom` | nothing | **PR #1 (this)** | +| transcript (splice recipe view + cross-junction codon fold) | `ogar-genom::transcript` | — | PR #2 | +| variant (normalized rows, overlay view, liftover) | `ogar-genom::variant` | — | PR #3 | +| word-stream and SIMD kernels (codon@every-pos `vpermb`, V4 `vpshufb`, revcomp word view) | `ogar-genom` behind an `ndarray` dep (`ndarray::simd` facade only) | ndarray | after PR #1–3; cargo-substrate review for the dep | +| resident storage (words + sidecars as an out-of-line table) | lance-graph side, per the `NodeRow` doctrine | Lance | later | +| population folds (bitplanes, selection) | `lance-graph-mask-risc` Programs | — | exists; needs an adapter | +| lineage (tree-sequence tables, replay, simplify) | new crate, after a probe | `ogar-genom::coord` | later | +| phenotype | trait only — a fold over (sequence view, regulatory masks, environment) | — | interface | +| ontology (SO / HGNC / GO namespaces, RO relations) | `ogar-obo` `NsSpec` rows + `ogar-ro` | — | **operator mint gate** | +| evidence (observation rows → `SupportReceipt`) | lance-graph contract | — | later | +| file adapters (2bit / FASTA / VCF / GFF / BED / tskit) | boundary crates; parsers upstream | noodles etc. | later | + +--- + +## 19. PR #1 — what landed + +`crates/ogar-genom` (zero deps, no `unsafe`, no classid mints): + +- `alphabet` — `Base` (DNA2), `BaseSet` (V4 nibble: union, intersection, + subset, complement, IUPAC both ways). +- `sequence` — `PackedSeq`, `encode`/`decode` with **exact** round trip + (case, `N` and all IUPAC runs), `AmbiguityRun`, soft-mask runs, + `base_set(pos)` overlay-aware. `U` is refused rather than silently read as `T`. +- `coord` — `Interval` (0-based half-open), `Strand`, `Frame`. +- `view` — `SeqView`: `sub` and `reverse_complement` move zero bytes; + `materialize_iupac` is the named materializer. +- `translate` — `CODON_TABLE` (NCBI 1, built from the TCAG table and checked + against an independent hand-written ACGT table), `translate()` as an + allocation-free `ExactSizeIterator`, ambiguity resolved only when + determinate. + +**Tests** (11). Each compares against an independent derivation: + +- naive `u8` reverse complement and translation; +- a hand-written codon table; +- the insulin `MALWMRLLPL` fixture; +- a counting global allocator proving views and folds allocate **0** times + (with an anti-vacuity half proving the counter fires on a materialization). + +**Disable-verified red-then-green, 6 of 6:** + +| disable | goes red in | +|---|---| +| identity complement | `v4_algebra_over_all_iupac_letters` | +| wrong table order | 4 tests | +| overlay ignored | `reverse_complement_is_a_view`, ambiguity test | +| reverse-sub arithmetic | `sub_views_compose_on_both_strands` | +| a collecting `base_sets` | `views_and_folds_allocate_nothing` | +| ambiguity-always-resolves | ambiguity test | + +## 20. Deliberately NOT implemented + +- Evolutionary simulator. +- Population or lineage storage. +- Phenotype models. +- Any file parser. +- Any classid, namespace or relation mint. +- SIMD kernels in the shipped crate. +- Lance persistence. +- Graph nodes per base. +- Epigenetic state in sequence bits. +- Any claim of causal truth from a model intervention. +- Any discussion of reference-data sourcing or terms. + +## 21. Architecture + +``` + ┌──────────────────── OGAR ontology (addresses, not bases) ─────────────────┐ + │ SO / HGNC / GO / MONDO / HPO classes · RO relations · operator mints │ + └───────────────┬──────────────────────────────────────────┬───────────────┘ + │ classid + (contig, Interval, strand) │ evidence receipts + ▼ ▼ + FASTA/2bit ─adapter─► WRITE ┌──────────────────────────┐ ┌───────────────────────────┐ + │ resident reference │ VIEW (0 B) │ SupportBasis / Evidence │ + │ DNA2 words + N / softmask │──sub, strand──►│ Observed · Text · Sim · Exp│ + │ sidecars (out-of-line) │ └───────────────────────────┘ + └──────┬──────────┬─────────┘ + GFF ─adapter─► recipes ───┤ │◄── VCF ─adapter─► WRITE sparse variant rows + liftover + ▼ ▼ + VIEW transcript VIEW haplotype = reference ⊕ overlay (MATERIALIZE by name only) + (ordered exons) │ + │ │ + ▼ ▼ + FOLD codon → aa FOLD expression / phenotype (sequence view × regulatory masks × environment) + │ + ▼ + FOLD fitness ─► MASK selection (AND + popcount over genotype bitplanes) + │ + ▼ + WRITE tree-sequence rows: edges(left,right,parent,child) + mutations + (lineage = DAG over intervals; replay = walk edges + apply mutations; simplify = compaction) + │ + └──────────────► next generation (repeat) +```