diff --git a/.claude/board/entries/2026-11-02-codegraph-bin-reader-timing-gate.md b/.claude/board/entries/2026-11-02-codegraph-bin-reader-timing-gate.md new file mode 100644 index 000000000..39082804d --- /dev/null +++ b/.claude/board/entries/2026-11-02-codegraph-bin-reader-timing-gate.md @@ -0,0 +1,17 @@ +# D-CODEGRAPH-0-BIN reader: unchanged refresh 2.58 µs; changed-crate intake 2.5 ms fails the 1 ms gate (2026-11-02) + +**Status:** MEASURED (release, `crates/lance-graph-codegraph/examples/bin_timing.rs`, mask-risc store of 17,155 rows written by `ruff_charon import`, N = 41 fixed in advance, 4 CPUs, load ≈ 0.3). Plan [`d-codegraph-0-binary-lanes-v1.md`](../plans/d-codegraph-0-binary-lanes-v1.md) §5. + +| measurement | median | gate | +|---|---|---| +| unchanged `refresh_crate` (`rows.bin`, 64-byte header read) | 2.58 µs (range 2.53–14.92) | ≤ 20 µs: met | +| changed-crate intake from `rows.bin` | 2.54–2.84 ms | ≤ 1 ms: **NOT met** | +| — `fs::read` / `CrateFile::parse` / `read_bin_as` | 0.16 / 0.35 / 2.04 ms | | +| same crate from `rows.tsv` (baseline) | 5.08–7.41 ms | | + +- The file is 556 KB as `rows.bin` against 3.05 MB as `rows.tsv`. +- **Where the gate fails:** `read_bin_as` spends about 2 ms in `Symbols::intern` (a hash plus an allocation per new name). This is the interner replacement the plan already assigns to chunk B3: `SYM_HASH` → global id, label locators into retained buffers, no intern at open or refresh, falsifiers F4 and F5. +- Verification (`parse`) is 0.35 ms. After B3 it is the next item to look at. +- Chunks B1 and B2 are in lance-graph PR #1929, with the gate reported as open. Merging them before B3 is the operator's decision. + +OPEN: B3 (interner) and the re-run of this gate; Quack counting (F7) behind the codec9-column gap. diff --git a/.claude/board/entries/README.md b/.claude/board/entries/README.md index 70c0197d3..a11d3bcf1 100644 --- a/.claude/board/entries/README.md +++ b/.claude/board/entries/README.md @@ -25,7 +25,7 @@ index row, (3) no duplicate entry id. Checks 1 and 2 are deliberately opposite directions; the stranding this convention prevents shows up in exactly one of them, never both. -551 entries, 2026-08-06 .. 2026-11-02. +552 entries, 2026-08-06 .. 2026-11-02. | date | entry id | finding | file | |---|---|---|---| @@ -36,6 +36,7 @@ exactly one of them, never both. | 2026-11-02 | `D-CODEGRAPH-0` | | [2026-11-02-d-codegraph-0-gap-map.md](2026-11-02-d-codegraph-0-gap-map.md) | | 2026-11-02 | `codegraph-review-operator-decisions` | | [2026-11-02-codegraph-review-operator-decisions.md](2026-11-02-codegraph-review-operator-decisions.md) | | 2026-11-02 | `codegraph-dr3-dt64-ruff-architecture-review` | | [2026-11-02-codegraph-dr3-dt64-ruff-architecture-review.md](2026-11-02-codegraph-dr3-dt64-ruff-architecture-review.md) | +| 2026-11-02 | `D-CODEGRAPH-0-BIN` | | [2026-11-02-codegraph-bin-reader-timing-gate.md](2026-11-02-codegraph-bin-reader-timing-gate.md) | | 2026-11-01 | `D-XLOG-NATIVE-7` | | [2026-11-01-xlog-resident-hypercube-context.md](2026-11-01-xlog-resident-hypercube-context.md) | | 2026-11-01 | `D-XLOG-NATIVE-8` | | [2026-11-01-xlog-behaviour-lens-f100.md](2026-11-01-xlog-behaviour-lens-f100.md) | | 2026-11-01 | `D-DT64-7-5` | | [2026-11-01-maptable-versioned-local-map.md](2026-11-01-maptable-versioned-local-map.md) | diff --git a/crates/lance-graph-codegraph/examples/bin_timing.rs b/crates/lance-graph-codegraph/examples/bin_timing.rs new file mode 100644 index 000000000..5d5917957 --- /dev/null +++ b/crates/lance-graph-codegraph/examples/bin_timing.rs @@ -0,0 +1,100 @@ +//! Timing gate for the binary rows format (plan `d-codegraph-0-binary-lanes-v1` +//! §5): an unchanged `refresh_crate` against a `rows.bin` store, and the intake +//! of one whole crate from `rows.bin` (read, verify, fill lanes) beside the +//! same crate read from `rows.tsv`. N runs each, pre-registered at 41; median +//! and range in microseconds. Run in release. +//! +//! Usage: `bin_timing ` where `` holds `.rows.bin` +//! and `.rows.tsv` (as `ruff_charon import` writes them). + +use std::hint::black_box; +use std::path::PathBuf; +use std::time::Instant; + +use lance_graph_codegraph::artifact::{self, Lanes, Symbols}; +use lance_graph_codegraph::artifact_bin::CrateFile; +use lance_graph_codegraph::Codegraph; + +const N: usize = 41; + +fn stats(name: &str, mut ns: Vec) { + ns.sort_unstable(); + let us = |v: u128| v as f64 / 1000.0; + println!( + "{name}: median {:.2} us, min {:.2} us, max {:.2} us (N={})", + us(ns[ns.len() / 2]), + us(ns[0]), + us(ns[ns.len() - 1]), + ns.len() + ); +} + +fn main() { + let mut args = std::env::args().skip(1); + let dir = PathBuf::from(args.next().expect("usage: bin_timing ")); + let krate = args.next().expect("usage: bin_timing "); + let bin = dir.join(format!("{krate}.rows.bin")); + let tsv = dir.join(format!("{krate}.rows.tsv")); + + let mut g = Codegraph::open(&dir).expect("store opens"); + let rows = { + let file = CrateFile::parse(std::fs::read(&bin).expect("bin")).expect("parse"); + file.n_rows() + }; + println!("crate {krate}: {rows} rows"); + + let mut unchanged = Vec::with_capacity(N); + for _ in 0..N { + let t = Instant::now(); + let s = g.refresh_crate(&dir, &krate).expect("refresh"); + unchanged.push(t.elapsed().as_nanos()); + black_box(s); + } + stats("refresh_crate (unchanged, rows.bin)", unchanged); + + let mut bin_intake = Vec::with_capacity(N); + for _ in 0..N { + let t = Instant::now(); + let file = CrateFile::parse(std::fs::read(&bin).expect("bin")).expect("parse"); + let mut syms = Symbols::default(); + let mut lanes = Lanes::default(); + artifact::read_bin_as(&file, &krate, &mut syms, &mut lanes).expect("lanes"); + bin_intake.push(t.elapsed().as_nanos()); + black_box(lanes.len()); + } + stats( + "changed-crate intake (rows.bin: read+verify+lanes)", + bin_intake, + ); + + let (mut r, mut v, mut l) = (Vec::new(), Vec::new(), Vec::new()); + for _ in 0..N { + let t = Instant::now(); + let buf = std::fs::read(&bin).expect("bin"); + r.push(t.elapsed().as_nanos()); + let t = Instant::now(); + let file = CrateFile::parse(buf).expect("parse"); + v.push(t.elapsed().as_nanos()); + let t = Instant::now(); + let mut syms = Symbols::default(); + let mut lanes = Lanes::default(); + artifact::read_bin_as(&file, &krate, &mut syms, &mut lanes).expect("lanes"); + l.push(t.elapsed().as_nanos()); + black_box(lanes.len()); + } + stats(" of which fs::read", r); + stats(" of which CrateFile::parse (verify)", v); + stats(" of which read_bin_as (intern + lanes)", l); + + let mut tsv_intake = Vec::with_capacity(N); + for _ in 0..N { + let t = Instant::now(); + let text = std::fs::read_to_string(&tsv).expect("tsv"); + let mut syms = Symbols::default(); + let mut lanes = Lanes::default(); + artifact::read_text_as(&text, &krate, &mut syms, &mut lanes).expect("lanes"); + tsv_intake.push(t.elapsed().as_nanos()); + black_box(lanes.len()); + } + stats("baseline intake (rows.tsv: read+parse+lanes)", tsv_intake); +} diff --git a/crates/lance-graph-codegraph/src/artifact.rs b/crates/lance-graph-codegraph/src/artifact.rs index 31fa3f882..ce5c3604b 100644 --- a/crates/lance-graph-codegraph/src/artifact.rs +++ b/crates/lance-graph-codegraph/src/artifact.rs @@ -11,11 +11,12 @@ //! Columns are located by name from the `#schema` line, so a file with //! permuted columns reads identically. -use std::collections::HashMap; +use std::collections::{BTreeMap, HashMap}; use std::fs; use std::hash::{BuildHasherDefault, Hasher}; -use std::path::Path; +use std::path::{Path, PathBuf}; +use crate::artifact_bin::{self, BinRefusal, CrateFile}; use crate::graph::IntakeRefusal; /// The only artifact version this reader accepts. @@ -97,7 +98,9 @@ pub fn pred_code(s: &str) -> u8 { pub fn digest(bytes: &[u8]) -> u64 { let mut h = FastHasher(0xcbf2_9ce4_8422_2325); h.write(bytes); - h.write_u64(bytes.len() as u64); + // Little-endian, never `write_u64`: its default folds native-endian + // bytes, and the ruff writer (`ruff_charon::store_bin::digest`) folds LE. + h.write(&(bytes.len() as u64).to_le_bytes()); h.finish() } @@ -427,6 +430,30 @@ impl Lanes { const SUFFIX: &str = ".rows.tsv"; +/// File-name suffix of the binary artifact, preferred over [`SUFFIX`] when both +/// exist for a crate. +pub const BIN_SUFFIX: &str = ".rows.bin"; + +/// Which file a crate is read from. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum CrateSource { + /// `.rows.bin`. + Bin(PathBuf), + /// `.rows.tsv`. + Tsv(PathBuf), +} + +/// The file `` is read from in `dir`: `.rows.bin` when it +/// exists, otherwise `.rows.tsv` when that exists, otherwise `None`. +pub fn crate_source(dir: &Path, krate: &str) -> Option { + let bin = dir.join(format!("{krate}{BIN_SUFFIX}")); + if bin.is_file() { + return Some(CrateSource::Bin(bin)); + } + let tsv = dir.join(format!("{krate}{SUFFIX}")); + tsv.is_file().then_some(CrateSource::Tsv(tsv)) +} + /// The text of the single file `.rows.tsv` of `dir`. pub fn read_crate_text(dir: &Path, krate: &str) -> Result { fs::read_to_string(dir.join(format!("{krate}{SUFFIX}"))).map_err(IntakeRefusal::Io) @@ -440,34 +467,138 @@ pub fn read_crate(dir: &Path, krate: &str, symbols: &mut Symbols) -> Result Result { - let mut paths = Vec::new(); + // stem -> whether a binary file exists for it + let mut stems: BTreeMap = BTreeMap::new(); for entry in fs::read_dir(dir).map_err(IntakeRefusal::Io)? { - let path = entry.map_err(IntakeRefusal::Io)?.path(); - let is_rows = path - .file_name() - .and_then(|n| n.to_str()) - .is_some_and(|n| n.ends_with(SUFFIX)); - if is_rows { - paths.push(path); + let name = entry.map_err(IntakeRefusal::Io)?.file_name(); + let Some(name) = name.to_str() else { + continue; + }; + if let Some(stem) = name.strip_suffix(BIN_SUFFIX) { + stems.insert(stem.to_owned(), true); + } else if let Some(stem) = name.strip_suffix(SUFFIX) { + stems.entry(stem.to_owned()).or_insert(false); } } - paths.sort(); let mut lanes = Lanes::default(); - for p in &paths { - let text = fs::read_to_string(p).map_err(IntakeRefusal::Io)?; - let stem = p - .file_name() - .and_then(|n| n.to_str()) - .and_then(|n| n.strip_suffix(SUFFIX)) - .unwrap_or(""); - read_text_as(&text, stem, symbols, &mut lanes)?; + for (stem, has_bin) in &stems { + if *has_bin { + let buf = + fs::read(dir.join(format!("{stem}{BIN_SUFFIX}"))).map_err(IntakeRefusal::Io)?; + let file = CrateFile::parse(buf).map_err(IntakeRefusal::Bin)?; + read_bin_as(&file, stem, symbols, &mut lanes)?; + } else { + let text = fs::read_to_string(dir.join(format!("{stem}{SUFFIX}"))) + .map_err(IntakeRefusal::Io)?; + read_text_as(&text, stem, symbols, &mut lanes)?; + } } Ok(lanes) } +/// A symbol id of a verified file, translated into the interner's id space. +fn translate(table: &[u32], id: u32, tag: u32, row: usize) -> Result { + usize::try_from(id) + .ok() + .and_then(|i| table.get(i)) + .copied() + .ok_or(IntakeRefusal::Bin(BinRefusal::IdRange { tag, row })) +} + +/// Append one verified binary file's rows to `lanes`, attributing each to +/// `krate`, and record the file's header digest under `krate`. +/// +/// The file's symbol table is interned once, in table order; rows then carry +/// translated ids, so no text is parsed and each distinct symbol, grade, file +/// and predicate label is interned once per file. Every lane [`read_text_as`] +/// fills is filled here in the same row order. +pub fn read_bin_as( + file: &CrateFile, + krate: &str, + sym: &mut impl Interner, + lanes: &mut Lanes, +) -> Result<(), IntakeRefusal> { + let kid = sym.intern(krate); + let mut table: Vec = Vec::with_capacity(file.n_symbols()); + for i in 0..file.header().n_symbols { + table.push(sym.intern(file.symbol(i))); + } + // predicate codeword -> (P_* code, interned label id for P_OTHER) + let mut preds: HashMap = HashMap::new(); + let mut grades: HashMap<&str, u32> = HashMap::new(); + let mut files: HashMap<&str, u32> = HashMap::new(); + let mut dash: Option = None; + for row in 0..file.n_rows() { + let (code, label_id) = match preds.get(&file.predicate_id(row)) { + Some(&hit) => hit, + None => { + let label = file.predicate(row); + let code = pred_code(label); + let id = if code == P_OTHER { + sym.intern(label) + } else { + NO_ID + }; + preds.insert(file.predicate_id(row), (code, id)); + (code, id) + } + }; + let subject = translate(&table, file.subject(row), artifact_bin::tag::SUBJ, row)?; + let object = translate(&table, file.object(row), artifact_bin::tag::OBJ, row)?; + let may = match file.may_target(row) { + Some(t) => translate(&table, t, artifact_bin::tag::MAYT, row)?, + None => NO_ID, + }; + let grade = file.grade(row); + let grade_id = match grades.get(grade) { + Some(&id) => id, + None => { + let id = sym.intern(grade); + grades.insert(grade, id); + id + } + }; + let file_id = match file.file(row) { + None => match dash { + Some(id) => id, + None => { + let id = sym.intern("-"); + dash = Some(id); + id + } + }, + Some(label) => match files.get(label) { + Some(&id) => id, + None => { + let id = sym.intern(label); + files.insert(label, id); + id + } + }, + }; + let (start, end) = file.span(row).unwrap_or((NO_SPAN, NO_SPAN)); + // Every fallible step is done: from here the row is pushed whole. + if code == P_OTHER { + lanes.other_pred.push((lanes.pred.len() as u32, label_id)); + } + lanes.pred.push(code); + lanes.subject.push(subject); + lanes.object.push(object); + lanes.grade.push(grade_id); + lanes.file.push(file_id); + lanes.start.push(start); + lanes.end.push(end); + lanes.krate.push(kid); + lanes.may_target.push(may); + } + lanes.set_digest(kid, file.header().digest); + Ok(()) +} + fn number(field: &str) -> Option { if field == "-" { Some(NO_SPAN) diff --git a/crates/lance-graph-codegraph/src/artifact_bin.rs b/crates/lance-graph-codegraph/src/artifact_bin.rs new file mode 100644 index 000000000..4ffea827c --- /dev/null +++ b/crates/lance-graph-codegraph/src/artifact_bin.rs @@ -0,0 +1,968 @@ +//! The zero-copy reader and verifier for `.rows.bin` (chunk B1 of +//! `.claude/plans/d-codegraph-0-binary-lanes-v1.md`). +//! +//! The byte layout is owned by the writer, `ruff_charon::store_bin`; this +//! module is a separate implementation that accepts exactly what that writer +//! writes and refuses exactly what its `read` refuses, in the same check +//! order. There is no dependency on ruff. +//! +//! A [`CrateFile`] owns ONE `Vec`. [`CrateFile::parse`] verifies the whole +//! file once (header, length, digest, directory, shapes, every id against its +//! table, every codeword, UTF-8) and records the byte range of each lane; +//! every accessor afterwards is a lens over that buffer: it reads +//! little-endian integers in place and returns borrowed `&str`s. No `String` +//! is built per row. +//! +//! **Validation invariant.** Every accessor assumes `parse` succeeded. After +//! that, the ranges it records are inside the buffer and have the lane +//! lengths the header promises, so the accessors use plain indexing (a row or +//! symbol id out of range is a caller bug and panics). The only slice +//! operations that can fail at parse time use `get` and return a refusal. +//! `&str` accessors re-check UTF-8 over the one label they return (the bytes +//! were validated at parse, so this cannot fail; the fallback is `""`). +//! There is no `unsafe` and no native-endian cast. + +use std::collections::BTreeMap; +use std::fmt; +use std::ops::Range; + +use crate::artifact::digest; + +/// Length of the fixed header. +pub const HEADER_LEN: usize = 64; +/// An absent id, span, owner, dispatch target or function declaration. +pub const NO_ID: u32 = u32::MAX; + +const MAGIC: [u8; 8] = *b"CHRNROWS"; +const MAJOR: u16 = 1; +const MINOR: u16 = 0; +const FLAG_PARTIAL: u32 = 1; +const CODEC9_MAX: usize = 511; +const KIND_MAX: usize = 3; +const DIR_ENTRY_LEN: usize = 16; +const PROV_FIELDS: usize = 7; + +const ENC_RAW: u16 = 0; +const ENC_CODEC9: u16 = 1; +const ENC_PRES: u16 = 2; +const ENC_KV: u16 = 3; + +/// Section tags. Bit 31 means must-understand; every tag here has it set. +pub mod tag { + /// Predicate magnitude lane. + pub const PRED_MAG: u32 = 0x8000_0001; + /// Predicate presence plane. + pub const PRED_PRES: u32 = 0x8000_0002; + /// Predicate codebook. + pub const PRED_CB: u32 = 0x8000_0003; + /// Grade magnitude lane. + pub const GRADE_MAG: u32 = 0x8000_0004; + /// Grade presence plane. + pub const GRADE_PRES: u32 = 0x8000_0005; + /// Grade codebook. + pub const GRADE_CB: u32 = 0x8000_0006; + /// File lane: codec9 magnitude bytes, or raw `u32` when there are more than 511 files. + pub const FILE: u32 = 0x8000_0007; + /// File presence plane, present only when `FILE` is codec9. + pub const FILE_PRES: u32 = 0x8000_0008; + /// File codebook. + pub const FILE_CB: u32 = 0x8000_0009; + /// Subject symbol ids. + pub const SUBJ: u32 = 0x8000_000A; + /// Object symbol ids. + pub const OBJ: u32 = 0x8000_000B; + /// Dispatch-target symbol ids (`MAY_DISPATCH` rows only). + pub const MAYT: u32 = 0x8000_000C; + /// Span starts. + pub const START: u32 = 0x8000_000D; + /// Span ends. + pub const END: u32 = 0x8000_000E; + /// Symbol blob offsets, `n_symbols + 1` entries. + pub const SYM_OFF: u32 = 0x8000_000F; + /// Symbol label bytes. + pub const SYM_BLOB: u32 = 0x8000_0010; + /// Per-symbol digest of the label. + pub const SYM_HASH: u32 = 0x8000_0011; + /// Per-symbol owner id. + pub const SYM_OWNER: u32 = 0x8000_0012; + /// Per-row Charon function declaration index. + pub const FUNDECL: u32 = 0x8000_0013; + /// Object-kind magnitude lane. + pub const OBJ_KIND_MAG: u32 = 0x8000_0014; + /// Object-kind presence plane. + pub const OBJ_KIND_PRES: u32 = 0x8000_0015; + /// Provenance record. + pub const PROV: u32 = 0x8000_0016; + /// Every tag a version-1 reader understands, in ascending order. + pub const ALL: [u32; 22] = [ + PRED_MAG, + PRED_PRES, + PRED_CB, + GRADE_MAG, + GRADE_PRES, + GRADE_CB, + FILE, + FILE_PRES, + FILE_CB, + SUBJ, + OBJ, + MAYT, + START, + END, + SYM_OFF, + SYM_BLOB, + SYM_HASH, + SYM_OWNER, + FUNDECL, + OBJ_KIND_MAG, + OBJ_KIND_PRES, + PROV, + ]; +} + +/// Why a `.rows.bin` file was refused; one variant per refusal. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum BinRefusal { + /// Wrong magic. + Magic, + /// A major version this reader does not know. + Major(u16), + /// A minor version newer than this reader. + Minor(u16), + /// Non-zero reserved header bytes. + Reserved, + /// The file is shorter or longer than its header says. + Truncated { + /// Length recorded in the header (0 when the header itself is cut). + header: u64, + /// Actual byte length. + actual: usize, + }, + /// The body does not match the header digest. + Digest { + /// Digest recorded in the header. + header: u64, + /// Digest of the body. + actual: u64, + }, + /// The section directory does not fit in the file. + Directory, + /// A directory entry that is misaligned, out of order, overlapping, + /// outside the file or duplicated. + Section { + /// The entry's tag. + tag: u32, + }, + /// An unknown section with the must-understand bit. + UnknownRequired { + /// The tag. + tag: u32, + }, + /// A required section is absent. + MissingSection { + /// The tag. + tag: u32, + }, + /// A section whose width, encoding or length is wrong. + LaneLength { + /// The tag. + tag: u32, + }, + /// An id at or past the end of the table it names. + IdRange { + /// The lane. + tag: u32, + /// The row (or symbol) index. + row: usize, + }, + /// A string that is not valid UTF-8, or an offset inside a character. + Utf8 { + /// The section. + tag: u32, + }, + /// A codeword that is not allowed in its lane. + Codeword { + /// The lane. + tag: u32, + /// The row. + row: usize, + }, +} + +impl fmt::Display for BinRefusal { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Magic => write!(f, "bad magic"), + Self::Major(v) => write!(f, "unsupported major version {v}"), + Self::Minor(v) => write!(f, "unsupported minor version {v}"), + Self::Reserved => write!(f, "reserved header bytes are not zero"), + Self::Truncated { header, actual } => { + write!(f, "file is {actual} bytes, header says {header}") + } + Self::Digest { header, actual } => { + write!(f, "digest mismatch: header {header:#x}, body {actual:#x}") + } + Self::Directory => write!(f, "section directory does not fit in the file"), + Self::Section { tag } => write!(f, "bad directory entry for section {tag:#x}"), + Self::UnknownRequired { tag } => write!(f, "unknown required section {tag:#x}"), + Self::MissingSection { tag } => write!(f, "missing section {tag:#x}"), + Self::LaneLength { tag } => write!(f, "section {tag:#x} has the wrong shape"), + Self::IdRange { tag, row } => write!(f, "id out of range in {tag:#x} at {row}"), + Self::Utf8 { tag } => write!(f, "invalid UTF-8 in section {tag:#x}"), + Self::Codeword { tag, row } => write!(f, "bad codeword in {tag:#x} at {row}"), + } + } +} + +impl std::error::Error for BinRefusal {} + +// --------------------------------------------------------------------------- +// header +// --------------------------------------------------------------------------- + +/// The fixed 64-byte header, read without touching the body (the +/// unchanged-refresh path). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Header { + /// Flag bits; bit 0 is "partial". + pub flags: u32, + /// Digest of every byte from offset 32 on. + pub digest: u64, + /// Total file length in bytes. + pub file_len: u64, + /// Row count. + pub n_rows: u32, + /// Symbol count. + pub n_symbols: u32, + /// File-name codebook size. + pub n_files: u32, + /// Directory entry count. + pub n_sections: u32, +} + +fn le16(b: &[u8], o: usize) -> u16 { + b.get(o..o + 2) + .and_then(|s| s.first_chunk::<2>()) + .map_or(0, |c| u16::from_le_bytes(*c)) +} + +fn le32(b: &[u8], o: usize) -> u32 { + b.get(o..o + 4) + .and_then(|s| s.first_chunk::<4>()) + .map_or(0, |c| u32::from_le_bytes(*c)) +} + +fn le64(b: &[u8], o: usize) -> u64 { + b.get(o..o + 8) + .and_then(|s| s.first_chunk::<8>()) + .map_or(0, |c| u64::from_le_bytes(*c)) +} + +fn len_u64(n: usize) -> u64 { + u64::try_from(n).unwrap_or(u64::MAX) +} + +impl Header { + /// Validates magic, major, minor and reserved; never reads past byte 64. + /// + /// # Errors + /// [`BinRefusal::Magic`], [`BinRefusal::Major`], [`BinRefusal::Minor`] or + /// [`BinRefusal::Reserved`], checked in that order. + pub fn parse(bytes: &[u8; HEADER_LEN]) -> Result { + if bytes[0..8] != MAGIC { + return Err(BinRefusal::Magic); + } + let major = le16(bytes, 8); + if major != MAJOR { + return Err(BinRefusal::Major(major)); + } + let minor = le16(bytes, 10); + if minor > MINOR { + return Err(BinRefusal::Minor(minor)); + } + if bytes[48..64].iter().any(|&b| b != 0) { + return Err(BinRefusal::Reserved); + } + Ok(Self { + flags: le32(bytes, 12), + digest: le64(bytes, 16), + file_len: le64(bytes, 24), + n_rows: le32(bytes, 32), + n_symbols: le32(bytes, 36), + n_files: le32(bytes, 40), + n_sections: le32(bytes, 44), + }) + } + + /// `read_exact` of exactly 64 bytes from `r`, then [`Header::parse`]. No + /// body buffer is allocated. + /// + /// # Errors + /// The I/O error of `read_exact` (a file shorter than 64 bytes is + /// `UnexpectedEof`); the inner result carries the header refusal. + pub fn read(r: &mut impl std::io::Read) -> std::io::Result> { + let mut b = [0u8; HEADER_LEN]; + r.read_exact(&mut b)?; + Ok(Self::parse(&b)) + } + + /// Whether the crate description was partial (Charon reported errors). + pub fn partial(&self) -> bool { + self.flags & FLAG_PARTIAL != 0 + } +} + +// --------------------------------------------------------------------------- +// provenance +// --------------------------------------------------------------------------- + +/// One provenance record. `None` is UNKNOWN and is distinct from every value, +/// including the string `"UNKNOWN"`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Prov { + /// Charon version. + pub charon_version: Option, + /// Charon commit. + pub charon_commit: Option, + /// LLBC schema version. + pub llbc_schema_version: Option, + /// Digest of the LLBC file. + pub llbc_digest: Option, + /// rustc commit. + pub rustc_commit: Option, + /// Target configuration. + pub target_config: Option, + /// Revision of the analysed source. + pub source_revision: Option, +} + +// --------------------------------------------------------------------------- +// layout of a verified file +// --------------------------------------------------------------------------- + +type R = Range; + +/// A codec9 id lane: magnitude bytes plus a presence plane (byte ranges). +#[derive(Debug, Clone)] +struct Lane9 { + mag: R, + pres: R, +} + +/// The file lane is codec9 up to 511 files, raw `u32` beyond. +#[derive(Debug, Clone)] +enum FileLane { + Codec9(Lane9), + Raw(R), +} + +/// A string codebook: `n` entries, `n + 1` offsets, then the blob. +#[derive(Debug, Clone)] +struct Book { + n: usize, + offs: R, + blob: usize, +} + +/// Everything a verified file's accessors need, as byte ranges into the buffer. +#[derive(Debug, Clone)] +struct Layout { + pred: Lane9, + grade: Lane9, + kind: Lane9, + file: FileLane, + pred_cb: Book, + grade_cb: Book, + file_cb: Book, + subj: R, + obj: R, + mayt: R, + start: R, + end: R, + fundecl: R, + sym_off: R, + sym_blob: R, + sym_hash: R, + sym_owner: R, + prov: Vec, +} + +struct Sec { + width: u16, + enc: u16, + range: R, +} + +type Secs = BTreeMap; + +fn find(secs: &Secs, tag: u32) -> Result<&Sec, BinRefusal> { + secs.get(&tag).ok_or(BinRefusal::MissingSection { tag }) +} + +fn lane_len(n: usize, width: usize, tag: u32) -> Result { + n.checked_mul(width).ok_or(BinRefusal::LaneLength { tag }) +} + +/// A fixed-shape section: exact element width, encoding and byte length. +fn lane(secs: &Secs, tag: u32, width: u16, enc: u16, len: usize) -> Result { + let s = find(secs, tag)?; + if s.width != width || s.enc != enc || s.range.len() != len { + return Err(BinRefusal::LaneLength { tag }); + } + Ok(s.range.clone()) +} + +fn lane9(secs: &Secs, mag: u32, pres: u32, n: usize) -> Result { + let m = lane(secs, mag, 1, ENC_CODEC9, n)?; + let p = lane(secs, pres, 8, ENC_PRES, lane_len(n.div_ceil(64), 8, pres)?)?; + Ok(Lane9 { mag: m, pres: p }) +} + +fn chunks4(b: &[u8]) -> &[[u8; 4]] { + b.as_chunks::<4>().0 +} + +fn chunks8(b: &[u8]) -> &[[u8; 8]] { + b.as_chunks::<8>().0 +} + +/// Only zero bytes, and fewer than 8 of them, may follow the last field. +fn only_padding(rest: &[u8], tag: u32) -> Result<(), BinRefusal> { + if rest.len() < 8 && rest.iter().all(|&b| b == 0) { + Ok(()) + } else { + Err(BinRefusal::LaneLength { tag }) + } +} + +/// Validate a KV codebook section and record where its offsets and blob are. +fn book(buf: &[u8], tag: u32, sec: &Sec) -> Result { + let bad = BinRefusal::LaneLength { tag }; + let data = &buf[sec.range.clone()]; + if !data.len().is_multiple_of(8) { + return Err(bad); + } + let n = usize::try_from(le32(data, 0)).map_err(|_| bad.clone())?; + let table = n + .checked_add(1) + .and_then(|k| k.checked_mul(4)) + .and_then(|k| k.checked_add(4)) + .filter(|&k| k <= data.len()) + .ok_or_else(|| bad.clone())?; + let offs = chunks4(&data[4..table]) + .iter() + .map(|c| u32::from_le_bytes(*c)) + .collect::>(); + let blob = &data[table..]; + let blob_len = offs + .last() + .and_then(|&o| usize::try_from(o).ok()) + .ok_or_else(|| bad.clone())?; + if offs.first() != Some(&0) || blob_len > blob.len() || offs.windows(2).any(|w| w[0] > w[1]) { + return Err(bad); + } + only_padding(&blob[blob_len..], tag)?; + for w in offs.windows(2) { + let (a, b) = (w[0] as usize, w[1] as usize); + std::str::from_utf8(&blob[a..b]).map_err(|_| BinRefusal::Utf8 { tag })?; + } + Ok(Book { + n, + offs: sec.range.start + 4..sec.range.start + table, + blob: sec.range.start + table, + }) +} + +fn decode_prov(data: &[u8]) -> Result, BinRefusal> { + let tag = tag::PROV; + let bad = BinRefusal::LaneLength { tag }; + if !data.len().is_multiple_of(8) || data.len() < 4 { + return Err(bad); + } + let n = le32(data, 0); + if n == 0 { + return Err(bad); + } + let mut pos = 4usize; + let mut out = Vec::new(); + for _ in 0..n { + let mut f: Vec> = Vec::with_capacity(PROV_FIELDS); + for _ in 0..PROV_FIELDS { + let len = data + .get(pos..pos + 4) + .map(|_| le32(data, pos)) + .ok_or_else(|| bad.clone())?; + pos += 4; + if len == NO_ID { + f.push(None); + continue; + } + let end = usize::try_from(len) + .ok() + .and_then(|l| pos.checked_add(l)) + .filter(|&e| e <= data.len()) + .ok_or_else(|| bad.clone())?; + let s = std::str::from_utf8(&data[pos..end]).map_err(|_| BinRefusal::Utf8 { tag })?; + f.push(Some(s.to_owned())); + pos = end; + } + let mut it = f.into_iter(); + let mut next = || it.next().flatten(); + out.push(Prov { + charon_version: next(), + charon_commit: next(), + llbc_schema_version: next(), + llbc_digest: next(), + rustc_commit: next(), + target_config: next(), + source_revision: next(), + }); + } + only_padding(&data[pos..], tag)?; + Ok(out) +} + +/// Codeword of `row` in a codec9 lane; `buf` is verified. +fn codeword_of(buf: &[u8], l: &Lane9, row: usize) -> usize { + let m = usize::from(buf[l.mag.start + row]); + let words = chunks8(&buf[l.pres.clone()]); + let p = (u64::from_le_bytes(words[row / 64]) >> (row % 64)) & 1; + m | (usize::try_from(p).unwrap_or(0) << 8) +} + +/// Parse the directory of a file whose header, length and digest are verified. +fn directory(buf: &[u8], h: &Header) -> Result { + let to_usize = |v: u32| usize::try_from(v).map_err(|_| BinRefusal::Directory); + let n_sections = to_usize(h.n_sections)?; + let dir_end = n_sections + .checked_mul(DIR_ENTRY_LEN) + .and_then(|d| d.checked_add(HEADER_LEN)) + .filter(|&e| e <= buf.len()) + .ok_or(BinRefusal::Directory)?; + + let mut secs: Secs = BTreeMap::new(); + let mut prev_end = dir_end; + for i in 0..n_sections { + let at = HEADER_LEN + i * DIR_ENTRY_LEN; + let t = le32(buf, at); + let offset = to_usize(le32(buf, at + 8))?; + let len = to_usize(le32(buf, at + 12))?; + let end = offset.checked_add(len).filter(|&e| e <= buf.len()); + let Some(end) = end.filter(|_| offset % 8 == 0 && offset >= prev_end) else { + return Err(BinRefusal::Section { tag: t }); + }; + prev_end = end; + let sec = Sec { + width: le16(buf, at + 4), + enc: le16(buf, at + 6), + range: offset..end, + }; + if secs.insert(t, sec).is_some() { + return Err(BinRefusal::Section { tag: t }); + } + } + for &t in secs.keys() { + if t & 0x8000_0000 != 0 && !tag::ALL.contains(&t) { + return Err(BinRefusal::UnknownRequired { tag: t }); + } + } + Ok(secs) +} + +/// A codebook codeword: 0 is refused (`Codeword`), past the book is `IdRange`. +fn check_cw(c: usize, n: usize, tag: u32, row: usize) -> Result<(), BinRefusal> { + if c == 0 { + Err(BinRefusal::Codeword { tag, row }) + } else if c > n { + Err(BinRefusal::IdRange { tag, row }) + } else { + Ok(()) + } +} + +fn check_sym(id: u32, n_syms: usize, tag: u32, row: usize) -> Result<(), BinRefusal> { + if usize::try_from(id).is_ok_and(|i| i < n_syms) { + Ok(()) + } else { + Err(BinRefusal::IdRange { tag, row }) + } +} + +/// Verify everything after the digest, in the writer-reader's check order. +#[expect(clippy::too_many_lines)] // one pass that validates every lane +fn layout(buf: &[u8], h: &Header) -> Result { + let secs = directory(buf, h)?; + for t in tag::ALL { + if t != tag::FILE_PRES && !secs.contains_key(&t) { + return Err(BinRefusal::MissingSection { tag: t }); + } + } + let n = usize::try_from(h.n_rows).unwrap_or(usize::MAX); + let n_syms = usize::try_from(h.n_symbols).unwrap_or(usize::MAX); + let n_files = usize::try_from(h.n_files).unwrap_or(usize::MAX); + + let pred = lane9(&secs, tag::PRED_MAG, tag::PRED_PRES, n)?; + let grade = lane9(&secs, tag::GRADE_MAG, tag::GRADE_PRES, n)?; + let kind = lane9(&secs, tag::OBJ_KIND_MAG, tag::OBJ_KIND_PRES, n)?; + let pred_cb = book(buf, tag::PRED_CB, find(&secs, tag::PRED_CB)?)?; + let grade_cb = book(buf, tag::GRADE_CB, find(&secs, tag::GRADE_CB)?)?; + let file_cb = book(buf, tag::FILE_CB, find(&secs, tag::FILE_CB)?)?; + for (t, b) in [(tag::PRED_CB, &pred_cb), (tag::GRADE_CB, &grade_cb)] { + if find(&secs, t)?.enc != ENC_KV || b.n > CODEC9_MAX { + return Err(BinRefusal::LaneLength { tag: t }); + } + } + if find(&secs, tag::FILE_CB)?.enc != ENC_KV || file_cb.n != n_files { + return Err(BinRefusal::LaneLength { tag: tag::FILE_CB }); + } + + let file = if n_files <= CODEC9_MAX { + FileLane::Codec9(lane9(&secs, tag::FILE, tag::FILE_PRES, n)?) + } else { + if secs.contains_key(&tag::FILE_PRES) { + return Err(BinRefusal::LaneLength { + tag: tag::FILE_PRES, + }); + } + FileLane::Raw(lane( + &secs, + tag::FILE, + 4, + ENC_RAW, + lane_len(n, 4, tag::FILE)?, + )?) + }; + + let u32_lane = |t: u32| lane(&secs, t, 4, ENC_RAW, lane_len(n, 4, t)?); + let subj = u32_lane(tag::SUBJ)?; + let obj = u32_lane(tag::OBJ)?; + let mayt = u32_lane(tag::MAYT)?; + let start = u32_lane(tag::START)?; + let end = u32_lane(tag::END)?; + let fundecl = u32_lane(tag::FUNDECL)?; + + let off_len = lane_len( + n_syms + .checked_add(1) + .ok_or(BinRefusal::LaneLength { tag: tag::SYM_OFF })?, + 4, + tag::SYM_OFF, + )?; + let sym_off = lane(&secs, tag::SYM_OFF, 4, ENC_RAW, off_len)?; + let blob_sec = find(&secs, tag::SYM_BLOB)?; + if blob_sec.width != 1 || blob_sec.enc != ENC_RAW { + return Err(BinRefusal::LaneLength { tag: tag::SYM_BLOB }); + } + let sym_blob = blob_sec.range.clone(); + let blob_str = std::str::from_utf8(&buf[sym_blob.clone()]) + .map_err(|_| BinRefusal::Utf8 { tag: tag::SYM_BLOB })?; + let offs: Vec = chunks4(&buf[sym_off.clone()]) + .iter() + .map(|c| u32::from_le_bytes(*c)) + .collect(); + let ok_offsets = offs.first() == Some(&0) + && offs.last().and_then(|&o| usize::try_from(o).ok()) == Some(blob_str.len()) + && offs.windows(2).all(|w| w[0] <= w[1]); + if !ok_offsets { + return Err(BinRefusal::LaneLength { tag: tag::SYM_OFF }); + } + for w in offs.windows(2) { + if !blob_str.is_char_boundary(w[0] as usize) || !blob_str.is_char_boundary(w[1] as usize) { + return Err(BinRefusal::Utf8 { tag: tag::SYM_BLOB }); + } + } + let sym_hash = lane( + &secs, + tag::SYM_HASH, + 8, + ENC_RAW, + lane_len(n_syms, 8, tag::SYM_HASH)?, + )?; + let sym_owner = lane( + &secs, + tag::SYM_OWNER, + 4, + ENC_RAW, + lane_len(n_syms, 4, tag::SYM_OWNER)?, + )?; + for (i, c) in chunks4(&buf[sym_owner.clone()]).iter().enumerate() { + let o = u32::from_le_bytes(*c); + if o != NO_ID && usize::try_from(o).is_ok_and(|o| o >= n_syms) { + return Err(BinRefusal::IdRange { + tag: tag::SYM_OWNER, + row: i, + }); + } + } + let prov_sec = find(&secs, tag::PROV)?; + if prov_sec.width != 1 || prov_sec.enc != ENC_KV { + return Err(BinRefusal::LaneLength { tag: tag::PROV }); + } + let prov = decode_prov(&buf[prov_sec.range.clone()])?; + + let at = |r: &R| chunks4(&buf[r.clone()]); + let (subj_v, obj_v, mayt_v) = (at(&subj), at(&obj), at(&mayt)); + let (start_v, end_v) = (at(&start), at(&end)); + for row in 0..n { + let kc = codeword_of(buf, &kind, row); + if kc == 0 || kc > KIND_MAX { + return Err(BinRefusal::Codeword { + tag: tag::OBJ_KIND_MAG, + row, + }); + } + check_cw(codeword_of(buf, &pred, row), pred_cb.n, tag::PRED_MAG, row)?; + check_cw( + codeword_of(buf, &grade, row), + grade_cb.n, + tag::GRADE_MAG, + row, + )?; + check_sym(u32::from_le_bytes(subj_v[row]), n_syms, tag::SUBJ, row)?; + check_sym(u32::from_le_bytes(obj_v[row]), n_syms, tag::OBJ, row)?; + let m = u32::from_le_bytes(mayt_v[row]); + if m != NO_ID { + check_sym(m, n_syms, tag::MAYT, row)?; + } + let has_file = match &file { + FileLane::Codec9(l) => match codeword_of(buf, l, row) { + 0 => false, + c => { + check_cw(c, file_cb.n, tag::FILE, row)?; + true + } + }, + FileLane::Raw(r) => match u32::from_le_bytes(chunks4(&buf[r.clone()])[row]) { + NO_ID => false, + v => { + if usize::try_from(v).is_ok_and(|i| i < file_cb.n) { + true + } else { + return Err(BinRefusal::IdRange { + tag: tag::FILE, + row, + }); + } + } + }, + }; + let (s, e) = ( + u32::from_le_bytes(start_v[row]), + u32::from_le_bytes(end_v[row]), + ); + if has_file { + if s == NO_ID { + return Err(BinRefusal::IdRange { + tag: tag::START, + row, + }); + } + if e == NO_ID { + return Err(BinRefusal::IdRange { tag: tag::END, row }); + } + } else { + if s != NO_ID { + return Err(BinRefusal::IdRange { + tag: tag::START, + row, + }); + } + if e != NO_ID { + return Err(BinRefusal::IdRange { tag: tag::END, row }); + } + } + } + + Ok(Layout { + pred, + grade, + kind, + file, + pred_cb, + grade_cb, + file_cb, + subj, + obj, + mayt, + start, + end, + fundecl, + sym_off, + sym_blob, + sym_hash, + sym_owner, + prov, + }) +} + +// --------------------------------------------------------------------------- +// the verified file +// --------------------------------------------------------------------------- + +/// One crate file, owned as ONE buffer; every accessor is a lens over it (no +/// copies, no per-row `String`). +/// +/// All accessors assume [`CrateFile::parse`] succeeded and index the buffer +/// directly; a row index `>= n_rows()` or a symbol id `>= n_symbols()` panics. +#[derive(Debug, Clone)] +pub struct CrateFile { + buf: Vec, + header: Header, + lay: Layout, +} + +impl CrateFile { + /// Full verification in the ruff reader's check order: header, file + /// length, digest over `[32..]`, directory, required and unknown tags, + /// lane shapes, then every id against its table, every codeword and all + /// UTF-8, once. + /// + /// # Errors + /// The first [`BinRefusal`] found. + pub fn parse(buf: Vec) -> Result { + let Some(h) = buf.first_chunk::() else { + return Err(BinRefusal::Truncated { + header: 0, + actual: buf.len(), + }); + }; + let header = Header::parse(h)?; + if header.file_len != len_u64(buf.len()) { + return Err(BinRefusal::Truncated { + header: header.file_len, + actual: buf.len(), + }); + } + let actual = digest(&buf[32..]); + if actual != header.digest { + return Err(BinRefusal::Digest { + header: header.digest, + actual, + }); + } + let lay = layout(&buf, &header)?; + Ok(Self { buf, header, lay }) + } + + /// The header, as verified. + pub fn header(&self) -> &Header { + &self.header + } + + /// Row count. + pub fn n_rows(&self) -> usize { + self.lay.subj.len() / 4 + } + + /// Symbol count. + pub fn n_symbols(&self) -> usize { + self.lay.sym_hash.len() / 8 + } + + fn u32_at(&self, r: &R, i: usize) -> u32 { + u32::from_le_bytes(chunks4(&self.buf[r.clone()])[i]) + } + + fn book_label(&self, b: &Book, codeword: usize) -> &str { + let i = codeword - 1; + let o = chunks4(&self.buf[b.offs.clone()]); + let (lo, hi) = (u32::from_le_bytes(o[i]), u32::from_le_bytes(o[i + 1])); + let s = b.blob + lo as usize; + let e = b.blob + hi as usize; + std::str::from_utf8(&self.buf[s..e]).unwrap_or("") + } + + /// The label of symbol `id`, a borrowed slice of the symbol blob. + pub fn symbol(&self, id: u32) -> &str { + let i = id as usize; + let lo = self.u32_at(&self.lay.sym_off, i) as usize; + let hi = self.u32_at(&self.lay.sym_off, i + 1) as usize; + let base = self.lay.sym_blob.start; + std::str::from_utf8(&self.buf[base + lo..base + hi]).unwrap_or("") + } + + /// The writer's digest of symbol `id`'s label. + pub fn sym_hash(&self, id: u32) -> u64 { + let r = &self.lay.sym_hash; + u64::from_le_bytes(chunks8(&self.buf[r.clone()])[id as usize]) + } + + /// The owner symbol of `id`; `None` when it has none. + pub fn sym_owner(&self, id: u32) -> Option { + match self.u32_at(&self.lay.sym_owner, id as usize) { + NO_ID => None, + v => Some(v), + } + } + + /// Subject symbol id of `row`. + pub fn subject(&self, row: usize) -> u32 { + self.u32_at(&self.lay.subj, row) + } + + /// Object symbol id of `row`. + pub fn object(&self, row: usize) -> u32 { + self.u32_at(&self.lay.obj, row) + } + + /// Dispatch-target symbol id of a `MAY_DISPATCH` row; `None` otherwise. + pub fn may_target(&self, row: usize) -> Option { + match self.u32_at(&self.lay.mayt, row) { + NO_ID => None, + v => Some(v), + } + } + + /// The codeword (`1..=511`) of `row`'s predicate. + pub fn predicate_id(&self, row: usize) -> u16 { + u16::try_from(codeword_of(&self.buf, &self.lay.pred, row)).unwrap_or(0) + } + + /// The predicate label of `row`, borrowed from the codebook. + pub fn predicate(&self, row: usize) -> &str { + let c = codeword_of(&self.buf, &self.lay.pred, row); + self.book_label(&self.lay.pred_cb, c) + } + + /// The grade label of `row`, borrowed from the codebook. + pub fn grade(&self, row: usize) -> &str { + let c = codeword_of(&self.buf, &self.lay.grade, row); + self.book_label(&self.lay.grade_cb, c) + } + + /// The source file of `row`; `None` when the row has no site. + pub fn file(&self, row: usize) -> Option<&str> { + let idx = match &self.lay.file { + FileLane::Codec9(l) => codeword_of(&self.buf, l, row).checked_sub(1)?, + FileLane::Raw(r) => match self.u32_at(r, row) { + NO_ID => return None, + v => v as usize, + }, + }; + Some(self.book_label(&self.lay.file_cb, idx + 1)) + } + + /// The `(start, end)` span of `row`; `None` when the row has no site. + pub fn span(&self, row: usize) -> Option<(u32, u32)> { + let s = self.u32_at(&self.lay.start, row); + (s != NO_ID).then(|| (s, self.u32_at(&self.lay.end, row))) + } + + /// The Charon function declaration index of `row`; `None` when absent. + pub fn fundecl(&self, row: usize) -> Option { + match self.u32_at(&self.lay.fundecl, row) { + NO_ID => None, + v => Some(v), + } + } + + /// The object kind of `row` (1 = SYMBOL). + pub fn obj_kind(&self, row: usize) -> u16 { + u16::try_from(codeword_of(&self.buf, &self.lay.kind, row)).unwrap_or(0) + } + + /// The provenance records (one per file today). + pub fn prov(&self) -> Vec { + self.lay.prov.clone() + } +} diff --git a/crates/lance-graph-codegraph/src/graph.rs b/crates/lance-graph-codegraph/src/graph.rs index 3329152f5..a10385507 100644 --- a/crates/lance-graph-codegraph/src/graph.rs +++ b/crates/lance-graph-codegraph/src/graph.rs @@ -11,9 +11,10 @@ use lance_graph_xlog::cube::{Cube, CubeRefusal, EdgeOp, RelId, StagedEdges}; use lance_graph_xlog::wide::{WideRows, WideStats}; use crate::artifact::{ - self, Interner, Lanes, StagedNames, SymbolStage, Symbols, NO_SPAN, P_CALLS, P_CRATE, P_DEFINES, - P_IMPLEMENTS, P_MAY, P_PUBLIC, P_REFERENCES, P_SYNTHETIC, P_UNRESOLVED, + self, CrateSource, Interner, Lanes, StagedNames, SymbolStage, Symbols, NO_SPAN, P_CALLS, + P_CRATE, P_DEFINES, P_IMPLEMENTS, P_MAY, P_PUBLIC, P_REFERENCES, P_SYNTHETIC, P_UNRESOLVED, }; +use crate::artifact_bin::{CrateFile, Header}; /// A symbol's position: its interning index, which is also its cube coordinate. pub type Pos = u32; @@ -43,6 +44,8 @@ pub enum IntakeRefusal { Map(MapRefusal), /// The cube was refused. Cube(CubeRefusal), + /// A `.rows.bin` file was refused. + Bin(crate::artifact_bin::BinRefusal), } /// Whether the extractor finished a crate. @@ -524,7 +527,8 @@ pub(crate) struct PreparedRefresh { } impl Codegraph { - /// Read every `*.rows.tsv` in `dir` and build the graph. + /// Read every crate of `dir` (`*.rows.bin`, or `*.rows.tsv` when no + /// binary file exists for that crate) and build the graph. pub fn open(dir: &Path) -> Result { let mut symbols = Symbols::default(); let lanes = artifact::read_dir(dir, &mut symbols)?; @@ -591,7 +595,8 @@ impl Codegraph { }) } - /// Re-read `.rows.tsv` from `dir` and bring the resident graph up + /// Re-read `.rows.bin` (or `.rows.tsv` when no binary file + /// exists) from `dir` and bring the resident graph up /// to date without rebuilding it ("sparse write, merge on read"). /// /// * **Digest skip.** The FNV-1a digest of every crate file is recorded @@ -665,23 +670,57 @@ impl Codegraph { dir: &Path, krate: &str, ) -> Result, RefreshRefusal> { - let text = artifact::read_crate_text(dir, krate).map_err(RefreshRefusal::Intake)?; // An unchanged file is no delta. The digest recorded when the crate // was last read (at open or at a successful refresh) decides, before - // any parsing, so the common "nothing happened" refresh costs one - // read and one pass over the bytes. - let file_digest = artifact::digest(text.as_bytes()); - if let Some(k) = self.symbols.get(krate) { - if self.lanes.digest_of(k) == Some(file_digest) { - return Ok(None); + // any parsing. A binary file carries that digest in its 64-byte + // header, so the common "nothing happened" refresh reads 64 bytes; a + // text file costs one read and one pass over the bytes. A crate first + // read from text and later refreshed from binary has a different + // digest and takes the full path. + let mut syms = SymbolStage::new(&self.symbols); + let mut fresh = Lanes::default(); + match artifact::crate_source(dir, krate) { + Some(CrateSource::Bin(path)) => { + let mut f = std::fs::File::open(&path) + .map_err(|e| RefreshRefusal::Intake(IntakeRefusal::Io(e)))?; + let header = Header::read(&mut f) + .map_err(|e| RefreshRefusal::Intake(IntakeRefusal::Io(e)))? + .map_err(|r| RefreshRefusal::Intake(IntakeRefusal::Bin(r)))?; + if let Some(k) = self.symbols.get(krate) { + if self.lanes.digest_of(k) == Some(header.digest) { + return Ok(None); + } + } + drop(f); + let buf = std::fs::read(&path) + .map_err(|e| RefreshRefusal::Intake(IntakeRefusal::Io(e)))?; + let file = CrateFile::parse(buf) + .map_err(|r| RefreshRefusal::Intake(IntakeRefusal::Bin(r)))?; + artifact::read_bin_as(&file, krate, &mut syms, &mut fresh) + .map_err(RefreshRefusal::Intake)?; + } + Some(CrateSource::Tsv(path)) => { + let text = std::fs::read_to_string(&path) + .map_err(|e| RefreshRefusal::Intake(IntakeRefusal::Io(e)))?; + let file_digest = artifact::digest(text.as_bytes()); + if let Some(k) = self.symbols.get(krate) { + if self.lanes.digest_of(k) == Some(file_digest) { + return Ok(None); + } + } + artifact::read_text_as(&text, krate, &mut syms, &mut fresh) + .map_err(RefreshRefusal::Intake)?; + } + None => { + return Err(RefreshRefusal::Intake(IntakeRefusal::Io( + std::io::Error::new( + std::io::ErrorKind::NotFound, + format!("no {krate}.rows.bin or {krate}.rows.tsv in the directory"), + ), + ))); } } - // ---- prepare: everything below borrows the graph immutably ---- - let mut syms = SymbolStage::new(&self.symbols); - let mut fresh = Lanes::default(); - artifact::read_text_as(&text, krate, &mut syms, &mut fresh) - .map_err(RefreshRefusal::Intake)?; let kid = syms.intern(krate); // Edges of the crate's old rows, before any new position exists. Every diff --git a/crates/lance-graph-codegraph/src/lib.rs b/crates/lance-graph-codegraph/src/lib.rs index 1c20cd76f..4a7144256 100644 --- a/crates/lance-graph-codegraph/src/lib.rs +++ b/crates/lance-graph-codegraph/src/lib.rs @@ -4,6 +4,7 @@ #![forbid(unsafe_code)] pub mod artifact; +pub mod artifact_bin; pub mod diag; pub mod graph; #[cfg(test)] diff --git a/crates/lance-graph-codegraph/tests/bin_intake.rs b/crates/lance-graph-codegraph/tests/bin_intake.rs new file mode 100644 index 000000000..fc8fc0eb6 --- /dev/null +++ b/crates/lance-graph-codegraph/tests/bin_intake.rs @@ -0,0 +1,295 @@ +//! Tests for the zero-copy `.rows.bin` reader (`artifact_bin`), over the +//! golden file copied from the ruff writer. + +use std::io::Read; +use std::path::PathBuf; + +use lance_graph_codegraph::artifact::digest; +use lance_graph_codegraph::artifact_bin::{tag, BinRefusal, CrateFile, Header, HEADER_LEN}; + +fn golden_path(name: &str) -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/data/golden") + .join(name) +} + +fn golden_bin() -> Vec { + std::fs::read(golden_path("cg_lib.rows.bin")).expect("read golden bin") +} + +fn golden_tsv() -> String { + std::fs::read_to_string(golden_path("cg_lib.rows.tsv")).expect("read golden tsv") +} + +/// Recompute the header digest over `[32..]` so later checks are isolated. +fn reseal(mut b: Vec) -> Vec { + let d = digest(&b[32..]); + b[16..24].copy_from_slice(&d.to_le_bytes()); + b +} + +fn rd32(b: &[u8], o: usize) -> u32 { + u32::from_le_bytes(b[o..o + 4].try_into().expect("4 bytes")) +} + +/// `(directory position, tag, offset, len)` of every directory entry. +fn dir(b: &[u8]) -> Vec<(usize, u32, usize, usize)> { + let n = rd32(b, 44) as usize; + (0..n) + .map(|i| { + let at = HEADER_LEN + i * 16; + ( + at, + rd32(b, at), + rd32(b, at + 8) as usize, + rd32(b, at + 12) as usize, + ) + }) + .collect() +} + +fn entry(b: &[u8], t: u32) -> (usize, usize, usize) { + let (at, _, off, len) = dir(b) + .into_iter() + .find(|e| e.1 == t) + .expect("tag in directory"); + (at, off, len) +} + +fn refusal(b: Vec) -> BinRefusal { + CrateFile::parse(b).expect_err("must be refused") +} + +/// Red if the digest folds a different length encoding than the ruff writer's. +#[test] +fn digest_matches_the_ruff_golden_vectors() { + assert_eq!(digest(b""), 0xd0dd_3f66_b426_3ceb); + assert_eq!(digest(b"a"), 0x9fc7_5a5c_9bce_51be); + assert_eq!(digest(b"CHRNROWS digest golden v1"), 0xfe0a_b233_a400_27ce); +} + +/// Red if any accessor disagrees with the TSV the same writer emitted, or if +/// the fixture stops containing a dispatch row or a siteless row. +#[test] +fn the_ruff_golden_parses_and_matches_its_tsv() { + let f = CrateFile::parse(golden_bin()).expect("golden parses"); + let tsv = golden_tsv(); + let lines: Vec<&str> = tsv + .lines() + .filter(|l| !l.starts_with('#') && !l.is_empty()) + .collect(); + assert!(!lines.is_empty()); + assert_eq!(f.n_rows(), lines.len()); + + let mut saw_may = false; + let mut saw_no_site = false; + for (row, line) in lines.iter().enumerate() { + let c: Vec<&str> = line.split('\t').collect(); + assert_eq!(c.len(), 7, "row {row}"); + assert_eq!(f.predicate(row), c[0], "predicate row {row}"); + assert_eq!(f.symbol(f.subject(row)), c[1], "subject row {row}"); + assert_eq!(f.symbol(f.object(row)), c[2], "object row {row}"); + assert_eq!(f.grade(row), c[3], "grade row {row}"); + if c[4] == "-" { + saw_no_site = true; + assert_eq!(f.file(row), None, "file row {row}"); + assert_eq!(f.span(row), None, "span row {row}"); + assert_eq!((c[5], c[6]), ("-", "-")); + } else { + assert_eq!(f.file(row), Some(c[4]), "file row {row}"); + let want = (c[5].parse::().unwrap(), c[6].parse::().unwrap()); + assert_eq!(f.span(row), Some(want), "span row {row}"); + } + if c[0] == "MAY_DISPATCH" { + saw_may = true; + assert!(f.may_target(row).is_some(), "may_target row {row}"); + } else { + assert_eq!(f.may_target(row), None, "may_target row {row}"); + } + assert_eq!(f.obj_kind(row), 1); + assert!(f.predicate_id(row) >= 1); + } + assert!(saw_may, "golden must contain a MAY_DISPATCH row"); + assert!(saw_no_site, "golden must contain a row with no site"); + + // Symbol side: hashes and owners are consistent with the labels. + for id in 0..u32::try_from(f.n_symbols()).unwrap() { + let label = f.symbol(id); + assert_eq!(f.sym_hash(id), digest(label.as_bytes()), "hash {id}"); + match (label.find("::"), f.sym_owner(id)) { + (Some(p), Some(o)) => assert_eq!(f.symbol(o), &label[..p]), + (None, None) => {} + other => panic!("owner mismatch for {label}: {other:?}"), + } + } +} + +struct Counting<'a> { + inner: &'a [u8], + requested: usize, +} + +impl Read for Counting<'_> { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + self.requested += buf.len(); + self.inner.read(buf) + } +} + +/// Red if `Header::read` asks for more than 64 bytes or returns another digest. +#[test] +fn the_header_alone_is_enough_for_the_unchanged_check() { + let bytes = golden_bin(); + let mut r = Counting { + inner: &bytes, + requested: 0, + }; + let h = Header::read(&mut r).expect("io").expect("header"); + assert_eq!(r.requested, HEADER_LEN); + let full = CrateFile::parse(bytes.clone()).expect("parse"); + assert_eq!(h.digest, full.header().digest); + assert_eq!(h, *full.header()); + assert_eq!(h.file_len, bytes.len() as u64); +} + +/// Red if a flipped body byte is not caught by the digest before decoding. +#[test] +fn a_flipped_body_byte_is_a_digest_refusal() { + let mut b = golden_bin(); + let last = b.len() - 9; + b[last] ^= 1; + assert!(matches!(refusal(b), BinRefusal::Digest { .. })); +} + +/// Red if a cut file is accepted or reported as anything but Truncated. +#[test] +fn a_truncated_file_is_refused() { + let mut b = golden_bin(); + b.truncate(b.len() - 8); + assert!(matches!(refusal(b), BinRefusal::Truncated { .. })); + assert!(matches!( + refusal(golden_bin()[..10].to_vec()), + BinRefusal::Truncated { + header: 0, + actual: 10 + } + )); +} + +/// Red if a newer major version is read. +#[test] +fn a_newer_major_is_refused() { + let mut b = golden_bin(); + b[8] += 1; + assert_eq!(refusal(reseal(b)), BinRefusal::Major(2)); +} + +/// Red if a newer minor version is read. +#[test] +fn a_newer_minor_is_refused() { + let mut b = golden_bin(); + b[10] += 1; + assert_eq!(refusal(reseal(b)), BinRefusal::Minor(1)); +} + +/// Red if a non-zero reserved header byte is ignored. +#[test] +fn a_reserved_byte_is_refused() { + let mut b = golden_bin(); + b[50] = 1; + assert_eq!(refusal(reseal(b)), BinRefusal::Reserved); +} + +/// Red if a wrong magic is accepted. +#[test] +fn a_wrong_magic_is_refused() { + let mut b = golden_bin(); + b[0] ^= 0xFF; + assert_eq!(refusal(reseal(b)), BinRefusal::Magic); +} + +/// Red if a subject id equal to `n_symbols` is not an IdRange refusal. +#[test] +fn a_subject_id_past_the_symbols_is_refused() { + let mut b = golden_bin(); + let n_syms = rd32(&b, 36); + let (_, off, _) = entry(&b, tag::SUBJ); + b[off..off + 4].copy_from_slice(&n_syms.to_le_bytes()); + assert_eq!( + refusal(reseal(b)), + BinRefusal::IdRange { + tag: tag::SUBJ, + row: 0 + } + ); +} + +/// Red if a section whose offset passes the end of the file is accepted. +#[test] +fn a_section_past_the_file_end_is_refused() { + let mut b = golden_bin(); + let (at, _, _) = entry(&b, tag::OBJ); + let past = u32::try_from(b.len() + 8).unwrap(); + b[at + 8..at + 12].copy_from_slice(&past.to_le_bytes()); + assert_eq!(refusal(reseal(b)), BinRefusal::Section { tag: tag::OBJ }); +} + +/// Red if an unknown must-understand section is skipped. +#[test] +fn an_unknown_required_tag_is_refused() { + let mut b = golden_bin(); + let (at, _, _) = entry(&b, tag::PROV); + let unknown = 0x8000_00FFu32; + b[at..at + 4].copy_from_slice(&unknown.to_le_bytes()); + assert_eq!( + refusal(reseal(b)), + BinRefusal::UnknownRequired { tag: unknown } + ); +} + +/// Red if a file without a SUBJ section parses. The entry is removed from the +/// directory (the later entries shift up, the count drops by one). +#[test] +fn a_missing_required_section_is_refused() { + let mut b = golden_bin(); + let (at, _, _) = entry(&b, tag::SUBJ); + let n = rd32(&b, 44); + let dir_end = HEADER_LEN + n as usize * 16; + b.copy_within(at + 16..dir_end, at); + // The freed last slot is now dead space between the directory and the + // first section; keep the file length by zeroing it. + for x in &mut b[dir_end - 16..dir_end] { + *x = 0; + } + b[44..48].copy_from_slice(&(n - 1).to_le_bytes()); + assert_eq!( + refusal(reseal(b)), + BinRefusal::MissingSection { tag: tag::SUBJ } + ); +} + +/// Red if the provenance record is not read back as the writer wrote it. +#[test] +fn prov_reads_the_writer_record() { + let f = CrateFile::parse(golden_bin()).expect("parse"); + let p = f.prov(); + assert_eq!(p.len(), 1); + assert!(p[0].charon_version.is_some()); + assert_eq!(p[0].charon_commit, None); +} + +/// Red if a function row loses its declaration index or the CRATE row gains one. +#[test] +fn fundecl_is_present_on_function_rows() { + let f = CrateFile::parse(golden_bin()).expect("parse"); + let mut crate_rows = 0; + for row in 0..f.n_rows() { + if f.predicate(row) == "CRATE" { + crate_rows += 1; + assert_eq!(f.fundecl(row), None, "CRATE row {row}"); + } else { + assert!(f.fundecl(row).is_some(), "row {row}"); + } + } + assert!(crate_rows >= 1, "golden must contain the CRATE row"); +} diff --git a/crates/lance-graph-codegraph/tests/bin_refresh.rs b/crates/lance-graph-codegraph/tests/bin_refresh.rs new file mode 100644 index 000000000..c69677693 --- /dev/null +++ b/crates/lance-graph-codegraph/tests/bin_refresh.rs @@ -0,0 +1,224 @@ +//! Intake and refresh over `*.rows.bin` stores, against the golden file pair +//! (`tests/data/golden/cg_lib.rows.{bin,tsv}`, the same rows in both forms). +//! Ids differ between a bin-read and a TSV-read graph (the interning order +//! differs), so graphs are compared by symbol LABEL, never by id. + +use std::fs; +use std::path::PathBuf; + +use lance_graph_codegraph::artifact_bin::BinRefusal; +use lance_graph_codegraph::{Codegraph, IntakeRefusal, Pos, RefreshRefusal, RefreshStats}; + +const BIN: &str = "cg_lib.rows.bin"; +const TSV: &str = "cg_lib.rows.tsv"; + +fn golden(name: &str) -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/data/golden") + .join(name) +} + +/// A fresh temp dir, unique per call (tests run in parallel). +fn temp_dir(tag: &str) -> PathBuf { + static NEXT: std::sync::atomic::AtomicU32 = std::sync::atomic::AtomicU32::new(0); + let n = NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let dir = std::env::temp_dir().join(format!("cg_bin_refresh_{}_{tag}_{n}", std::process::id())); + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).expect("temp dir"); + dir +} + +/// A store holding only the golden `.rows.bin`. +fn bin_store(tag: &str) -> PathBuf { + let dir = temp_dir(tag); + fs::copy(golden(BIN), dir.join(BIN)).expect("copy bin"); + dir +} + +/// A store holding only the golden `.rows.tsv`. +fn tsv_store(tag: &str) -> PathBuf { + let dir = temp_dir(tag); + fs::copy(golden(TSV), dir.join(TSV)).expect("copy tsv"); + dir +} + +fn names(g: &Codegraph, mask: &[u64]) -> Vec { + let mut v: Vec = (0..mask.len() * 64) + .filter(|&p| mask[p / 64] >> (p % 64) & 1 == 1) + .filter_map(|p| g.symbol(p as Pos).map(str::to_string)) + .collect(); + v.sort(); + v +} + +fn all_names(g: &Codegraph) -> Vec { + let mut v: Vec = (0..g.words() * 64) + .filter_map(|p| g.symbol(p as Pos).map(str::to_string)) + .collect(); + v.sort(); + v +} + +/// One line per symbol with every graded answer as sorted LABELS. +fn answers(g: &mut Codegraph) -> Vec { + let mut out = Vec::new(); + for name in all_names(g) { + let p = g.pos(&name).expect("pos"); + let c = g.callers(p).expect("callers"); + let e = g.callees(p).expect("callees"); + let i = g.impact(p).expect("impact"); + let r = g.references(p); + out.push(format!( + "{name} | callers {:?} {:?} {:?} sites {} | callees {:?} {:?} {:?} sites {} | \ + impact {:?} {:?} {:?} | refs {} {}", + names(g, &c.graded.confirmed), + names(g, &c.graded.possible), + names(g, &c.graded.unknown), + c.sites, + names(g, &e.graded.confirmed), + names(g, &e.graded.possible), + names(g, &e.graded.unknown), + e.sites, + names(g, &i.confirmed), + names(g, &i.possible), + names(g, &i.unknown), + r.confirmed, + r.possible, + )); + } + out.sort(); + out +} + +fn callers_of(g: &mut Codegraph, name: &str) -> Vec { + let p = g.pos(name).expect("pos"); + let c = g.callers(p).expect("callers"); + names(g, &c.graded.confirmed) +} + +const CALLER_ROW: &str = + "CALLS_RESOLVED\tcg_lib::shapes::impl_Shape_for_Square::area\tcg_lib::util::double\t"; + +/// Red if the bin intake and the TSV intake build different graphs: a lane +/// filled wrongly, a symbol translated through the wrong table slot, or a +/// grade/file/predicate mapped to the wrong code (disable: translate subject +/// ids with the object table offset by one). +#[test] +fn bin_and_tsv_open_to_the_same_graph() { + let mut from_bin = Codegraph::open(&bin_store("f1_bin")).expect("open bin"); + let mut from_tsv = Codegraph::open(&tsv_store("f1_tsv")).expect("open tsv"); + + assert_eq!(from_bin.rows(), from_tsv.rows()); + assert_eq!(all_names(&from_bin), all_names(&from_tsv)); + assert!(all_names(&from_bin).len() > 10, "fixture is not trivial"); + for row in 0..from_bin.rows() { + assert_eq!(from_bin.row_predicate(row), from_tsv.row_predicate(row)); + assert_eq!(from_bin.row_subject(row), from_tsv.row_subject(row)); + assert_eq!(from_bin.row_object(row), from_tsv.row_object(row)); + assert_eq!(from_bin.site(row), from_tsv.site(row), "site of row {row}"); + } + assert_eq!( + from_bin.crate_status("cg_lib"), + from_tsv.crate_status("cg_lib") + ); + + // Anti-vacuity: the comparison below must cover a non-empty caller set. + assert_eq!( + callers_of(&mut from_bin, "cg_lib::util::double"), + vec!["cg_lib::shapes::impl_Shape_for_Square::area".to_string()] + ); + + assert_eq!(answers(&mut from_bin), answers(&mut from_tsv)); +} + +/// Red if an unchanged bin refresh reads past the 64-byte header: the body is +/// overwritten (its digest would no longer match) yet the refresh must still +/// answer "no delta" (disable: parse the whole file before comparing digests). +#[test] +fn an_unchanged_bin_refresh_reads_only_the_header() { + let dir = bin_store("f2"); + let mut g = Codegraph::open(&dir).expect("open"); + let before = answers(&mut g); + + let path = dir.join(BIN); + let mut bytes = fs::read(&path).expect("read"); + assert!(bytes.len() > 64); + for b in &mut bytes[64..] { + *b = 0xEE; + } + fs::write(&path, &bytes).expect("write"); + + let stats = g.refresh_crate(&dir, "cg_lib").expect("refresh"); + assert_eq!(stats, RefreshStats::default()); + assert_eq!(answers(&mut g), before); +} + +/// Red if a changed header digest is trusted without verifying the body +/// (disable: skip `CrateFile::parse` in `prepare_refresh`, or compare only +/// the header). Also checks the refusal changed nothing. +#[test] +fn a_changed_bin_body_is_verified() { + let dir = bin_store("f2_twin"); + let mut g = Codegraph::open(&dir).expect("open"); + let before = answers(&mut g); + + let path = dir.join(BIN); + let mut bytes = fs::read(&path).expect("read"); + bytes[16] ^= 0x01; // the header digest now differs from the recorded one + for b in &mut bytes[64..] { + *b = 0xEE; + } + fs::write(&path, &bytes).expect("write"); + + match g.refresh_crate(&dir, "cg_lib") { + Err(RefreshRefusal::Intake(IntakeRefusal::Bin(BinRefusal::Digest { .. }))) => {} + other => panic!("expected a digest refusal, got {other:?}"), + } + assert_eq!(answers(&mut g), before); +} + +/// Red if a directory holding both files reads the TSV (disable: prefer +/// `.rows.tsv` in `read_dir`). The TSV copy lacks one call row, so the two +/// stores answer differently. +#[test] +fn bin_is_preferred_over_tsv() { + let tsv = fs::read_to_string(golden(TSV)).expect("read tsv"); + let kept: Vec<&str> = tsv.lines().filter(|l| !l.starts_with(CALLER_ROW)).collect(); + assert_eq!( + kept.len() + 1, + tsv.lines().count(), + "exactly one row removed" + ); + let reduced = format!("{}\n", kept.join("\n")); + + let both = temp_dir("pref_both"); + fs::copy(golden(BIN), both.join(BIN)).expect("copy bin"); + fs::write(both.join(TSV), &reduced).expect("write reduced tsv"); + + let reduced_only = temp_dir("pref_tsv"); + fs::write(reduced_only.join(TSV), &reduced).expect("write reduced tsv"); + + let mut g_both = Codegraph::open(&both).expect("open both"); + let mut g_bin = Codegraph::open(&bin_store("pref_bin")).expect("open bin"); + let mut g_tsv = Codegraph::open(&reduced_only).expect("open reduced tsv"); + + // The removed row really changes an answer. + assert_eq!(g_tsv.rows() + 1, g_bin.rows()); + assert!(callers_of(&mut g_tsv, "cg_lib::util::double").is_empty()); + assert!(!callers_of(&mut g_bin, "cg_lib::util::double").is_empty()); + assert_ne!(answers(&mut g_tsv), answers(&mut g_bin)); + + // The mixed directory is the bin graph, not the TSV one. + assert_eq!(g_both.rows(), g_bin.rows()); + assert_eq!(answers(&mut g_both), answers(&mut g_bin)); +} + +/// Red if a store with no binary file stops opening (disable: make `read_dir` +/// list only `.rows.bin`). +#[test] +fn a_tsv_only_store_still_opens() { + let g = Codegraph::open(&tsv_store("tsv_only")).expect("open tsv"); + assert!(g.rows() > 0); + assert!(g.words() > 0); + assert!(g.pos("cg_lib::util::double").is_some()); +} diff --git a/crates/lance-graph-codegraph/tests/data/golden/README.md b/crates/lance-graph-codegraph/tests/data/golden/README.md new file mode 100644 index 000000000..7c636f541 --- /dev/null +++ b/crates/lance-graph-codegraph/tests/data/golden/README.md @@ -0,0 +1 @@ +These two files, `cg_lib.rows.bin` and `cg_lib.rows.tsv`, are copied byte-identical from ruff `crates/ruff_charon/tests/data/golden/` (writer commit `6c3d1bf8da`); they are the binary and the TSV form of the same rows, so re-copy both whenever ruff re-blesses them, never one alone. `tests/bin_intake.rs` parses the bin with the zero-copy reader and asserts row for row that it describes the same rows as the TSV. diff --git a/crates/lance-graph-codegraph/tests/data/golden/cg_lib.rows.bin b/crates/lance-graph-codegraph/tests/data/golden/cg_lib.rows.bin new file mode 100644 index 000000000..1e222d01a Binary files /dev/null and b/crates/lance-graph-codegraph/tests/data/golden/cg_lib.rows.bin differ diff --git a/crates/lance-graph-codegraph/tests/data/golden/cg_lib.rows.tsv b/crates/lance-graph-codegraph/tests/data/golden/cg_lib.rows.tsv new file mode 100644 index 000000000..27aa71c8e --- /dev/null +++ b/crates/lance-graph-codegraph/tests/data/golden/cg_lib.rows.tsv @@ -0,0 +1,31 @@ +#version 2 +#schema predicate subject object grade file start end +DEFINES cg_lib::area_of cg_lib::area_of COMPILER cg_lib/src/lib.rs 81 144 +PUBLIC cg_lib::area_of cg_lib::area_of COMPILER cg_lib/src/lib.rs 81 144 +MAY_DISPATCH cg_lib::area_of cg_lib::shapes::Shape::area via Clause COMPILER cg_lib/src/lib.rs 134 142 +DEFINES cg_lib::area_dyn cg_lib::area_dyn COMPILER cg_lib/src/lib.rs 146 208 +PUBLIC cg_lib::area_dyn cg_lib::area_dyn COMPILER cg_lib/src/lib.rs 146 208 +UNRESOLVED_CALL cg_lib::area_dyn dynamic COMPILER cg_lib/src/lib.rs 198 206 +DEFINES cg_lib::macro_use::checked cg_lib::macro_use::checked COMPILER cg_lib/src/macro_use.rs 32 91 +PUBLIC cg_lib::macro_use::checked cg_lib::macro_use::checked COMPILER cg_lib/src/macro_use.rs 32 91 +CALLS_RESOLVED cg_lib::macro_use::checked cg_lib::util::triple COMPILER cg_lib/src/macro_use.rs 73 78 +CALLS_RESOLVED cg_lib::macro_use::checked core::panicking::assert_failed COMPILER cg_lib/src/macro_use.rs 62 82 +DEFINES cg_lib::util::double cg_lib::util::double COMPILER cg_lib/src/util.rs 28 70 +PUBLIC cg_lib::util::double cg_lib::util::double COMPILER cg_lib/src/util.rs 28 70 +DEFINES cg_lib::util::triple cg_lib::util::triple COMPILER cg_lib/src/util.rs 72 114 +PUBLIC cg_lib::util::triple cg_lib::util::triple COMPILER cg_lib/src/util.rs 72 114 +DEFINES cg_lib::util::unit cg_lib::util::unit COMPILER cg_lib/src/util.rs 116 157 +PUBLIC cg_lib::util::unit cg_lib::util::unit COMPILER cg_lib/src/util.rs 116 157 +DEFINES cg_lib::util::private_unused cg_lib::util::private_unused COMPILER cg_lib/src/util.rs 159 195 +DEFINES cg_lib::shapes::impl_Shape_for_Square::area cg_lib::shapes::impl_Shape_for_Square::area COMPILER cg_lib/src/shapes.rs 192 257 +PUBLIC cg_lib::shapes::impl_Shape_for_Square::area cg_lib::shapes::impl_Shape_for_Square::area COMPILER cg_lib/src/shapes.rs 192 257 +IMPLEMENTS cg_lib::shapes::impl_Shape_for_Square::area cg_lib::shapes::Shape::area COMPILER cg_lib/src/shapes.rs 192 257 +CALLS_RESOLVED cg_lib::shapes::impl_Shape_for_Square::area cg_lib::util::double COMPILER cg_lib/src/shapes.rs 224 238 +DEFINES cg_lib::shapes::impl_Shape_for_Rect::area cg_lib::shapes::impl_Shape_for_Rect::area COMPILER cg_lib/src/shapes.rs 287 340 +PUBLIC cg_lib::shapes::impl_Shape_for_Rect::area cg_lib::shapes::impl_Shape_for_Rect::area COMPILER cg_lib/src/shapes.rs 287 340 +IMPLEMENTS cg_lib::shapes::impl_Shape_for_Rect::area cg_lib::shapes::Shape::area COMPILER cg_lib/src/shapes.rs 287 340 +DEFINES cg_lib::shapes::{Square}::side cg_lib::shapes::{Square}::side COMPILER cg_lib/src/shapes.rs 362 410 +PUBLIC cg_lib::shapes::{Square}::side cg_lib::shapes::{Square}::side COMPILER cg_lib/src/shapes.rs 362 410 +DEFINES cg_lib::macro_use::checked::{promoted_const} cg_lib::macro_use::checked::{promoted_const} COMPILER cg_lib/src/macro_use.rs 62 82 +SYNTHETIC cg_lib::macro_use::checked::{promoted_const} cg_lib::macro_use::checked::{promoted_const} COMPILER cg_lib/src/macro_use.rs 62 82 +CRATE cg_lib COMPLETE COMPILER - - -