From 9609971f3eb5c84fb1f85dee04861b96c74da887 Mon Sep 17 00:00:00 2001 From: Torstein Tauno Svendsen Date: Fri, 2 Oct 2026 23:37:15 +0200 Subject: [PATCH] list and info no longer load every grain to count its chunks summarize_store called grain::load on each store's grain to read chunk_count() and discard the rest: the whole file, plus a copy of every filter. list walks the stores one after another, so its peak was about twice the largest grain. grain::coverage returns the grain's size and chunk count from the commit when it still describes the file, otherwise from one buffered walk. It opens nothing for writing, so list stays usable on a store it can only read. Three stores with grains of 68, 17 and 17 MB: list peaked at 145 MB before, 7 MB after. Co-Authored-By: Claude Sonnet 5.5 --- src/grain.rs | 55 ++++++++++++++++++++++++++++++++++++++++++++++++++++ src/query.rs | 7 +------ 2 files changed, 56 insertions(+), 6 deletions(-) diff --git a/src/grain.rs b/src/grain.rs index f29489a..5fc0d2f 100644 --- a/src/grain.rs +++ b/src/grain.rs @@ -320,6 +320,19 @@ impl Grain { } } +/// The grain's size and how many chunks it covers, without loading it: +/// the commit when it still describes the file, otherwise one walk. Opens +/// nothing for writing, so a caller that may only read the store can use it. +pub fn coverage(dir: &Path, name: &str) -> Option<(u64, usize)> { + let f = File::open(format::grain_path(dir, name)).ok()?; + let s = stamp(&f).ok()??; + let tail = match load_commit(dir, name, &s) { + Some(t) => t, + None => walk(&f, s.first, s.len, u64::MAX).ok()?, + }; + Some((s.len, tail.count as usize)) +} + pub fn load(path: &Path) -> anyhow::Result { let buf = fs::read(path).with_context(|| format!("reading grain index {}", path.display()))?; let Some(first) = first_record_offset(&buf) else { @@ -1104,4 +1117,46 @@ mod tests { } assert!(!g.may_contain_all(0, &marker(0))); } + + #[test] + fn coverage_agrees_with_loading_the_grain() { + let d = TempDir::new(); + let _st = a_store(d.path(), 7); + extend_grain(d.path(), "a.log").unwrap(); + let loaded = load(&format::grain_path(d.path(), "a.log")).unwrap(); + let len = grain_bytes(d.path()).len() as u64; + + assert_eq!( + coverage(d.path(), "a.log"), + Some((len, loaded.chunk_count())) + ); + // Without the commit it walks, and gets the same answer without + // writing one: the caller may be unable to write the store at all. + fs::remove_file(format::grain_commit_path(d.path(), "a.log")).unwrap(); + assert_eq!(coverage(d.path(), "a.log"), Some((len, 7))); + assert!(!format::grain_commit_path(d.path(), "a.log").exists()); + } + + #[test] + fn coverage_does_not_count_a_torn_tail_or_a_missing_grain() { + let d = TempDir::new(); + assert_eq!(coverage(d.path(), "a.log"), None); + let _st = a_store(d.path(), 4); + extend_grain(d.path(), "a.log").unwrap(); + let g = OpenOptions::new() + .append(true) + .open(format::grain_path(d.path(), "a.log")) + .unwrap(); + (&g).write_all(&100u32.to_le_bytes()).unwrap(); + drop(g); + fs::remove_file(format::grain_commit_path(d.path(), "a.log")).unwrap(); + assert_eq!(coverage(d.path(), "a.log").map(|c| c.1), Some(4)); + + fs::write( + format::grain_path(d.path(), "a.log"), + b"not a grain at all, no", + ) + .unwrap(); + assert_eq!(coverage(d.path(), "a.log"), None); + } } diff --git a/src/query.rs b/src/query.rs index 7f323f9..0c4fdb8 100644 --- a/src/query.rs +++ b/src/query.rs @@ -2682,12 +2682,7 @@ pub fn summarize_store( let dropped = std::fs::File::open(&rings_path) .and_then(|f| format::read_header_dropped(&f)) .unwrap_or_default(); - let gpath = format::grain_path(dir, name); - let grain = std::fs::metadata(&gpath).ok().and_then(|m| { - crate::grain::load(&gpath) - .ok() - .map(|g| (m.len(), g.chunk_count())) - }); + let grain = crate::grain::coverage(dir, name); let get = |k: &str| { bark.and_then(|b| b.get(k)) .and_then(|v| v.as_str())