diff --git a/Cargo.lock b/Cargo.lock index f23838b..f584310 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -134,7 +134,7 @@ checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" [[package]] name = "chaintools" -version = "0.0.11" +version = "0.0.12" dependencies = [ "clap", "flate2", diff --git a/Cargo.toml b/Cargo.toml index e858f92..853ef1f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "chaintools" -version = "0.0.11" +version = "0.0.12" edition = "2024" authors = ["alejandrogzi "] license = "MIT" diff --git a/README.md b/README.md index f8bb6ee..f9466e1 100644 --- a/README.md +++ b/README.md @@ -10,8 +10,8 @@

- - Version Badge + + Version Badge Crates.io Version @@ -29,7 +29,7 @@ work with .chain files in Rust

- docs . + docs . usage . tools . chains @@ -60,7 +60,7 @@ Add this to your `Cargo.toml`: ```toml [dependencies] -chaintools = { version = "0.0.11", features = ["mmap", "gzip", "parallel"] } +chaintools = { version = "0.0.12", features = ["mmap", "gzip", "parallel"] } ``` ## Benchmarks diff --git a/assets/changelog/Changelog.md b/assets/changelog/Changelog.md index 55d7eba..77bcb84 100644 --- a/assets/changelog/Changelog.md +++ b/assets/changelog/Changelog.md @@ -5,6 +5,23 @@ All notable changes to **chaintools** are documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.0.12] - 2026-08-10 + +### Added + +- **`compare --memory-ceiling ` / `-M`** — the in-memory ceiling for + canonical mappings is now configurable per run instead of a fixed + constant; the ceiling value is reported in the error when exceeded. + +### Changed + +- **`compare` default memory ceiling raised from 8 to 16 GiB** — the + canonical-mapping estimate is checked against the configurable ceiling + (default 16 GiB) before canonicalization, so larger inputs pass by + default while oversized ones still fail cleanly instead of OOMing. + Per-key external sorting remains the planned upgrade path for inputs + beyond the ceiling. + ## [0.0.11] - 2026-08-09 ### Added @@ -320,6 +337,8 @@ First stable release. - Automatic gzip (`.chain.gz`) detection and decompression (`gzip` feature). - Test suite, benchmark binary, CI workflows, Docker image, and rustdoc documentation. +[0.0.12]: https://github.com/alejandrogzi/chaintools/releases/tag/v0.0.12 +[0.0.11]: https://github.com/alejandrogzi/chaintools/releases/tag/v0.0.11 [0.0.10]: https://github.com/alejandrogzi/chaintools/releases/tag/v0.0.10 [0.0.9]: https://github.com/alejandrogzi/chaintools/releases/tag/v0.0.9 [0.0.8]: https://github.com/alejandrogzi/chaintools/releases/tag/v0.0.8 diff --git a/assets/tools/compare.md b/assets/tools/compare.md index eca4826..0d8e12d 100644 --- a/assets/tools/compare.md +++ b/assets/tools/compare.md @@ -47,6 +47,7 @@ Both inputs must be paths; there is no stdin mode. - `--by-sequence`: add the per-target agreement table and top-chain lists. - `--top `: with `--by-sequence`, show the top `N` scored chains per target. Default `3`. +- `--memory-ceiling ` / `-M`: fail when the canonical-mapping estimate exceeds this many GiB. Default `16`. ## What is compared @@ -149,7 +150,7 @@ file is not an error — that may be a real coverage difference. ## Memory Canonical mappings are held in memory (one ~32-byte segment per block, both -files). The estimate is checked against a fixed 8 GiB ceiling before -canonicalization; past it the command fails with a clear error instead of -OOMing. Per-key external sorting is the planned upgrade path for larger inputs -and is not implemented. +files). The estimate is checked against a configurable ceiling (default 16 GiB, +set via `--memory-ceiling `) before canonicalization; past it the command +fails with a clear error instead of OOMing. Per-key external sorting is the +planned upgrade path for larger inputs and is not implemented. diff --git a/src/cli/compare.rs b/src/cli/compare.rs index 82c099b..fdc4cdb 100644 --- a/src/cli/compare.rs +++ b/src/cli/compare.rs @@ -12,8 +12,8 @@ use clap::Args; use super::CliError; use super::stats::{self, ChainStats, TargetStats}; -// ponytail: fixed in-memory ceiling; add per-key external sort if real inputs exceed 8 GiB. -const CANONICAL_MEMORY_BUDGET: u128 = 8 * 1024 * 1024 * 1024; +// ponytail: in-memory ceiling, configurable via --memory-ceiling; add per-key external sort if real inputs exceed it. +const DEFAULT_MEMORY_CEILING_GIB: u64 = 16; const ESTIMATED_SEGMENT_BYTES: u128 = 32; #[derive(Debug, Args)] @@ -34,6 +34,15 @@ pub struct CompareArgs { help = "Top scored chains to show per target with --by-sequence" )] top: usize, + + #[arg( + short = 'M', + long = "memory-ceiling", + value_name = "GIB", + default_value_t = DEFAULT_MEMORY_CEILING_GIB, + help = "Memory ceiling in GiB for canonical mappings before compare fails" + )] + memory_ceiling: u64, } #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] @@ -123,7 +132,7 @@ where let stats_a = analyze_path(&args.chain_a)?; let stats_b = analyze_path(&args.chain_b)?; ensure_compatible(&stats_a, &stats_b)?; - check_memory_budget(stats_a.blocks, stats_b.blocks)?; + check_memory_budget(stats_a.blocks, stats_b.blocks, args.memory_ceiling)?; let mut names = NameInterner::default(); let mappings_a = canonicalize_path(&args.chain_a, stats_a.blocks, &mut names)?; @@ -184,11 +193,12 @@ fn check_sizes( Ok(()) } -fn check_memory_budget(a_blocks: u64, b_blocks: u64) -> Result<(), CliError> { +fn check_memory_budget(a_blocks: u64, b_blocks: u64, ceiling_gib: u64) -> Result<(), CliError> { let estimated = (u128::from(a_blocks) + u128::from(b_blocks)) * ESTIMATED_SEGMENT_BYTES * 2; - if estimated > CANONICAL_MEMORY_BUDGET { + let budget = u128::from(ceiling_gib) * 1024 * 1024 * 1024; + if estimated > budget { return Err(CliError::Message(format!( - "canonical mapping estimate ({:.2} GiB) exceeds compare's fixed 8 GiB ceiling; per-key external sort is the upgrade path and is not implemented", + "canonical mapping estimate ({:.2} GiB) exceeds compare's {ceiling_gib} GiB memory ceiling; per-key external sort is the upgrade path and is not implemented", estimated as f64 / (1024.0 * 1024.0 * 1024.0) ))); } @@ -802,8 +812,19 @@ fn percent_ratio_u128(numerator: u128, denominator: u128) -> String { #[cfg(test)] mod tests { use super::*; + use clap::Parser; use std::io::Cursor; + #[derive(Debug, Parser)] + struct CompareHarness { + #[command(flatten)] + args: CompareArgs, + } + + fn parse_compare_args(argv: &[&str]) -> CompareArgs { + CompareHarness::try_parse_from(argv).expect("args parse").args + } + fn stats_text(text: &str) -> ChainStats { stats::analyze(&mut StreamingReader::new(Cursor::new(text.as_bytes()))) .expect("valid stats fixture") @@ -913,4 +934,28 @@ mod tests { assert_eq!(coverage.query_covered_bp, 20); assert_eq!(coverage.query_multi_bp, 10); } + + #[test] + fn memory_budget_respects_ceiling() { + // estimate = (a_blocks + b_blocks) * 32 bytes * 2 files = 64 bytes/block. + // The 16 GiB default admits exactly 2^28 blocks and rejects one more; + // a raised ceiling admits the overflow case. + assert!(check_memory_budget(268_435_456, 0, 16).is_ok()); + assert!(check_memory_budget(268_435_457, 0, 16).is_err()); + assert!(check_memory_budget(268_435_457, 0, 17).is_ok()); + } + + #[test] + fn parses_memory_ceiling_flag() { + let args = parse_compare_args(&[ + "compare", "-a", "a.chain", "-b", "b.chain", "--memory-ceiling", "32", + ]); + assert_eq!(args.memory_ceiling, 32); + + let args = parse_compare_args(&["compare", "-a", "a.chain", "-b", "b.chain", "-M", "4"]); + assert_eq!(args.memory_ceiling, 4); + + let args = parse_compare_args(&["compare", "-a", "a.chain", "-b", "b.chain"]); + assert_eq!(args.memory_ceiling, DEFAULT_MEMORY_CEILING_GIB); + } }