From 2fa0ef6a9f92ea9ad241d3809a14d6c0c75bdaba Mon Sep 17 00:00:00 2001 From: alejandrogzi Date: Sat, 13 Jun 2026 16:40:51 +0200 Subject: [PATCH] BREAKING CHANGE: v0.0.8 -> sort/merge --rename impl + merge preserves unique metadata lines --- Cargo.lock | 2 +- Cargo.toml | 2 +- README.md | 8 +- assets/changelog/Changelog.md | 26 +++++ assets/tools/merge.md | 6 + assets/tools/sort.md | 13 ++- src/cli/merge.rs | 204 ++++++++++++++++++++++++++++++---- src/cli/sort.rs | 99 ++++++++++++++++- src/cli/sort_core.rs | 44 +++++++- src/io/writer.rs | 53 ++++++++- src/lib.rs | 4 +- 11 files changed, 415 insertions(+), 46 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a674f9c..8e7c449 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -66,7 +66,7 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "chaintools" -version = "0.0.7" +version = "0.0.8" dependencies = [ "clap", "flate2", diff --git a/Cargo.toml b/Cargo.toml index ac8fb2d..82daf6b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "chaintools" -version = "0.0.7" +version = "0.0.8" edition = "2024" authors = ["alejandrogzi "] license = "MIT" diff --git a/README.md b/README.md index 63e6932..8f62717 100644 --- a/README.md +++ b/README.md @@ -10,8 +10,8 @@

- - Version Badge + + Version Badge Crates.io Version @@ -29,7 +29,7 @@ work with .chain files in Rust

- docs . + docs . usage . tools . chains @@ -60,7 +60,7 @@ Add this to your `Cargo.toml`: ```toml [dependencies] -chaintools = { version = "0.0.7", features = ["mmap", "gzip", "parallel"] } +chaintools = { version = "0.0.8", features = ["mmap", "gzip", "parallel"] } ``` ## Benchmarks diff --git a/assets/changelog/Changelog.md b/assets/changelog/Changelog.md index 803020f..9cd3320 100644 --- a/assets/changelog/Changelog.md +++ b/assets/changelog/Changelog.md @@ -5,6 +5,32 @@ All notable changes to **chaintools** are documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.0.8] - 2025-06-13 + +### Added + +- **`--rename`/`-r` flag for `merge`** — `chaintools merge --rename` reassigns chain + IDs sequentially in sorted output order, so the first emitted chain receives + `id 1`, the next `2`, and so on. The flag implies sorting: it defaults to + `--sort-by score` when no key is given, and respects an explicit `--sort-by ` + when one is provided. Renaming is applied at final emission for both the in-memory + and external (spill-to-disk) merge paths, with no extra pass over the data. Backed + by new `write_chain_dense_with_id` / `write_chain_header_with_id` writer helpers. + +- **`--rename`/`-r` flag for `sort`** — `chaintools sort --rename` reassigns chain + IDs sequentially in sorted output order, following the selected `--sort-by` key + (default `score`). When combined with `--out-index`, the recorded offsets are + computed against the renamed output so they remain consistent with the written + bytes. + +### Changed + +- **`merge` deduplicates metadata** — when merging with `--sort-by`/`--rename`, + identical metadata (`#`) lines that appear in more than one input (for example, + files produced by a previous `split`) are now emitted once, in first-seen order. + Deduplication is performed at the byte level and does not affect `sort`, which + continues to preserve a single input's metadata verbatim. + ## [0.0.7] - 2026-06-11 ### Added diff --git a/assets/tools/merge.md b/assets/tools/merge.md index bcc9807..1a02fc6 100644 --- a/assets/tools/merge.md +++ b/assets/tools/merge.md @@ -42,8 +42,14 @@ Sorting: - no `--sort-by`: merge input text in input order without reparsing - `--sort-by score|id|reference|query`: parse all inputs, sort deterministically, and write canonical chain text +Renaming: +- `--rename`/`-r`: reassign chain IDs sequentially in sorted output order, so the first emitted chain gets `id 1`, the next `2`, and so on +- `--rename` implies sorting: it defaults to `--sort-by score` when no key is given, and respects an explicit `--sort-by ` when one is provided +- renaming is applied only to the final sorted output (both the in-memory and external-merge paths); original IDs are otherwise preserved + Notes: - `reference` is the target/reference-name sort used elsewhere as target sort - unsorted merge preserves input text order and only inserts record separators when needed between files - sorted merge preserves metadata lines, then emits sorted chains +- sorted merge deduplicates metadata lines at the byte level: an identical metadata line appearing in more than one input (e.g. files produced by a previous split) is emitted once, in first-seen order - `--max-gb ` controls the in-memory working set for sorted merge and defaults to `8` diff --git a/assets/tools/sort.md b/assets/tools/sort.md index 11e04d0..7a656f9 100644 --- a/assets/tools/sort.md +++ b/assets/tools/sort.md @@ -53,6 +53,14 @@ Sort chain records from a chain file or standard input. - `--max-gb `: spill sorted runs to temporary files when the in-memory working set grows past this limit. The default is `8`. +## Renaming + +- `--rename`/`-r`: reassign chain IDs sequentially in sorted output order, so + the first emitted chain gets `id 1`, the next `2`, and so on. The IDs follow + the selected `--sort-by` key (default `score`). +- When combined with `--out-index`, the index offsets are computed against the + renamed output, so they stay consistent with the written bytes. + ## Differences from UCSC chainSort / chainMergeSort - `sort` is the only user-facing sorting command. Large inputs are handled by @@ -64,8 +72,9 @@ Sort chain records from a chain file or standard input. separate UCSC boolean flags. - Equal primary keys are normalized to a deterministic order. Ties are broken by chain id, then target fields, then query fields, then remaining chain content. -- The implementation preserves existing chain ids. It does not renumber them - the way UCSC `chainMergeSort` does by default. +- The implementation preserves existing chain ids by default. Pass `--rename` + to renumber them sequentially in sorted order, similar to UCSC + `chainMergeSort`. - `--out-index` is not allowed together with `--gzip` because index offsets are defined on uncompressed output bytes. - Metadata lines beginning with `#` are copied to the beginning of the output in diff --git a/src/cli/merge.rs b/src/cli/merge.rs index d9b7a5a..97e6ab6 100644 --- a/src/cli/merge.rs +++ b/src/cli/merge.rs @@ -5,16 +5,16 @@ use std::fs::File; use std::io::{BufRead, BufReader, BufWriter, Read, Write}; use std::path::{Path, PathBuf}; -use chaintools::{StreamingReader, io::storage::is_gz_path}; +use chaintools::{io::storage::is_gz_path, StreamingReader}; use clap::{Args, ValueEnum}; #[cfg(feature = "gzip")] -use flate2::{Compression, read::MultiGzDecoder, write::GzEncoder}; +use flate2::{read::MultiGzDecoder, write::GzEncoder, Compression}; -use super::CliError; use super::sort_core::{ - OUTPUT_BUFFER_CAPACITY, SortAccumulator, SortCriterion, emit_sorted_chains, - write_metadata_lines, + emit_sorted_chains, write_metadata_lines, SortAccumulator, SortCriterion, + OUTPUT_BUFFER_CAPACITY, }; +use super::CliError; const BYTES_PER_GB: f64 = 1_000_000_000.0; const DEFAULT_MAX_GB: f64 = 16.0; @@ -67,10 +67,18 @@ pub struct MergeArgs { long = "sort-by", value_name = "KEY", value_enum, - help = "Sort merged output by the selected primary key" + help = "Sort merged output by the selected primary key [ options: score, id, reference, query ]" )] sort_by: Option, + #[arg( + short = 'r', + long = "rename", + help = "Reassign chain IDs sequentially (1, 2, 3, …) in sorted output order. \ + Implies --sort-by score unless another --sort-by key is given." + )] + rename: bool, + #[arg( short = 'G', long = "gzip", @@ -185,12 +193,25 @@ where args.out_chain.display() ); - if let Some(sort_by) = args.sort_by { + // `--rename` implies sorting; default to score when no key is given, but + // respect an explicit `--sort-by` key when one is provided. + let sort_by = args.sort_by.or(if args.rename { + Some(MergeSortBy::Score) + } else { + None + }); + + if let Some(sort_by) = sort_by { log::info!("merging sorted by {sort_by:?}"); + if args.rename { + log::info!("renaming chain ids sequentially in sorted order"); + } let max_in_memory_bytes = args.max_in_memory_bytes()?; let temp_dir = output_directory(&args.out_chain); + // Merge combines many files, so identical metadata lines (e.g. from + // previously split inputs) are deduplicated as they are collected. let mut accumulator = - SortAccumulator::new(sort_by.criterion(), max_in_memory_bytes, &temp_dir); + SortAccumulator::new(sort_by.criterion(), max_in_memory_bytes, &temp_dir, true); for path in &inputs { log::debug!("reading {}", path.display()); @@ -202,7 +223,7 @@ where let chains = accumulator.chains_pushed(); let runs_spilled = accumulator.runs_spilled(); let (metadata, sorted) = accumulator.finish()?; - emit_sorted_output(&args, &metadata, sorted, sort_by.criterion())?; + emit_sorted_output(&args, &metadata, sorted, sort_by.criterion(), args.rename)?; super::log_summary( "merge", &[ @@ -378,6 +399,7 @@ fn output_directory(out_chain: &Path) -> PathBuf { /// * `metadata` - Metadata lines to output /// * `sorted` - Sorted runs to merge /// * `sort_by` - Sort criterion +/// * `rename` - When `true`, reassign chain IDs sequentially in output order /// /// # Output /// @@ -387,13 +409,14 @@ fn emit_sorted_output( metadata: &[Vec], sorted: super::sort_core::SortedInput, sort_by: SortCriterion, + rename: bool, ) -> Result<(), CliError> { #[cfg(feature = "gzip")] if args.gzip { let writer = open_output_writer(&args.out_chain)?; let mut encoder = GzEncoder::new(writer, Compression::fast()); write_metadata_lines(&mut encoder, metadata)?; - emit_sorted_chains(&mut encoder, sorted, sort_by)?; + emit_sorted_chains(&mut encoder, sorted, sort_by, rename)?; encoder.try_finish()?; encoder.get_mut().flush()?; return Ok(()); @@ -401,7 +424,7 @@ fn emit_sorted_output( let mut writer = open_output_writer(&args.out_chain)?; write_metadata_lines(&mut writer, metadata)?; - emit_sorted_chains(&mut writer, sorted, sort_by)?; + emit_sorted_chains(&mut writer, sorted, sort_by, rename)?; writer.flush()?; Ok(()) } @@ -770,10 +793,9 @@ mod tests { input.arg(), ]); - assert!( - err.to_string() - .contains("--out-chain must not be the same path as input chain") - ); + assert!(err + .to_string() + .contains("--out-chain must not be the same path as input chain")); } #[test] @@ -836,6 +858,144 @@ mod tests { ); } + #[test] + fn merge_rename_defaults_to_score_order() { + let input_a = TempPath::new("chain"); + let input_b = TempPath::new("chain"); + let output = TempPath::new("chain"); + // Lower score, originally id 7. + write_file( + &input_a.path, + b"chain 5 chr2 100 + 0 5 qry2 100 + 0 5 7\n5\n\n", + ); + // Higher score, originally id 3. + write_file( + &input_b.path, + b"chain 9 chr1 100 + 0 5 qry1 100 + 0 5 3\n5\n\n", + ); + + run_ok(vec![ + arg("--chains"), + input_a.arg(), + input_b.arg(), + arg("--out-chain"), + output.arg(), + arg("--rename"), + ]); + + // --rename implies --sort-by score (descending), then ids become 1, 2. + assert_eq!( + std::fs::read_to_string(&output.path).unwrap(), + "chain 9 chr1 100 + 0 5 qry1 100 + 0 5 1\n5\n\nchain 5 chr2 100 + 0 5 qry2 100 + 0 5 2\n5\n\n" + ); + } + + #[test] + fn merge_rename_respects_explicit_sort_by() { + let input_a = TempPath::new("chain"); + let input_b = TempPath::new("chain"); + let output = TempPath::new("chain"); + // Higher score but later reference; originally id 7. + write_file( + &input_a.path, + b"chain 9 chr2 100 + 0 5 qry2 100 + 0 5 7\n5\n\n", + ); + // Lower score but earlier reference; originally id 3. + write_file( + &input_b.path, + b"chain 5 chr1 100 + 0 5 qry1 100 + 0 5 3\n5\n\n", + ); + + run_ok(vec![ + arg("--chains"), + input_a.arg(), + input_b.arg(), + arg("--out-chain"), + output.arg(), + arg("--rename"), + arg("--sort-by"), + arg("reference"), + ]); + + // Explicit --sort-by reference wins: chr1 first, then chr2; ids 1, 2. + assert_eq!( + std::fs::read_to_string(&output.path).unwrap(), + "chain 5 chr1 100 + 0 5 qry1 100 + 0 5 1\n5\n\nchain 9 chr2 100 + 0 5 qry2 100 + 0 5 2\n5\n\n" + ); + } + + #[test] + fn merge_rename_across_spill_path() { + let input_a = TempPath::new("chain"); + let input_b = TempPath::new("chain"); + let input_c = TempPath::new("chain"); + let output = TempPath::new("chain"); + write_file( + &input_a.path, + b"chain 5 chr3 100 + 0 5 qry3 100 + 0 5 50\n5\n\n", + ); + write_file( + &input_b.path, + b"chain 9 chr1 100 + 0 5 qry1 100 + 0 5 60\n5\n\n", + ); + write_file( + &input_c.path, + b"chain 7 chr2 100 + 0 5 qry2 100 + 0 5 70\n5\n\n", + ); + + // A tiny memory budget forces per-chain spilling, exercising the + // external k-way merge (SortedInput::Runs) emission path. + run_ok(vec![ + arg("--chains"), + input_a.arg(), + input_b.arg(), + input_c.arg(), + arg("--out-chain"), + output.arg(), + arg("--rename"), + arg("--max-gb"), + arg("0.0000000001"), + ]); + + // Score descending (9, 7, 5) renamed to 1, 2, 3. + assert_eq!( + std::fs::read_to_string(&output.path).unwrap(), + "chain 9 chr1 100 + 0 5 qry1 100 + 0 5 1\n5\n\nchain 7 chr2 100 + 0 5 qry2 100 + 0 5 2\n5\n\nchain 5 chr3 100 + 0 5 qry3 100 + 0 5 3\n5\n\n" + ); + } + + #[test] + fn merge_deduplicates_repeated_metadata() { + let input_a = TempPath::new("chain"); + let input_b = TempPath::new("chain"); + let output = TempPath::new("chain"); + write_file( + &input_a.path, + b"#common\n#a-only\nchain 5 chr1 100 + 0 5 qry1 100 + 0 5 1\n5\n\n", + ); + write_file( + &input_b.path, + b"#common\n#b-only\nchain 9 chr2 100 + 0 5 qry2 100 + 0 5 2\n5\n\n", + ); + + run_ok(vec![ + arg("--chains"), + input_a.arg(), + input_b.arg(), + arg("--out-chain"), + output.arg(), + arg("--sort-by"), + arg("score"), + ]); + + // `#common` appears in both inputs but is emitted once, in first-seen + // order; the unique metadata follows, then chains sorted by score. + assert_eq!( + std::fs::read_to_string(&output.path).unwrap(), + "#common\n#a-only\n#b-only\nchain 9 chr2 100 + 0 5 qry2 100 + 0 5 2\n5\n\nchain 5 chr1 100 + 0 5 qry1 100 + 0 5 1\n5\n\n" + ); + } + #[cfg(feature = "gzip")] #[test] fn merge_reads_gzip_input_and_writes_gzip_output() { @@ -882,10 +1042,9 @@ mod tests { arg("--gzip"), ]); - assert!( - err.to_string() - .contains("--gzip requires chaintools to be built with the `gzip` feature") - ); + assert!(err + .to_string() + .contains("--gzip requires chaintools to be built with the `gzip` feature")); } #[test] @@ -905,9 +1064,8 @@ mod tests { arg("0"), ]); - assert!( - err.to_string() - .contains("--max-gb must be greater than zero") - ); + assert!(err + .to_string() + .contains("--max-gb must be greater than zero")); } } diff --git a/src/cli/sort.rs b/src/cli/sort.rs index e02ce51..7f7ae6a 100644 --- a/src/cli/sort.rs +++ b/src/cli/sort.rs @@ -63,6 +63,13 @@ pub struct SortArgs { )] sort_by: SortBy, + #[arg( + short = 'r', + long = "rename", + help = "Reassign chain IDs sequentially (1, 2, 3, …) in sorted output order, following the selected --sort-by key." + )] + rename: bool, + #[arg( short = 'I', long = "out-index", @@ -303,6 +310,9 @@ where args.sort_by, args.max_gb ); + if args.rename { + log::info!("renaming chain ids sequentially in sorted order"); + } let collected = if let Some(path) = &args.chain { let mut reader = StreamingReader::from_path(path)?; @@ -432,8 +442,10 @@ fn collect_sorted_input( reader: &mut StreamingReader, temp_dir: &Path, ) -> Result { + // Sort reads a single input, so metadata lines are preserved verbatim + // (no cross-file deduplication). let mut accumulator = - SortAccumulator::new(args.sort_by.criterion(), max_in_memory_bytes, temp_dir); + SortAccumulator::new(args.sort_by.criterion(), max_in_memory_bytes, temp_dir, false); accumulator.push_stream(reader)?; // Capture counts before `finish` consumes the accumulator. let chains = accumulator.chains_pushed(); @@ -472,7 +484,7 @@ fn emit_output( let writer = open_output_writer(args, stdout)?; let mut encoder = GzEncoder::new(writer, Compression::fast()); write_metadata_lines(&mut encoder, metadata)?; - emit_sorted_chains(&mut encoder, sorted, args.sort_by.criterion())?; + emit_sorted_chains(&mut encoder, sorted, args.sort_by.criterion(), args.rename)?; encoder.try_finish()?; encoder.get_mut().flush()?; return Ok(()); @@ -484,13 +496,19 @@ fn emit_output( let mut index_writer = BufWriter::with_capacity(OUTPUT_BUFFER_CAPACITY, File::create(index_path)?); write_metadata_lines(&mut counted, metadata)?; - emit_sorted_chains_with_index(&mut counted, &mut index_writer, sorted, args.sort_by)?; + emit_sorted_chains_with_index( + &mut counted, + &mut index_writer, + sorted, + args.sort_by, + args.rename, + )?; counted.flush()?; index_writer.flush()?; } else { let mut writer = writer; write_metadata_lines(&mut writer, metadata)?; - emit_sorted_chains(&mut writer, sorted, args.sort_by.criterion())?; + emit_sorted_chains(&mut writer, sorted, args.sort_by.criterion(), args.rename)?; writer.flush()?; } @@ -534,29 +552,48 @@ fn open_output_writer<'a, W: Write>( /// * `index_writer` - Index file writer /// * `sorted` - Sorted input (in-memory or runs) /// * `sort_by` - Sort criterion +/// * `rename` - When `true`, chain IDs are reassigned sequentially in output order /// /// # Output /// /// Returns `Ok(())` on success or `Err(CliError)` on failure +/// +/// When `rename` is set, the chains are written with sequential IDs while the +/// index keeps recording the byte offsets reported by the counting writer, so +/// the offsets stay consistent with the (renamed) output. fn emit_sorted_chains_with_index( writer: &mut CountingWriter, index_writer: &mut BufWriter, sorted: SortedInput, sort_by: SortBy, + rename: bool, ) -> Result<(), CliError> { let mut tracker = IndexTracker::new(sort_by.criterion()); + let mut next_id: u64 = 1; match sorted { SortedInput::InMemory(records) => { for chain in &records { tracker.before_chain(index_writer, writer.position(), chain)?; - chaintools::write_chain_dense(writer, chain)?; + if rename { + chaintools::write_chain_dense_with_id(writer, chain, next_id)?; + next_id += 1; + } else { + chaintools::write_chain_dense(writer, chain)?; + } } } SortedInput::Runs(runs) => { with_merged_runs(&runs, sort_by.criterion(), |chain| { tracker.before_chain(index_writer, writer.position(), chain)?; - chaintools::write_chain_dense(writer, chain).map_err(CliError::from) + if rename { + chaintools::write_chain_dense_with_id(writer, chain, next_id) + .map_err(CliError::from)?; + next_id += 1; + Ok(()) + } else { + chaintools::write_chain_dense(writer, chain).map_err(CliError::from) + } })?; } } @@ -731,6 +768,56 @@ mod tests { assert_eq!(stderr, b""); } + #[test] + fn rename_reassigns_ids_in_score_order() { + let (stdout, stderr) = run_ok( + vec![arg("--rename")], + b"chain 5 chr2 10 + 0 5 qry2 10 + 0 5 50\n5\n\nchain 9 chr1 10 + 0 5 qry1 10 + 0 5 60\n5\n\n", + ); + + // Default --sort-by score (descending), ids renamed to 1, 2. + assert_eq!( + String::from_utf8(stdout).unwrap(), + "chain 9 chr1 10 + 0 5 qry1 10 + 0 5 1\n5\n\nchain 5 chr2 10 + 0 5 qry2 10 + 0 5 2\n5\n\n" + ); + assert_eq!(stderr, b""); + } + + #[test] + fn rename_follows_explicit_target_sort() { + let (stdout, stderr) = run_ok( + vec![arg("--rename"), arg("--sort-by"), arg("target")], + b"chain 9 chr2 10 + 0 5 qry2 10 + 0 5 50\n5\n\nchain 5 chr1 10 + 0 5 qry1 10 + 0 5 60\n5\n\n", + ); + + // Sorted by target (chr1 before chr2); ids follow that order: 1, 2. + assert_eq!( + String::from_utf8(stdout).unwrap(), + "chain 5 chr1 10 + 0 5 qry1 10 + 0 5 1\n5\n\nchain 9 chr2 10 + 0 5 qry2 10 + 0 5 2\n5\n\n" + ); + assert_eq!(stderr, b""); + } + + #[test] + fn rename_with_index_offsets_match_renamed_output() { + let index = TempPath::new("tab"); + + // Original ids are 6 digits wide; renaming to 1/2 shrinks the headers, + // so the second chain's offset differs from what the original ids would + // imply (0x36). The index must reflect the renamed bytes (0x31). + let (stdout, stderr) = run_ok( + vec![arg("--rename"), arg("--out-index"), index.arg()], + b"chain 300 chr3 1000 + 0 30 qry1 500 + 0 30 999999\n30\n\nchain 100 chr1 1000 + 10 50 qry3 500 + 15 55 888888\n40\n\n", + ); + + assert_eq!( + String::from_utf8(stdout).unwrap(), + "chain 300 chr3 1000 + 0 30 qry1 500 + 0 30 1\n30\n\nchain 100 chr1 1000 + 10 50 qry3 500 + 15 55 2\n40\n\n" + ); + assert_eq!(fs::read_to_string(&index.path).unwrap(), "0\t300\n31\t100\n"); + assert_eq!(stderr, b""); + } + #[test] fn metadata_lines_are_emitted_before_sorted_chains() { let (stdout, stderr) = run_ok( diff --git a/src/cli/sort_core.rs b/src/cli/sort_core.rs index 26bad2a..f03df87 100644 --- a/src/cli/sort_core.rs +++ b/src/cli/sort_core.rs @@ -2,13 +2,15 @@ // Distributed under the terms of the Apache License, Version 2.0. use std::cmp::Ordering; -use std::collections::BinaryHeap; +use std::collections::{BinaryHeap, HashSet}; use std::fs::{self, File, OpenOptions}; use std::io::{self, BufRead, BufWriter, Write}; use std::path::{Path, PathBuf}; use std::time::{SystemTime, UNIX_EPOCH}; -use chaintools::{Block, OwnedChain, StreamItem, StreamingReader, write_chain_dense}; +use chaintools::{ + Block, OwnedChain, StreamItem, StreamingReader, write_chain_dense, write_chain_dense_with_id, +}; #[cfg(feature = "parallel")] use rayon::prelude::ParallelSliceMut; @@ -105,7 +107,10 @@ impl Drop for TempRun { /// * `max_in_memory_bytes` - Maximum bytes to keep in memory before spilling /// * `temp_dir` - Directory for temporary run files /// * `next_generated_id` - Counter for generated chain IDs -/// * `metadata` - Preserved metadata lines (comments) +/// * `metadata` - Preserved metadata lines (comments), in first-seen order +/// * `dedup_metadata` - When `true`, identical metadata lines are emitted once +/// * `seen_metadata` - Byte-level set of metadata lines already collected (used +/// only when `dedup_metadata` is enabled) /// * `records` - Chains currently in memory /// * `runs` - Temporary run files for external sorting /// * `chunk_bytes` - Estimated memory usage of current records @@ -116,6 +121,8 @@ pub(super) struct SortAccumulator<'a> { temp_dir: &'a Path, next_generated_id: u64, metadata: Vec>, + dedup_metadata: bool, + seen_metadata: HashSet>, records: Vec, runs: Vec, chunk_bytes: u64, @@ -129,6 +136,7 @@ impl<'a> SortAccumulator<'a> { sort_by: SortCriterion, max_in_memory_bytes: u64, temp_dir: &'a Path, + dedup_metadata: bool, ) -> Self { Self { sort_by, @@ -136,6 +144,8 @@ impl<'a> SortAccumulator<'a> { temp_dir, next_generated_id: 1, metadata: Vec::new(), + dedup_metadata, + seen_metadata: HashSet::new(), records: Vec::new(), runs: Vec::new(), chunk_bytes: 0, @@ -153,7 +163,14 @@ impl<'a> SortAccumulator<'a> { while let Some(item) = reader.next_item()? { match item { - StreamItem::MetaLine(line) => self.metadata.push(line), + StreamItem::MetaLine(line) => { + // Drop metadata lines already collected (e.g. identical + // headers from previously split files). The set is keyed on + // the raw bytes, so the comparison stays at the byte level. + if !self.dedup_metadata || self.seen_metadata.insert(line.clone()) { + self.metadata.push(line); + } + } StreamItem::Header(header) => { let blocks = reader.read_blocks(header.offset)?; let chain = header.into_chain(blocks); @@ -240,6 +257,8 @@ impl<'a> SortAccumulator<'a> { /// * `writer` - Output writer /// * `sorted` - The sorted input (in-memory or runs) /// * `sort_by` - The sort criterion used +/// * `rename` - When `true`, chain IDs are reassigned sequentially (`1, 2, 3, …`) +/// in emission order; when `false`, each chain keeps its original ID /// /// # Output /// @@ -248,16 +267,29 @@ pub(super) fn emit_sorted_chains( writer: &mut W, sorted: SortedInput, sort_by: SortCriterion, + rename: bool, ) -> Result<(), CliError> { + let mut next_id: u64 = 1; match sorted { SortedInput::InMemory(records) => { for chain in &records { - write_chain_dense(writer, chain)?; + if rename { + write_chain_dense_with_id(writer, chain, next_id)?; + next_id += 1; + } else { + write_chain_dense(writer, chain)?; + } } } SortedInput::Runs(runs) => { with_merged_runs(&runs, sort_by, |chain| { - write_chain_dense(writer, chain).map_err(CliError::from) + if rename { + write_chain_dense_with_id(writer, chain, next_id).map_err(CliError::from)?; + next_id += 1; + Ok(()) + } else { + write_chain_dense(writer, chain).map_err(CliError::from) + } })?; } } diff --git a/src/io/writer.rs b/src/io/writer.rs index 06a9f08..8295517 100644 --- a/src/io/writer.rs +++ b/src/io/writer.rs @@ -49,6 +49,30 @@ pub fn write_chain_dense(writer: &mut W, chain: &OwnedChain) -> Result write_dense_blocks(writer, &chain.blocks) } +/// Writes a chain in dense format with an overridden chain ID. +/// +/// Identical to [`write_chain_dense`], except the header's `id` field is +/// replaced by the provided `id` rather than the chain's own id. This is used +/// by `merge --rename` to reassign sequential IDs without mutating the chain. +/// +/// # Arguments +/// +/// * `writer` - Output writer +/// * `chain` - The owned chain to write +/// * `id` - The chain ID to emit in the header +/// +/// # Output +/// +/// Returns `Ok(())` on success or `Err(ChainError)` on failure +pub fn write_chain_dense_with_id( + writer: &mut W, + chain: &OwnedChain, + id: u64, +) -> Result<(), ChainError> { + write_chain_header_with_id(writer, chain, id)?; + write_dense_blocks(writer, &chain.blocks) +} + /// Writes a chain header line. /// /// Writes the "chain" header line with all chain metadata in the standard format: @@ -89,6 +113,33 @@ pub fn write_chain_dense(writer: &mut W, chain: &OwnedChain) -> Result /// write_chain_header(&mut buf, &header)?; /// ``` pub fn write_chain_header(writer: &mut W, chain: &C) -> Result<(), ChainError> +where + W: Write, + C: ChainLike, +{ + write_chain_header_with_id(writer, chain, chain.id()) +} + +/// Writes a chain header line with an overridden chain ID. +/// +/// Identical to [`write_chain_header`], except the trailing `id` field is set +/// to the provided `id` instead of the chain's own id. The chain's other +/// fields are written unchanged. +/// +/// # Arguments +/// +/// * `writer` - Output writer +/// * `chain` - The chain-like object to write (any type implementing ChainLike) +/// * `id` - The chain ID to emit in the header +/// +/// # Output +/// +/// Returns `Ok(())` on success or `Err(ChainError)` on failure +pub fn write_chain_header_with_id( + writer: &mut W, + chain: &C, + id: u64, +) -> Result<(), ChainError> where W: Write, C: ChainLike, @@ -111,7 +162,7 @@ where strand_to_byte(chain.query_strand()) as char, chain.query_start(), chain.query_end(), - chain.id() + id )?; Ok(()) } diff --git a/src/lib.rs b/src/lib.rs index 7a80ddf..501fc3d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -119,8 +119,8 @@ pub use io::reader::Reader; pub use io::storage::ByteSlice; pub use io::stream::{OwnedChain, OwnedChainHeader, OwnedChainParts, StreamItem, StreamingReader}; pub use io::writer::{ - write_chain_dense, write_chain_header, write_dense_blocks, write_metadata_line, - write_metadata_lines, + write_chain_dense, write_chain_dense_with_id, write_chain_header, write_chain_header_with_id, + write_dense_blocks, write_metadata_line, write_metadata_lines, }; #[cfg(feature = "sequence")]