diff --git a/CHANGELOG.md b/CHANGELOG.md index 54969ee..32143b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # CHANGELOG +## Unreleased + +- Performance: reuse one indexed BAM/CRAM reader per worker thread in `base-depth` and both `only-depth` modes instead of reopening the input for every region ([#110](https://github.com/sstadick/perbase/issues/110)). + ## 1.4.0 - [fix](https://github.com/sstadick/perbase/pull/104): properly parse args by revering global structopt setting from @chenrui333 diff --git a/src/commands/base_depth.rs b/src/commands/base_depth.rs index 6715caf..1187715 100644 --- a/src/commands/base_depth.rs +++ b/src/commands/base_depth.rs @@ -12,7 +12,7 @@ use perbase_lib::{ read_filter::{DefaultReadFilter, ReadFilter}, reference, utils, }; -use rust_htslib::{bam, bam::Read}; +use rust_htslib::bam::Read; use std::{convert::TryInto, path::PathBuf}; use structopt::{StructOpt, clap::AppSettings}; @@ -239,85 +239,78 @@ impl RegionProcessor for BaseProcessor { /// the defined filters fn process_region(&self, tid: u32, start: u32, stop: u32) -> Vec { trace!("Processing region {}(tid):{}-{}", tid, start, stop); - // Create a reader - let mut reader = - bam::IndexedReader::from_path(&self.reads).expect("Indexed Reader for region"); - - // If passed add ref_fasta - if let Some(ref_fasta) = &self.ref_fasta { - reader.set_reference(ref_fasta).expect("Set ref"); - } - - let header = reader.header().to_owned(); - // fetch the region of interest - reader.fetch((tid, start, stop)).expect("Fetched a region"); - // Walk over pileups - let mut pileup = reader.pileup(); - pileup.set_max_depth(std::cmp::min(i32::MAX.try_into().unwrap(), self.max_depth)); - let result: Vec = pileup - .flat_map(|p| { - let pileup = p.expect("Extracted a pileup"); - // Verify that we are within the bounds of the chunk we are iterating on - let pileup_depth = pileup.depth(); - if pileup.pos() >= start && pileup.pos() < stop { - let mut pos = if self.mate_fix { - PileupPosition::from_pileup_mate_aware( - pileup, - &header, - &self.read_filter, - self.min_base_quality_score, - self.mate_fix_strategy, - ) + utils::with_indexed_reader(&self.reads, self.ref_fasta.as_deref(), |reader| { + let header = reader.header().to_owned(); + // fetch the region of interest + reader.fetch((tid, start, stop)).expect("Fetched a region"); + // Walk over pileups + let mut pileup = reader.pileup(); + pileup.set_max_depth(std::cmp::min(i32::MAX.try_into().unwrap(), self.max_depth)); + let result: Vec = pileup + .flat_map(|p| { + let pileup = p.expect("Extracted a pileup"); + // Verify that we are within the bounds of the chunk we are iterating on + let pileup_depth = pileup.depth(); + if pileup.pos() >= start && pileup.pos() < stop { + let mut pos = if self.mate_fix { + PileupPosition::from_pileup_mate_aware( + pileup, + &header, + &self.read_filter, + self.min_base_quality_score, + self.mate_fix_strategy, + ) + } else { + PileupPosition::from_pileup( + pileup, + &header, + &self.read_filter, + self.min_base_quality_score, + ) + }; + // Add the ref base if reference is available + if let Some(buffer) = &self.ref_buffer { + let seq = buffer + .seq(&pos.ref_seq) + .expect("Fetched reference sequence"); + pos.ref_base = Some(char::from( + *seq.get(pos.pos as usize) + .expect("Input SAM does not match reference"), + )); + } + if pileup_depth > (self.max_depth_warnings_cutoff) { + pos.near_max_depth = true; + } + pos.pos += self.coord_base; + Some(pos) } else { - PileupPosition::from_pileup( - pileup, - &header, - &self.read_filter, - self.min_base_quality_score, - ) - }; - // Add the ref base if reference is available - if let Some(buffer) = &self.ref_buffer { - let seq = buffer - .seq(&pos.ref_seq) - .expect("Fetched reference sequence"); - pos.ref_base = Some(char::from( - *seq.get(pos.pos as usize) - .expect("Input SAM does not match reference"), - )); + None } - if pileup_depth > (self.max_depth_warnings_cutoff) { - pos.near_max_depth = true; + }) + .collect(); + + if self.keep_zeros { + let mut new_result = vec![]; + let name = PileupPosition::compact_refseq(&header, tid); + let mut pos = start; + + for position in result.into_iter() { + while pos < (position.pos - self.coord_base) { + new_result.push(PileupPosition::new(name.clone(), pos + self.coord_base)); + pos += 1; } - pos.pos += self.coord_base; - Some(pos) - } else { - None + new_result.push(position); + pos += 1; } - }) - .collect(); - - if self.keep_zeros { - let mut new_result = vec![]; - let name = PileupPosition::compact_refseq(&header, tid); - let mut pos = start; - - for position in result.into_iter() { - while pos < (position.pos - self.coord_base) { + while pos < stop { new_result.push(PileupPosition::new(name.clone(), pos + self.coord_base)); pos += 1; } - new_result.push(position); - pos += 1; - } - while pos < stop { - new_result.push(PileupPosition::new(name.clone(), pos + self.coord_base)); - pos += 1; + new_result + } else { + result } - new_result - } else { - result - } + }) } } diff --git a/src/commands/only_depth.rs b/src/commands/only_depth.rs index 1033a42..e1912f3 100644 --- a/src/commands/only_depth.rs +++ b/src/commands/only_depth.rs @@ -288,173 +288,159 @@ impl OnlyDepthProcessor { /// Process a region, taking into account REF_SKIPs and mates fn process_region(&self, tid: u32, start: u32, stop: u32) -> Vec { - // Create a reader - let mut reader = - bam::IndexedReader::from_path(&self.reads).expect("Indexed Reader for region"); - - // If passed add ref_fasta - if let Some(ref_fasta) = &self.ref_fasta { - reader.set_reference(ref_fasta).expect("Set ref"); - } - - let header = reader.header().to_owned(); - // fetch the region of interest - reader.fetch((tid, start, stop)).expect("Fetched a region"); - - let mut counter: Vec = vec![0; (stop - start) as usize]; - let mut maties = HashMap::new(); - - // Walk over each read, counting the starts and ends - for record in reader - .rc_records() - .map(|r| r.expect("Read record")) - .filter(|read| self.read_filter.filter_read(read, None)) - .flat_map(|record| IterAlignedBlocks::new(record, self.mate_fix)) - { - let rec_start = u32::try_from(record.0).expect("check overflow"); - let rec_stop = u32::try_from(record.1).expect("check overflow"); - - // NB: since we are splitting the region, it's possible the region we are looking at - // may occur before the ROI, or after the ROI - if rec_start >= stop || rec_stop <= start { - continue; - } + utils::with_indexed_reader(&self.reads, self.ref_fasta.as_deref(), |reader| { + let header = reader.header().to_owned(); + // fetch the region of interest + reader.fetch((tid, start, stop)).expect("Fetched a region"); + + let mut counter: Vec = vec![0; (stop - start) as usize]; + let mut maties = HashMap::new(); + + // Walk over each read, counting the starts and ends + for record in reader + .rc_records() + .map(|r| r.expect("Read record")) + .filter(|read| self.read_filter.filter_read(read, None)) + .flat_map(|record| IterAlignedBlocks::new(record, self.mate_fix)) + { + let rec_start = u32::try_from(record.0).expect("check overflow"); + let rec_stop = u32::try_from(record.1).expect("check overflow"); + + // NB: since we are splitting the region, it's possible the region we are looking at + // may occur before the ROI, or after the ROI + if rec_start >= stop || rec_stop <= start { + continue; + } - // rectify start / stop with region boundaries - // increment the start of the region - let adjusted_start = if rec_start < start { - 0 - } else { - (rec_start - start) as usize - }; - - let mut dont_count_stop = false; // if this interval extends past the end of the region, don't count an end for it - let adjusted_stop = if rec_stop >= stop { - dont_count_stop = true; - counter.len() - 1 - } else { - (rec_stop - start) as usize - }; - - // check if this read has a mate that will be seen within this region - // that this works for both mates in pair - if self.mate_fix && record.2 { - // TODO: figure out better way of passing qname around, get rid of Lazy - // let qname = String::from(std::str::from_utf8(record.3).expect("Convert qname")); - let intervals = maties.entry(record.3).or_insert(vec![]); - intervals.push(Interval { - start: adjusted_start, - stop: adjusted_stop, - val: dont_count_stop, - }); - } else { - counter[adjusted_start] += 1; - if !dont_count_stop { - // check if the end of interval extended past region end - counter[adjusted_stop] -= 1; + // rectify start / stop with region boundaries + // increment the start of the region + let adjusted_start = if rec_start < start { + 0 + } else { + (rec_start - start) as usize + }; + + let mut dont_count_stop = false; // if this interval extends past the end of the region, don't count an end for it + let adjusted_stop = if rec_stop >= stop { + dont_count_stop = true; + counter.len() - 1 + } else { + (rec_stop - start) as usize + }; + + // check if this read has a mate that will be seen within this region + // that this works for both mates in pair + if self.mate_fix && record.2 { + // TODO: figure out better way of passing qname around, get rid of Lazy + // let qname = String::from(std::str::from_utf8(record.3).expect("Convert qname")); + let intervals = maties.entry(record.3).or_insert(vec![]); + intervals.push(Interval { + start: adjusted_start, + stop: adjusted_stop, + val: dont_count_stop, + }); + } else { + counter[adjusted_start] += 1; + if !dont_count_stop { + // check if the end of interval extended past region end + counter[adjusted_stop] -= 1; + } } } - } - if self.mate_fix { - // check maties - for (_qname, ivs) in maties.drain() { - let mut lapper = Lapper::new(ivs); - lapper.merge_overlaps(); - for iv in lapper.intervals { - counter[iv.start] += 1; - if !iv.val { - // check if the end of interval extended past region end - counter[iv.stop] -= 1; + if self.mate_fix { + // check maties + for (_qname, ivs) in maties.drain() { + let mut lapper = Lapper::new(ivs); + lapper.merge_overlaps(); + for iv in lapper.intervals { + counter[iv.start] += 1; + if !iv.val { + // check if the end of interval extended past region end + counter[iv.stop] -= 1; + } } } } - } - // Sum the counter and merge same-depth ranges of positions - let contig = std::str::from_utf8(header.tid2name(tid)).unwrap(); - self.sum_counter(counter, contig, start, stop) + // Sum the counter and merge same-depth ranges of positions + let contig = std::str::from_utf8(header.tid2name(tid)).unwrap(); + self.sum_counter(counter, contig, start, stop) + }) } fn process_region_fast(&self, tid: u32, start: u32, stop: u32) -> Vec { - // Create a reader - let mut reader = - bam::IndexedReader::from_path(&self.reads).expect("Indexed Reader for region"); - - // If passed add ref_fasta - if let Some(ref_fasta) = &self.ref_fasta { - reader.set_reference(ref_fasta).expect("Set ref"); - } - - let header = reader.header().to_owned(); - // fetch the region of interest - reader.fetch((tid, start, stop)).expect("Fetched a region"); - - let mut counter: Vec = vec![0; (stop - start) as usize]; - let mut maties = HashMap::new(); - - // Walk over each read, counting the starts and ends - for record in reader - .rc_records() - .map(|r| r.expect("Read record")) - .filter(|read| self.read_filter.filter_read(read, None)) - { - let rec_start = u32::try_from(record.reference_start()).expect("check overflow"); - let rec_stop = u32::try_from(record.reference_end()).expect("check overflow"); - - // rectify start / stop with region boundaries - // NB: impossible for rec_start > start since this is from fetch and we aren't splitting bam - let adjusted_start = if rec_start < start { - 0 - } else { - (rec_start - start) as usize - }; - - let mut dont_count_stop = false; // set this flag if this interval extends past the end of our region - let adjusted_stop = if rec_stop >= stop { - dont_count_stop = true; - counter.len() - 1 - } else { - (rec_stop - start) as usize - }; - - // check if this read has a mate that will be seen within this region - if self.mate_fix && OnlyDepth::maybe_overlaps_mate(&record) { - let qname = - String::from(std::str::from_utf8(record.qname()).expect("Convert qname")); - let intervals = maties.entry(qname).or_insert(vec![]); - intervals.push(Interval { - start: adjusted_start, - stop: adjusted_stop, - val: dont_count_stop, - }); - } else { - counter[adjusted_start] += 1; - // Check if end of interval extended past region end - if !dont_count_stop { - counter[adjusted_stop] -= 1; + utils::with_indexed_reader(&self.reads, self.ref_fasta.as_deref(), |reader| { + let header = reader.header().to_owned(); + // fetch the region of interest + reader.fetch((tid, start, stop)).expect("Fetched a region"); + + let mut counter: Vec = vec![0; (stop - start) as usize]; + let mut maties = HashMap::new(); + + // Walk over each read, counting the starts and ends + for record in reader + .rc_records() + .map(|r| r.expect("Read record")) + .filter(|read| self.read_filter.filter_read(read, None)) + { + let rec_start = u32::try_from(record.reference_start()).expect("check overflow"); + let rec_stop = u32::try_from(record.reference_end()).expect("check overflow"); + + // rectify start / stop with region boundaries + // NB: impossible for rec_start > start since this is from fetch and we aren't splitting bam + let adjusted_start = if rec_start < start { + 0 + } else { + (rec_start - start) as usize + }; + + let mut dont_count_stop = false; // set this flag if this interval extends past the end of our region + let adjusted_stop = if rec_stop >= stop { + dont_count_stop = true; + counter.len() - 1 + } else { + (rec_stop - start) as usize + }; + + // check if this read has a mate that will be seen within this region + if self.mate_fix && OnlyDepth::maybe_overlaps_mate(&record) { + let qname = + String::from(std::str::from_utf8(record.qname()).expect("Convert qname")); + let intervals = maties.entry(qname).or_insert(vec![]); + intervals.push(Interval { + start: adjusted_start, + stop: adjusted_stop, + val: dont_count_stop, + }); + } else { + counter[adjusted_start] += 1; + // Check if end of interval extended past region end + if !dont_count_stop { + counter[adjusted_stop] -= 1; + } } } - } - if self.mate_fix { - // check maties - for (_qname, ivs) in maties.drain() { - let mut lapper = Lapper::new(ivs); - lapper.merge_overlaps(); - for iv in lapper.intervals { - counter[iv.start] += 1; - // Check if end of interval extended past region end - if !iv.val { - counter[iv.stop] -= 1; + if self.mate_fix { + // check maties + for (_qname, ivs) in maties.drain() { + let mut lapper = Lapper::new(ivs); + lapper.merge_overlaps(); + for iv in lapper.intervals { + counter[iv.start] += 1; + // Check if end of interval extended past region end + if !iv.val { + counter[iv.stop] -= 1; + } } } } - } - // Sum the counter and merge same-depth ranges of positions - let contig = std::str::from_utf8(header.tid2name(tid)).unwrap(); - self.sum_counter(counter, contig, start, stop) + // Sum the counter and merge same-depth ranges of positions + let contig = std::str::from_utf8(header.tid2name(tid)).unwrap(); + self.sum_counter(counter, contig, start, stop) + }) } } diff --git a/src/lib/utils.rs b/src/lib/utils.rs index 58d2f4b..ef5d36f 100644 --- a/src/lib/utils.rs +++ b/src/lib/utils.rs @@ -4,14 +4,57 @@ use grep_cli::stdout; use gzp::{BgzfSyncReader, Compression, ZBuilder, deflate::Bgzf}; use lazy_static::lazy_static; use log::{error, warn}; +use rust_htslib::bam; use std::{ + cell::RefCell, ffi::OsStr, fs::File, io::{self, BufReader, BufWriter, Read, Write}, - path::Path, + path::{Path, PathBuf}, }; use termcolor::ColorChoice; +thread_local! { + static INDEXED_READER: RefCell, bam::IndexedReader)>> = const { RefCell::new(None) }; +} + +/// Use a lazily opened indexed BAM/CRAM reader local to the calling thread. +/// +/// Changing either input path replaces the cached reader. Callers must fetch their +/// region and finish consuming records or pileups within the callback. The reader +/// is closed when replaced or when the thread exits; callbacks must not reenter +/// this function on the same thread. +/// +/// # Panics +/// +/// Panics if the input cannot be opened, the reference cannot be set, or the +/// callback reenters this function. +pub fn with_indexed_reader( + reads: &Path, + ref_fasta: Option<&Path>, + f: impl FnOnce(&mut bam::IndexedReader) -> T, +) -> T { + INDEXED_READER.with(|cell| { + let mut slot = cell.borrow_mut(); + if slot + .as_ref() + .is_none_or(|(path, reference, _)| path != reads || reference.as_deref() != ref_fasta) + { + let mut reader = + bam::IndexedReader::from_path(reads).expect("Indexed Reader for region"); + if let Some(reference) = ref_fasta { + reader.set_reference(reference).expect("Set ref"); + } + *slot = Some(( + reads.to_path_buf(), + ref_fasta.map(Path::to_path_buf), + reader, + )); + } + f(&mut slot.as_mut().unwrap().2) + }) +} + /// Set rayon global thread pool size. /// /// # Errors @@ -156,3 +199,120 @@ lazy_static! { /// Return the number of cpus as an &str pub static ref NUM_CPU: String = num_cpus::get().to_string(); } + +#[cfg(test)] +mod tests { + use super::*; + use rust_htslib::bam::Read; + + #[test] + fn reused_cram_reader_switches_references() { + let dir = tempfile::tempdir().unwrap(); + let references = [dir.path().join("a.fa"), dir.path().join("c.fa")]; + for (path, base) in references.iter().zip(["A", "C"]) { + std::fs::write(path, format!(">chr1\n{}\n", base.repeat(100))).unwrap(); + rust_htslib::faidx::build(path).unwrap(); + } + let mut header = bam::Header::new(); + header.push_record( + bam::header::HeaderRecord::new(b"SQ") + .push_tag(b"SN", "chr1") + .push_tag(b"LN", 100) + .push_tag(b"UR", references[0].to_str().unwrap()), + ); + let path = dir.path().join("reads.cram"); + let mut writer = bam::Writer::from_path(&path, &header, bam::Format::Cram).unwrap(); + writer.set_reference(&references[0]).unwrap(); + let record = bam::Record::from_sam( + &bam::HeaderView::from_header(&header), + b"read\t0\tchr1\t11\t60\t10M\t*\t0\t0\tAAAAAAAAAA\tIIIIIIIIII", + ) + .unwrap(); + writer.write(&record).unwrap(); + drop(writer); + bam::index::build(&path, None, bam::index::Type::Bai, 1).unwrap(); + + for (reference, expected) in [ + (Some(references[0].as_path()), b'A'), + (Some(references[1].as_path()), b'C'), + (Some(references[0].as_path()), b'A'), + ] { + let sequences = with_indexed_reader(&path, reference, |reader| { + // Deliberately decode against different reference bases to make + // reference switching observable, rather than just changing paths + // to identical FASTAs. Only this test disables MD5 validation. + reader + .set_cram_options(rust_htslib::htslib::hts_fmt_option_CRAM_OPT_IGNORE_MD5, 1) + .unwrap(); + reader.fetch((0, 10, 20)).unwrap(); + reader + .records() + .map(|record| record.unwrap().seq().as_bytes()) + .collect::>() + }); + assert_eq!(sequences, vec![vec![expected; 10]]); + } + } + + #[test] + fn reused_reader_fetches_regions_and_switches_inputs() { + let dir = tempfile::tempdir().unwrap(); + let mut header = bam::Header::new(); + for name in ["chr1", "chr2"] { + header.push_record( + bam::header::HeaderRecord::new(b"SQ") + .push_tag(b"SN", name) + .push_tag(b"LN", 100), + ); + } + let view = bam::HeaderView::from_header(&header); + let paths = [dir.path().join("a.bam"), dir.path().join("b.bam")]; + for (path, base) in paths.iter().zip(["A", "T"]) { + let mut writer = bam::Writer::from_path(path, &header, bam::Format::Bam).unwrap(); + for (name, pos, seq) in [("chr1", 11, base), ("chr1", 31, "C"), ("chr2", 6, "G")] { + let record = bam::Record::from_sam( + &view, + format!("read\t0\t{name}\t{pos}\t60\t1M\t*\t0\t0\t{seq}\tI").as_bytes(), + ) + .unwrap(); + writer.write(&record).unwrap(); + } + drop(writer); + bam::index::build(path, None, bam::index::Type::Bai, 1).unwrap(); + } + + // Revisit regions backwards, after empty fetches and across contigs, + // then switch files and return to the original input on the same thread. + for (file, base) in [(0, b'A'), (1, b'T'), (0, b'A')] { + for (tid, start, stop, expected) in [ + (1, 0, 10, vec![b'G']), + (0, 30, 40, vec![b'C']), + (0, 0, 5, vec![]), + (0, 10, 20, vec![base]), + (0, 10, 20, vec![base]), + ] { + let actual = with_indexed_reader(&paths[file], None, |reader| { + reader.fetch((tid, start, stop)).unwrap(); + reader + .records() + .map(|record| record.unwrap().seq()[0]) + .collect::>() + }); + assert_eq!(actual, expected); + let positions = with_indexed_reader(&paths[file], None, |reader| { + reader.fetch((tid, start, stop)).unwrap(); + reader + .pileup() + .map(|p| p.unwrap().pos()) + .collect::>() + }); + let expected_positions = if expected.is_empty() { + vec![] + } else { + vec![if tid == 1 { 5 } else { start }] + }; + assert_eq!(positions, expected_positions); + } + } + } +}