From 192e7204433ba8b8e3d2a087f1290b9bbaea7221 Mon Sep 17 00:00:00 2001 From: Seth Stadick Date: Sat, 14 Nov 2026 21:58:20 +0000 Subject: [PATCH] Optimize depth pipelines and add upstream Seqair backend Reuse ordered output buffers and read-name groups, incorporate safe mate and formatting fast paths, and stream only-depth from reusable records. Use published Seqair 0.10.1 without the experimental fork or unchecked vector operations. Preserve depth admission and reject unsupported BAM long-CIGAR pileups explicitly. Amp-Thread-ID: https://ampcode.com/threads/T-01a1ca76-7f80-72d6-ac92-b1b4e5bacdcf Co-authored-by: Amp --- .github/workflows/rust.yml | 10 +- CHANGELOG.md | 8 + Cargo.lock | 307 +++++++++-- Cargo.toml | 6 + README.md | 39 ++ src/commands/base_depth.rs | 244 +++++++-- src/commands/base_depth/stream.rs | 265 ++++++++++ src/commands/mod.rs | 1 + src/commands/only_depth.rs | 421 ++++----------- src/commands/only_depth/stream.rs | 357 +++++++++++++ src/commands/stream.rs | 185 +++++++ src/lib/par_granges.rs | 81 +-- src/lib/position/mate_fix.rs | 172 +++--- src/lib/position/mod.rs | 2 + src/lib/position/pileup_position.rs | 201 +++++-- src/lib/position/seqair.rs | 778 ++++++++++++++++++++++++++++ src/lib/read_filter.rs | 13 +- 17 files changed, 2547 insertions(+), 543 deletions(-) create mode 100644 src/commands/base_depth/stream.rs create mode 100644 src/commands/only_depth/stream.rs create mode 100644 src/commands/stream.rs create mode 100644 src/lib/position/seqair.rs diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 8bcb7ae..f43d948 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -12,10 +12,16 @@ env: jobs: build: runs-on: ubuntu-latest + strategy: + matrix: + features: ["", "seqair"] steps: - uses: actions/checkout@v2 + - uses: dtolnay/rust-toolchain@stable + with: + toolchain: "1.98.1" - name: Build - run: cargo build --verbose + run: cargo build --locked --verbose --features "${{ matrix.features }}" - name: Run tests - run: cargo test --verbose + run: cargo test --locked --verbose --features "${{ matrix.features }}" diff --git a/CHANGELOG.md b/CHANGELOG.md index 3595d0f..982d0ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # CHANGELOG +## Unreleased + +- Performance: reuse same-name read groups across pileup positions in `base-depth --mate-fix`. +- Add an experimental upstream Seqair 0.10.1 backend for `base-depth` and `only-depth`, selected with `--seqair` when built with `--features seqair`. HTSlib remains the default. The optional backend requires Rust 1.98.1 or newer. Seqair base-depth rejects BAM long-CIGAR placeholders with an actionable error; HTSlib and Seqair only-depth handle them. +- Performance: both depth commands stream bounded, ordered TSV blocks formatted by counting workers, with safe small-integer formatting. Reuse readers, coverage arrays, and scratch space rather than retaining output rows. Small queue budgets no longer limit counting concurrency. +- Performance: `only-depth` reuses one record and merges mate intervals directly instead of building interval-query indexes. Its Seqair path requests only core fields, plus names for mate correction. +- Credit: incorporate [Rob's singleton/pair, disjoint-name, and formatting suggestions](https://gist.github.com/rob-p/a5066b01ca6ccecba0bf2eb3998b88b0) without unchecked vector writes or a Seqair fork. + ## 1.4.1 - Performance: reuse one indexed BAM/CRAM reader per worker thread in `base-depth` and both `only-depth` modes instead of reopening the input for every region ([#110](https://github.com/sstadick/perbase/issues/110)). diff --git a/Cargo.lock b/Cargo.lock index 396ff6b..e4f6674 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + [[package]] name = "ansi_term" version = "0.12.1" @@ -123,7 +129,7 @@ dependencies = [ "proc-macro2 1.0.103", "quote 1.0.42", "regex", - "rustc-hash", + "rustc-hash 1.1.0", "shlex", "syn 2.0.111", ] @@ -147,7 +153,7 @@ dependencies = [ "enum-map", "fxhash", "getset", - "itertools 0.12.1", + "itertools 0.14.0", "itertools-num", "multimap", "ndarray", @@ -162,7 +168,7 @@ dependencies = [ "statrs", "strum 0.23.0", "strum_macros 0.26.4", - "thiserror 2.0.17", + "thiserror 2.0.21", "triple_accel", "vec_map", ] @@ -207,6 +213,16 @@ version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" +[[package]] +name = "borsh" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "553c5d846a6ba5150c65e3b1b8ec073bcf1abc20f9b7220de384a4443ea4e20a" +dependencies = [ + "bytes", + "cfg_aliases", +] + [[package]] name = "bstr" version = "1.12.1" @@ -240,6 +256,26 @@ version = "0.6.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e" +[[package]] +name = "bytemuck" +version = "1.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a1f896587b6f2c069c73d2f0913e2d590c3990285cd2f0b6aa02b786b4c679c" +dependencies = [ + "proc-macro2 1.0.103", + "quote 1.0.42", + "syn 3.0.10", +] + [[package]] name = "byteorder" version = "1.5.0" @@ -252,6 +288,16 @@ version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3" +[[package]] +name = "bz2ngrs" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd61b2ae118b8491d5a6225ce870c1d35c40ad88eb4413b985f91ffb0ff14128" +dependencies = [ + "fearless_simd", + "fearless_simd_macros", +] + [[package]] name = "cc" version = "1.2.49" @@ -279,6 +325,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + [[package]] name = "clang-sys" version = "1.8.1" @@ -544,6 +596,23 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +[[package]] +name = "fearless_simd" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "629c9d6b6f30b5604089e0980f4d924a013aacb1654a6cdf5d3d852224f68651" + +[[package]] +name = "fearless_simd_macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e91d94c6885390b81782d7ee60fd6583b33907333a8b340de25ccf468353315" +dependencies = [ + "proc-macro2 1.0.103", + "quote 1.0.42", + "syn 3.0.10", +] + [[package]] name = "feature-probe" version = "0.1.1" @@ -564,13 +633,14 @@ checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" [[package]] name = "flate2" -version = "1.1.5" +version = "1.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfe33edd8e85a12a67454e37f8c75e730830d83e313556ab9ebf9ee7fbeb3bfb" +checksum = "626d4928326b646e70f65ed36fc7cf9f206bdb68a0019ee7c6b4709da4e084c4" dependencies = [ "crc32fast", "libz-ng-sys", "miniz_oxide", + "zlib-rs", ] [[package]] @@ -591,6 +661,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -752,7 +828,7 @@ dependencies = [ "libz-ng-sys", "log", "num_cpus", - "thiserror 2.0.17", + "thiserror 2.0.21", ] [[package]] @@ -761,6 +837,17 @@ version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + [[package]] name = "heck" version = "0.3.3" @@ -841,7 +928,7 @@ dependencies = [ "icu_normalizer_data", "icu_properties", "icu_provider", - "smallvec", + "smallvec 1.15.1", "zerovec", ] @@ -893,7 +980,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" dependencies = [ "idna_adapter", - "smallvec", + "smallvec 1.15.1", "utf8_iter", ] @@ -920,7 +1007,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ad4bb2b565bca0645f4d68c5c9af97fba094e9791da685bf83cb5f3ce74acf2" dependencies = [ "equivalent", - "hashbrown", + "hashbrown 0.16.1", ] [[package]] @@ -1026,18 +1113,18 @@ checksum = "37c93d8daa9d8a012fd8ab92f088405fb202ea0b6ab73ee2482ae66af4f42091" [[package]] name = "libdeflate-sys" -version = "1.25.0" +version = "1.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23bd6304ebf75390d8a99b88bdf2a266f62647838140cb64af8e6702f6e3fddc" +checksum = "d7870e5fbd2766179a937c725fb11f4ca0ef025d982beb61bd3ce755425bd19c" dependencies = [ "cc", ] [[package]] name = "libdeflater" -version = "1.25.0" +version = "1.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5d4880e6d634d3d029d65fa016038e788cc728a17b782684726fb34ee140caf" +checksum = "0cbb8285d2e2bd4b8d4155840ae81929354b4ba0d7b3c272c84761ffa2141e1f" dependencies = [ "libdeflate-sys", ] @@ -1120,6 +1207,17 @@ version = "0.11.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9106e1d747ffd48e6be5bb2d97fa706ed25b144fbee4d5c02eae110cd8d6badd" +[[package]] +name = "lzma-sys" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "matrixmultiply" version = "0.3.10" @@ -1130,6 +1228,12 @@ dependencies = [ "rawpointer", ] +[[package]] +name = "md5" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ebb8d8732c6a6df3d8f032a82911cfc747e00efb95cc46e8d0acd5b5b88570c" + [[package]] name = "memchr" version = "2.7.6" @@ -1144,9 +1248,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.8.9" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" dependencies = [ "adler2", "simd-adler32", @@ -1319,6 +1423,7 @@ dependencies = [ "grep-cli", "gzp", "itertools 0.14.0", + "itoa", "lazy_static", "log", "lru_time_cache", @@ -1328,6 +1433,8 @@ dependencies = [ "rstest", "rust-htslib", "rust-lapper", + "seqair", + "seqair-types", "serde", "smartstring", "structopt", @@ -1711,7 +1818,7 @@ dependencies = [ "linear-map", "newtype_derive", "regex", - "thiserror 2.0.17", + "thiserror 2.0.21", "url", ] @@ -1730,6 +1837,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + [[package]] name = "rustc_version" version = "0.1.7" @@ -1803,11 +1916,51 @@ version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" +[[package]] +name = "seqair" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd1fa614d8719a571613516c055cdf7e796009c111cb96146458212ef2524c3" +dependencies = [ + "bytemuck", + "bz2ngrs", + "fearless_simd", + "fearless_simd_macros", + "flate2", + "hashbrown 0.17.1", + "indexmap", + "itoa", + "libdeflater", + "md5", + "rayon-core", + "rustc-hash 2.1.3", + "seqair-types", + "thiserror 2.0.21", + "tracing", + "xz2", +] + +[[package]] +name = "seqair-types" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c927d5656e7e2df15744eefa743f3bd281dd99ac0d131cdde5de2d66ede8d78" +dependencies = [ + "bytemuck", + "fearless_simd", + "fearless_simd_macros", + "serde", + "smallvec 2.0.0-beta.4", + "smol_str", + "thiserror 2.0.21", + "winnow 1.0.5", +] + [[package]] name = "serde" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" dependencies = [ "serde_core", "serde_derive", @@ -1815,22 +1968,22 @@ dependencies = [ [[package]] name = "serde_core" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2 1.0.103", "quote 1.0.42", - "syn 2.0.111", + "syn 3.0.10", ] [[package]] @@ -1869,6 +2022,22 @@ version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +[[package]] +name = "smallvec" +version = "2.0.0-beta.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1a521aaffa9ff9b9f5c7f50a8eb60aebd11d6467d43684128d5552f6ce02003" +dependencies = [ + "serde_core", + "smallvec-macro", +] + +[[package]] +name = "smallvec-macro" +version = "0.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8aa3a1ebad7c7d912534fac818e45203fe094907e97806786285f8f440222e5a" + [[package]] name = "smartstring" version = "1.0.1" @@ -1881,6 +2050,16 @@ dependencies = [ "version_check", ] +[[package]] +name = "smol_str" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eeb7109d23f319619d4cc0de5464613c8b7b1a09db2774a58101459cf6be2f10" +dependencies = [ + "borsh", + "serde_core", +] + [[package]] name = "spin" version = "0.9.8" @@ -2018,6 +2197,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ae9ccbea3a7b0b09443036490c8f34a8b1b760fd345e5c0dae0aac718faea79" +dependencies = [ + "proc-macro2 1.0.103", + "quote 1.0.42", + "unicode-ident", +] + [[package]] name = "synstructure" version = "0.13.2" @@ -2071,11 +2261,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.17" +version = "2.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" dependencies = [ - "thiserror-impl 2.0.17", + "thiserror-impl 2.0.21", ] [[package]] @@ -2091,13 +2281,13 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.17" +version = "2.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" dependencies = [ "proc-macro2 1.0.103", "quote 1.0.42", - "syn 2.0.111", + "syn 3.0.10", ] [[package]] @@ -2128,7 +2318,7 @@ dependencies = [ "indexmap", "toml_datetime", "toml_parser", - "winnow", + "winnow 0.7.14", ] [[package]] @@ -2137,7 +2327,38 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0cbe268d35bdb4bb5a56a2de88d0ad0eb70af5384a99d648cd4b3d04039800e" dependencies = [ - "winnow", + "winnow 0.7.14", +] + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2 1.0.103", + "quote 1.0.42", + "syn 2.0.111", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", ] [[package]] @@ -2351,6 +2572,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "winnow" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c3b93199df0ab874cdf0c6a94f0a0dc3423fe41f1cafb62b8bd7b1deb277aef" +dependencies = [ + "memchr", +] + [[package]] name = "wit-bindgen" version = "0.46.0" @@ -2363,6 +2593,15 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" +[[package]] +name = "xz2" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2" +dependencies = [ + "lzma-sys", +] + [[package]] name = "yoke" version = "0.8.1" @@ -2459,3 +2698,9 @@ dependencies = [ "quote 1.0.42", "syn 2.0.111", ] + +[[package]] +name = "zlib-rs" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112" diff --git a/Cargo.toml b/Cargo.toml index a4be815..7508765 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,9 @@ path = "src/lib/mod.rs" name = "perbase" path = "src/main.rs" +[features] +seqair = ["dep:seqair", "dep:seqair-types"] + [dependencies] anyhow = "1.0.41" bio = "3.0.0" @@ -33,6 +36,7 @@ env_logger = "0.11.8" grep-cli = "0.1.5" gzp = "2.0" itertools = "0.14.0" +itoa = "1.0" lazy_static = "1.4.0" log = "0.4.11" lru_time_cache = "0.11.1" @@ -40,6 +44,8 @@ num_cpus = "1.13.0" rayon = "1.4.0" rust-htslib = {version = "0.51", default-features = false, features = ["libdeflate"]} rust-lapper = "1.0" +seqair = { version = "0.10.1", optional = true } +seqair-types = { version = "0.4.1", optional = true } serde = { version = "1.0.116", features = ["derive"] } smartstring = { version = "1.0.1", features = ["serde"] } structopt = "0.3.18" diff --git a/README.md b/README.md index c119e8a..cc4cd82 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,45 @@ The `base-depth` tool walks over every position in the BAM/CRAM file and calcula **Note on empty SEQ records:** BAM records with empty SEQ fields (represented as `*` in SAM format) are handled gracefully. These reads still count toward depth, but since the actual nucleotide cannot be determined, they are counted as `N`. +**Experimental Seqair backend:** Build with the optional `seqair` feature using +Rust 1.98.1 or newer, then pass `--seqair` to `base-depth` or `only-depth`: + +```bash +cargo +1.98.1 build --release --features seqair --locked +./target/release/perbase base-depth reads.bam --seqair --mate-fix --threads 32 +./target/release/perbase only-depth reads.bam --seqair --mate-fix --threads 32 +``` + +This uses published upstream Seqair 0.10.1 for indexed reading and pileup +generation while retaining Perbase's filters, mate-resolution strategies, and whole-read maximum-depth +admission rules. `base-depth` loads each processed region's reads into memory; +memory use depends on coverage, region size, and worker count. Auxiliary tags +are not retained. `only-depth` reuses a raw record and a coverage-difference +array, requesting read names only when fixing mate overlaps. It does not build +a pileup store or request sequence, quality, or auxiliary fields. CRAM input requires +`--ref-fasta`. Without `--seqair`, HTSlib remains the backend, including in builds +with this feature. Other subcommands remain unchanged. + +**Long-CIGAR limitation:** Seqair 0.10.1's pileup store does not expand BAM +`CG` tags. `base-depth --seqair` reports an error for these records instead of +silently counting their placeholder CIGARs. Rerun without `--seqair` to use +HTSlib. The Seqair `only-depth` raw-reader path handles these records. + +Both depth commands, with either backend, format rows in counting workers and +send bounded blocks of up to 64 KiB to a writer that drains regions in order. +Workers reuse readers and scratch allocations instead of retaining output +rows. `-C` sets the approximate queue budget in GB per worker, with a minimum +of one 64 KiB queued block per worker. Worker scratch buffers and input storage +are additional memory. Small queue budgets do not reduce the worker count. +Region boundaries and output ordering are unchanged. + +The safe small-integer formatting and singleton/pair fast paths incorporate +[Rob's optimization suggestions](https://gist.github.com/rob-p/a5066b01ca6ccecba0bf2eb3998b88b0). +The application uses ordinary reserved vectors, not unchecked pushes or custom +allocators. Seqair library optimizations remain a separate follow-up; this +build does not depend on our experimental fork. Run backend comparison tests with +`cargo +1.98.1 test --release --features seqair --locked`. + The output columns are as follows: | Column | Description | diff --git a/src/commands/base_depth.rs b/src/commands/base_depth.rs index 8302a71..05c7f8e 100644 --- a/src/commands/base_depth.rs +++ b/src/commands/base_depth.rs @@ -6,9 +6,15 @@ use anyhow::Result; use bio::io::fasta::IndexedReader; use log::*; +#[cfg(feature = "seqair")] +use perbase_lib::position::seqair::SeqairCounter; use perbase_lib::{ par_granges::{self, RegionProcessor}, - position::{Position, mate_fix::MateResolutionStrategy, pileup_position::PileupPosition}, + position::{ + Position, + mate_fix::MateResolutionStrategy, + pileup_position::{MateCounter, PileupPosition}, + }, read_filter::{DefaultReadFilter, ReadFilter}, reference, utils, }; @@ -16,6 +22,8 @@ use rust_htslib::bam::Read; use std::{convert::TryInto, path::PathBuf}; use structopt::{StructOpt, clap::AppSettings}; +mod stream; + /// Calculate the depth at each base, per-nucleotide. #[derive(StructOpt)] #[structopt(author, setting = AppSettings::ArgRequiredElseHelp)] @@ -76,6 +84,11 @@ pub struct BaseDepth { #[structopt(long, short = "m")] mate_fix: bool, + /// Use the experimental Seqair reader and pileup engine instead of HTSlib. + #[cfg(feature = "seqair")] + #[structopt(long)] + seqair: bool, + /// If `mate_fix` is true, select the method to use for mate fixing. #[structopt(long, default_value = "original")] mate_resolution_strategy: MateResolutionStrategy, @@ -119,7 +132,7 @@ impl BaseDepth { info!("Running base-depth on: {:?}", self.reads); let cpus = utils::determine_allowed_cpus(self.threads)?; - let mut writer = utils::get_writer( + let writer = utils::get_writer( &self.output, self.bgzip, true, @@ -142,6 +155,23 @@ impl BaseDepth { self.min_base_quality_score, ); + #[cfg(feature = "seqair")] + let base_processor = BaseProcessor { + seqair: if self.seqair { + Some(SeqairCounter::open( + &self.reads, + self.ref_fasta.as_deref(), + read_filter, + self.mate_fix.then_some(self.mate_resolution_strategy), + self.max_depth, + self.min_base_quality_score, + )?) + } else { + None + }, + ..base_processor + }; + let par_granges_runner = par_granges::ParGranges::new( self.reads.clone(), self.ref_fasta.clone(), @@ -154,13 +184,17 @@ impl BaseDepth { base_processor, ); - let receiver = par_granges_runner.process()?; - - for pos in receiver.into_iter() { - writer.serialize(pos)? - } - - writer.flush()?; + let (processor, regions) = par_granges_runner.into_regions()?; + let mut output = writer.into_inner().map_err(|error| error.into_error())?; + stream::run( + &processor, + ®ions, + cpus.saturating_sub(2).max(1), + self.chunksize, + self.channel_size_modifier, + &mut output, + )?; + output.flush()?; Ok(()) } } @@ -189,6 +223,8 @@ struct BaseProcessor { max_depth_warnings_cutoff: u32, /// an optional base quality score. If Some(number) if the base quality is not >= that number the base is treated as an `N` min_base_quality_score: Option, + #[cfg(feature = "seqair")] + seqair: Option, } impl BaseProcessor { @@ -225,6 +261,8 @@ impl BaseProcessor { // Set cutoff to 1% of whatever max_depth is. max_depth_warnings_cutoff: max_depth - (max_depth as f64 * 0.01) as u32, min_base_quality_score, + #[cfg(feature = "seqair")] + seqair: None, } } } @@ -241,43 +279,46 @@ impl RegionProcessor for BaseProcessor { trace!("Processing region {}(tid):{}-{}", tid, start, stop); utils::with_indexed_reader(&self.reads, self.ref_fasta.as_deref(), |reader| { let header = reader.header().to_owned(); - // fetch the region of interest - reader.fetch((tid, start, stop)).expect("Fetched a region"); - // Walk over pileups - let mut pileup = reader.pileup(); - pileup.set_max_depth(std::cmp::min(i32::MAX.try_into().unwrap(), self.max_depth)); - let result: Vec = pileup - .flat_map(|p| { - let pileup = p.expect("Extracted a pileup"); - // Verify that we are within the bounds of the chunk we are iterating on - let pileup_depth = pileup.depth(); - if pileup.pos() >= start && pileup.pos() < stop { - let mut pos = if self.mate_fix { - PileupPosition::from_pileup_mate_aware( - pileup, - &header, - &self.read_filter, - self.min_base_quality_score, - self.mate_fix_strategy, - ) + let result = { + // fetch the region of interest + reader.fetch((tid, start, stop)).expect("Fetched a region"); + // Walk over pileups + let mut pileup = reader.pileup(); + pileup.set_max_depth(std::cmp::min(i32::MAX.try_into().unwrap(), self.max_depth)); + let mut mate_counter = MateCounter::default(); + pileup + .flat_map(|p| { + let pileup = p.expect("Extracted a pileup"); + // Verify that we are within the bounds of the chunk we are iterating on + let pileup_depth = pileup.depth(); + if pileup.pos() >= start && pileup.pos() < stop { + let mut pos = if self.mate_fix { + mate_counter.count( + pileup, + &header, + &self.read_filter, + self.min_base_quality_score, + self.mate_fix_strategy, + ) + } else { + PileupPosition::from_pileup( + pileup, + &header, + &self.read_filter, + self.min_base_quality_score, + ) + }; + if pileup_depth > (self.max_depth_warnings_cutoff) { + pos.near_max_depth = true; + } + pos.pos += self.coord_base; + Some(pos) } else { - PileupPosition::from_pileup( - pileup, - &header, - &self.read_filter, - self.min_base_quality_score, - ) - }; - if pileup_depth > (self.max_depth_warnings_cutoff) { - pos.near_max_depth = true; + None } - pos.pos += self.coord_base; - Some(pos) - } else { - None - } - }) - .collect(); + }) + .collect::>() + }; let mut result = if self.keep_zeros { let mut new_result = vec![]; @@ -340,7 +381,12 @@ mod tests { ) { let dir = tempdir().unwrap(); let reference = dir.path().join("reference.fa"); - std::fs::write(&reference, ">chr1\nACGTACGTAA\n>chr2\nTTGCAATGCC\n").unwrap(); + let second = if format == "bam" { + "TTGCAATGC\"" + } else { + "TTGCAATGCC" + }; + std::fs::write(&reference, format!(">chr1\nACGTACGTAA\n>chr2\n{second}\n")).unwrap(); rust_htslib::faidx::build(&reference).unwrap(); let mut header = bam::Header::new(); for name in ["chr1", "chr2"] { @@ -395,7 +441,17 @@ mod tests { if keep_zeros { args.push("-k"); } - BaseDepth::from_iter_safe(args).unwrap().run().unwrap(); + BaseDepth::from_iter_safe(args.clone()) + .unwrap() + .run() + .unwrap(); + #[cfg(feature = "seqair")] + { + let expected = std::fs::read(&output).unwrap(); + args.push("--seqair"); + BaseDepth::from_iter_safe(args).unwrap().run().unwrap(); + assert_eq!(std::fs::read(&output).unwrap(), expected); + } let mut reader = csv::ReaderBuilder::new() .delimiter(b'\t') @@ -409,7 +465,7 @@ mod tests { .collect::, _>>() .unwrap(); let mut expected = vec![]; - for (name, sequence) in [("chr1", "ACGTACGTAA"), ("chr2", "TTGCAATGCC")] { + for (name, sequence) in [("chr1", "ACGTACGTAA"), ("chr2", second)] { for (pos, base) in sequence.chars().enumerate() { let expected_depth = u32::from(name == "chr1" && [2, 3, 6, 7].contains(&pos)); if keep_zeros || expected_depth > 0 { @@ -426,6 +482,100 @@ mod tests { } } + #[cfg(feature = "seqair")] + #[test] + fn streaming_preserves_regions_and_handles_output_failure() { + let dir = tempdir().unwrap(); + let reads = dir.path().join("reads.bam"); + let output = dir.path().join("output.tsv"); + let bed = dir.path().join("regions.bed"); + let reference = dir.path().join("reference.fa"); + std::fs::write( + &reference, + format!( + ">a\"quoted\n{}\n>b\n{}\n", + "acgTN".repeat(6000), + "G".repeat(19) + ), + ) + .unwrap(); + rust_htslib::faidx::build(&reference).unwrap(); + let mut header = bam::Header::new(); + for (name, length) in [("a\"quoted", 30000), ("b", 19)] { + header.push_record( + bam::header::HeaderRecord::new(b"SQ") + .push_tag(b"SN", name) + .push_tag(b"LN", length), + ); + } + let view = bam::HeaderView::from_header(&header); + let mut writer = bam::Writer::from_path(&reads, &header, bam::Format::Bam).unwrap(); + writer + .write( + &Record::from_sam(&view, b"read\t0\ta\"quoted\t3\t60\t4M\t*\t0\t0\tACGT\tIIII") + .unwrap(), + ) + .unwrap(); + drop(writer); + bam::index::build(&reads, None, bam::index::Type::Bai, 1).unwrap(); + std::fs::write( + &bed, + "b\t0\t19\na\"quoted\t0\t20000\na\"quoted\t15000\t25000\na\"quoted\t50\t50\n", + ) + .unwrap(); + let run = |seqair: bool, budget: &str, target: &std::path::Path, with_reference: bool| { + let mut args = vec![ + "base-depth", + reads.to_str().unwrap(), + "-b", + bed.to_str().unwrap(), + "-M", + "-k", + "-m", + "-D", + "1", + "-t", + "6", + "-c", + "5000", + "-C", + budget, + "-o", + target.to_str().unwrap(), + ]; + if seqair { + args.push("--seqair"); + } + if with_reference { + args.extend(["-r", reference.to_str().unwrap()]); + } + BaseDepth::from_iter_safe(args).unwrap().run() + }; + // Multiple blocks per region, duplicate positions, empty regions, + // slot reuse, and both single-slot and parallel execution. + for with_reference in [false, true] { + run(false, "0.15", &output, with_reference).unwrap(); + let expected = std::fs::read(&output).unwrap(); + for budget in ["0", "0.15"] { + for seqair in [false, true] { + run(seqair, budget, &output, with_reference).unwrap(); + assert_eq!(std::fs::read(&output).unwrap(), expected); + if std::path::Path::new("/dev/full").exists() { + assert!( + run( + seqair, + budget, + std::path::Path::new("/dev/full"), + with_reference + ) + .is_err() + ); + } + } + } + } + } + #[fixture] fn read_filter() -> DefaultReadFilter { DefaultReadFilter::new(0, 512, 0) diff --git a/src/commands/base_depth/stream.rs b/src/commands/base_depth/stream.rs new file mode 100644 index 0000000..bbd355a --- /dev/null +++ b/src/commands/base_depth/stream.rs @@ -0,0 +1,265 @@ +//! Count and format in workers; write bounded byte blocks in region order. +use super::{BaseProcessor, DefaultReadFilter, MateCounter, PileupPosition}; +use crate::commands::stream::{self, Config, Output, append_unsigned}; +use anyhow::{Context, Result}; +#[cfg(feature = "seqair")] +use perbase_lib::position::seqair::SeqairWorker; +use perbase_lib::{par_granges::Region, utils}; +use rust_htslib::bam::Read; +use std::io::Write; + +#[cfg(not(feature = "seqair"))] +type Worker = (); +#[cfg(feature = "seqair")] +type Worker<'a> = Option>; + +const COLUMNS: &[u8] = b"DEPTH\tA\tC\tG\tT\tN\tR\tY\tS\tW\tK\tM\tINS\tDEL\tREF_SKIP\tFAIL\tCOUNT_OF_MATE_RESOLUTIONS\tNEAR_MAX_DEPTH\n"; + +pub(super) fn run( + processor: &BaseProcessor, + regions: &[Region], + workers: usize, + chunksize: u32, + modifier: f64, + output: &mut dyn Write, +) -> Result<()> { + let names = + utils::with_indexed_reader(&processor.reads, processor.ref_fasta.as_deref(), |reader| { + reader + .header() + .target_names() + .iter() + .map(|name| String::from_utf8(name.to_vec())) + .collect::, _>>() + })?; + let longest = names.iter().map(String::len).max().unwrap_or(0); + let mut header = b"REF\tPOS\t".to_vec(); + if processor.ref_fasta.is_some() { + header.extend_from_slice(b"REF_BASE\t"); + } + header.extend_from_slice(COLUMNS); + stream::run( + regions, + Config { + workers, + chunksize, + modifier, + estimated_row_bytes: longest + 64, + }, + &header, + output, + || { + #[cfg(feature = "seqair")] + { + processor + .seqair + .as_ref() + .map(|counter| counter.worker()) + .transpose() + } + #[cfg(not(feature = "seqair"))] + { + Ok(()) + } + }, + |worker, region, output| write_region(processor, worker, region, &names, output), + ) +} + +fn write_region( + processor: &BaseProcessor, + worker: &mut Worker, + (tid, start, stop): Region, + names: &[String], + output: &mut Output<'_>, +) -> Result<()> { + let name = &names[tid as usize]; + let quoted = stream::quote_name(name); + // Fetch lazily: an empty region without --keep-zeros needs no reference. + let mut reference = None; + let mut emit = |pos: u32, row: &PileupPosition| -> Result<()> { + let base = if let Some(buffer) = &processor.ref_buffer { + if reference.is_none() { + reference = Some(buffer.seq(name)?); + } + Some(char::from( + *reference + .as_ref() + .unwrap() + .get(pos as usize) + .context("Input SAM does not match reference")?, + )) + } else { + None + }; + append_row(output.text, "ed, pos + processor.coord_base, base, row); + output.flush() + }; + let mut next = start; + let zero = PileupPosition::default(); + let mut count_row = |row: &PileupPosition| { + if processor.keep_zeros { + while next < row.pos { + emit(next, &zero)?; + next += 1; + } + } + next = row.pos + 1; + emit(row.pos, row) + }; + #[cfg(feature = "seqair")] + let used_seqair = if let Some(worker) = worker { + worker.for_each_position_ref(tid, start, stop, &mut count_row)?; + true + } else { + false + }; + #[cfg(not(feature = "seqair"))] + let used_seqair = { + let _ = worker; + false + }; + if !used_seqair && start < stop { + // Use the same scheduling, formatting, and output buffers for both + // backends. Only reading, pileup generation, and counting differ. + utils::with_indexed_reader( + &processor.reads, + processor.ref_fasta.as_deref(), + |reader| -> Result<()> { + let header = reader.header().to_owned(); + reader.fetch((tid, start, stop))?; + let mut pileups = reader.pileup(); + pileups.set_max_depth(processor.max_depth.min(i32::MAX as u32)); + // Name groups must not accumulate across an entire genome. + let mut mates = MateCounter::default(); + for pileup in pileups { + let pileup = pileup?; + if pileup.pos() < start || pileup.pos() >= stop { + continue; + } + let depth = pileup.depth(); + let mut row = if processor.mate_fix { + mates.count( + pileup, + &header, + &processor.read_filter, + processor.min_base_quality_score, + processor.mate_fix_strategy, + ) + } else { + PileupPosition::from_pileup( + pileup, + &header, + &processor.read_filter, + processor.min_base_quality_score, + ) + }; + row.near_max_depth = depth > processor.max_depth_warnings_cutoff; + count_row(&row)?; + } + Ok(()) + }, + )?; + } + if processor.keep_zeros { + while next < stop { + emit(next, &zero)?; + next += 1; + } + } + Ok(()) +} + +fn append_row(text: &mut Vec, name: &[u8], pos: u32, base: Option, row: &PileupPosition) { + text.extend_from_slice(name); + text.push(b'\t'); + append_unsigned(text, pos); + if let Some(base) = base { + text.push(b'\t'); + match base { + '"' => text.extend_from_slice(b"\"\"\"\""), + '\t' | '\r' | '\n' => text.extend_from_slice(&[b'"', base as u8, b'"']), + _ => text.extend_from_slice(base.encode_utf8(&mut [0; 4]).as_bytes()), + } + } + for value in [ + row.depth, + row.a, + row.c, + row.g, + row.t, + row.n, + row.r, + row.y, + row.s, + row.w, + row.k, + row.m, + row.ins, + row.del, + row.ref_skip, + row.fail, + row.count_of_mate_resolutions, + ] { + text.push(b'\t'); + append_unsigned(text, value); + } + text.extend_from_slice(if row.near_max_depth { + b"\ttrue\n" + } else { + b"\tfalse\n" + }); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rows_match_csv_at_decimal_boundaries() { + let mut values: Vec = (0..=101).collect(); + for power in [1_000, 10_000, 100_000, 1_000_000, 1_000_000_000] { + values.extend([power - 1, power, power + 1]); + } + values.push(u32::MAX); + for value in values { + for base in [None, Some('A'), Some('"'), Some('\t'), Some('é')] { + let row = PileupPosition { + ref_seq: "chr20".into(), + pos: value, + ref_base: base, + depth: value, + a: 1, + c: 9, + g: 10, + t: 99, + n: 100, + r: 101, + y: 999, + s: 1_000, + w: 10_000, + k: 100_000, + m: 1_000_000, + ins: 2, + del: 3, + ref_skip: 4, + fail: 5, + count_of_mate_resolutions: 6, + near_max_depth: value % 2 == 0, + }; + let mut actual = Vec::new(); + append_row(&mut actual, b"chr20", row.pos, base, &row); + let mut csv = csv::WriterBuilder::new() + .delimiter(b'\t') + .has_headers(false) + .from_writer(Vec::new()); + csv.serialize(&row).unwrap(); + assert_eq!( + actual, + csv.into_inner().unwrap(), + "value={value}, base={base:?}" + ); + } + } + } +} diff --git a/src/commands/mod.rs b/src/commands/mod.rs index 72e768a..16ec595 100644 --- a/src/commands/mod.rs +++ b/src/commands/mod.rs @@ -1,3 +1,4 @@ pub mod base_depth; pub mod merge_adjacent; pub mod only_depth; +mod stream; diff --git a/src/commands/only_depth.rs b/src/commands/only_depth.rs index e1912f3..de1e562 100644 --- a/src/commands/only_depth.rs +++ b/src/commands/only_depth.rs @@ -7,20 +7,16 @@ use anyhow::Result; use log::*; use perbase_lib::{ par_granges::{self, RegionProcessor}, - position::{ - Position, - range_positions::{BedFormatRangePositions, RangePositions}, - }, + position::range_positions::RangePositions, read_filter::{DefaultReadFilter, ReadFilter}, utils, }; -use rust_htslib::{bam, bam::Read, bam::ext::BamRecordExtensions, bam::record::Cigar}; -use rust_lapper::{Interval, Lapper}; -use smartstring::alias::String; -use std::{collections::HashMap, path::PathBuf}; -use std::{convert::TryFrom, rc::Rc}; +use rust_htslib::{bam, bam::ext::BamRecordExtensions}; +use std::path::PathBuf; use structopt::{StructOpt, clap::AppSettings}; +mod stream; + /// Calculate only the depth at each base. #[derive(StructOpt)] #[structopt(author, setting = AppSettings::ArgRequiredElseHelp)] @@ -112,6 +108,11 @@ pub struct OnlyDepth { /// Output positions as 0-based instead of 1-based. #[structopt(long, short = "z")] zero_base: bool, + + /// Read alignments with Seqair instead of HTSlib. + #[cfg(feature = "seqair")] + #[structopt(long)] + seqair: bool, } impl OnlyDepth { @@ -119,7 +120,7 @@ impl OnlyDepth { info!("Running only-depth on: {:?}", self.reads); let cpus = utils::determine_allowed_cpus(self.threads)?; - let mut writer = utils::get_writer( + let writer = utils::get_writer( &self.output, self.bgzip, !self.bed_format, @@ -152,18 +153,23 @@ impl OnlyDepth { processor, ); - let receiver = par_granges_runner.process()?; - - if self.bed_format { - for pos in receiver.into_iter() { - writer.serialize(BedFormatRangePositions::from(pos))? - } - } else { - for pos in receiver.into_iter() { - writer.serialize(pos)? - } - } - writer.flush()?; + let (processor, regions) = par_granges_runner.into_regions()?; + let mut output = writer.into_inner().map_err(|error| error.into_error())?; + #[cfg(feature = "seqair")] + let use_seqair = self.seqair; + #[cfg(not(feature = "seqair"))] + let use_seqair = false; + stream::run( + &processor, + ®ions, + cpus.saturating_sub(2).max(1), + self.chunksize, + self.channel_size_modifier, + self.bed_format, + use_seqair, + &mut output, + )?; + output.flush()?; Ok(()) } @@ -184,7 +190,7 @@ impl OnlyDepth { } /// Holds the info needed for [par_io::RegionProcessor] implementation -struct OnlyDepthProcessor { +struct OnlyDepthProcessor { /// path to indexed BAM/CRAM reads: PathBuf, /// path to indexed ref file @@ -198,12 +204,12 @@ struct OnlyDepthProcessor { /// Indicate whether or not to keep positions with 0 depth keep_zeros: bool, /// implementation of [position::ReadFilter] that will be used - read_filter: F, + read_filter: DefaultReadFilter, /// 0-based or 1-based coordinate output coord_base: u32, } -impl OnlyDepthProcessor { +impl OnlyDepthProcessor { /// Create a new OnlyDepthProcessor #[allow(clippy::too_many_arguments)] fn new( @@ -214,7 +220,7 @@ impl OnlyDepthProcessor { no_merge: bool, keep_zeros: bool, coord_base: u32, - read_filter: F, + read_filter: DefaultReadFilter, ) -> Self { Self { reads, @@ -227,293 +233,15 @@ impl OnlyDepthProcessor { read_filter, } } - - /// Sum the counts within the region to get the depths at each RangePosition - #[inline] - fn sum_counter( - &self, - counter: Vec, - contig: &str, - region_start: u32, - region_stop: u32, - ) -> Vec { - if self.no_merge { - let mut sum: i32 = 0; - let mut results = vec![]; - for (i, count) in counter.iter().enumerate() { - sum += count; - if sum != 0 || self.keep_zeros { - let mut pos = RangePositions::new( - String::from(contig), - region_start + i as u32 + self.coord_base, - ); - pos.depth = u32::try_from(sum).expect("All depths are positive"); - pos.end = region_start + i as u32 + self.coord_base + 1; - results.push(pos); - } - } - results - } else { - // Sum the counter and merge same-depth ranges of positions - let mut sum: i32 = 0; - let mut results = vec![]; - let mut curr_start = region_start; - let mut curr_depth = counter[0]; - for (i, count) in counter.iter().enumerate() { - sum += count; - // freeze pos and start new one - if curr_depth != sum { - let mut pos = - RangePositions::new(String::from(contig), curr_start + self.coord_base); - pos.depth = u32::try_from(curr_depth).expect("All depths are positive"); - pos.end = region_start + i as u32 + self.coord_base; - - curr_start = region_start + i as u32; - curr_depth = sum; - results.push(pos); - } - - if i == counter.len() - 1 { - // We've hit the end - let mut pos = - RangePositions::new(String::from(contig), curr_start + self.coord_base); - pos.depth = u32::try_from(curr_depth).expect("All depths are positive"); - pos.end = region_stop + self.coord_base; - results.push(pos); - } - } - results - } - } - - /// Process a region, taking into account REF_SKIPs and mates - fn process_region(&self, tid: u32, start: u32, stop: u32) -> Vec { - utils::with_indexed_reader(&self.reads, self.ref_fasta.as_deref(), |reader| { - let header = reader.header().to_owned(); - // fetch the region of interest - reader.fetch((tid, start, stop)).expect("Fetched a region"); - - let mut counter: Vec = vec![0; (stop - start) as usize]; - let mut maties = HashMap::new(); - - // Walk over each read, counting the starts and ends - for record in reader - .rc_records() - .map(|r| r.expect("Read record")) - .filter(|read| self.read_filter.filter_read(read, None)) - .flat_map(|record| IterAlignedBlocks::new(record, self.mate_fix)) - { - let rec_start = u32::try_from(record.0).expect("check overflow"); - let rec_stop = u32::try_from(record.1).expect("check overflow"); - - // NB: since we are splitting the region, it's possible the region we are looking at - // may occur before the ROI, or after the ROI - if rec_start >= stop || rec_stop <= start { - continue; - } - - // rectify start / stop with region boundaries - // increment the start of the region - let adjusted_start = if rec_start < start { - 0 - } else { - (rec_start - start) as usize - }; - - let mut dont_count_stop = false; // if this interval extends past the end of the region, don't count an end for it - let adjusted_stop = if rec_stop >= stop { - dont_count_stop = true; - counter.len() - 1 - } else { - (rec_stop - start) as usize - }; - - // check if this read has a mate that will be seen within this region - // that this works for both mates in pair - if self.mate_fix && record.2 { - // TODO: figure out better way of passing qname around, get rid of Lazy - // let qname = String::from(std::str::from_utf8(record.3).expect("Convert qname")); - let intervals = maties.entry(record.3).or_insert(vec![]); - intervals.push(Interval { - start: adjusted_start, - stop: adjusted_stop, - val: dont_count_stop, - }); - } else { - counter[adjusted_start] += 1; - if !dont_count_stop { - // check if the end of interval extended past region end - counter[adjusted_stop] -= 1; - } - } - } - - if self.mate_fix { - // check maties - for (_qname, ivs) in maties.drain() { - let mut lapper = Lapper::new(ivs); - lapper.merge_overlaps(); - for iv in lapper.intervals { - counter[iv.start] += 1; - if !iv.val { - // check if the end of interval extended past region end - counter[iv.stop] -= 1; - } - } - } - } - - // Sum the counter and merge same-depth ranges of positions - let contig = std::str::from_utf8(header.tid2name(tid)).unwrap(); - self.sum_counter(counter, contig, start, stop) - }) - } - - fn process_region_fast(&self, tid: u32, start: u32, stop: u32) -> Vec { - utils::with_indexed_reader(&self.reads, self.ref_fasta.as_deref(), |reader| { - let header = reader.header().to_owned(); - // fetch the region of interest - reader.fetch((tid, start, stop)).expect("Fetched a region"); - - let mut counter: Vec = vec![0; (stop - start) as usize]; - let mut maties = HashMap::new(); - - // Walk over each read, counting the starts and ends - for record in reader - .rc_records() - .map(|r| r.expect("Read record")) - .filter(|read| self.read_filter.filter_read(read, None)) - { - let rec_start = u32::try_from(record.reference_start()).expect("check overflow"); - let rec_stop = u32::try_from(record.reference_end()).expect("check overflow"); - - // rectify start / stop with region boundaries - // NB: impossible for rec_start > start since this is from fetch and we aren't splitting bam - let adjusted_start = if rec_start < start { - 0 - } else { - (rec_start - start) as usize - }; - - let mut dont_count_stop = false; // set this flag if this interval extends past the end of our region - let adjusted_stop = if rec_stop >= stop { - dont_count_stop = true; - counter.len() - 1 - } else { - (rec_stop - start) as usize - }; - - // check if this read has a mate that will be seen within this region - if self.mate_fix && OnlyDepth::maybe_overlaps_mate(&record) { - let qname = - String::from(std::str::from_utf8(record.qname()).expect("Convert qname")); - let intervals = maties.entry(qname).or_insert(vec![]); - intervals.push(Interval { - start: adjusted_start, - stop: adjusted_stop, - val: dont_count_stop, - }); - } else { - counter[adjusted_start] += 1; - // Check if end of interval extended past region end - if !dont_count_stop { - counter[adjusted_stop] -= 1; - } - } - } - - if self.mate_fix { - // check maties - for (_qname, ivs) in maties.drain() { - let mut lapper = Lapper::new(ivs); - lapper.merge_overlaps(); - for iv in lapper.intervals { - counter[iv.start] += 1; - // Check if end of interval extended past region end - if !iv.val { - counter[iv.stop] -= 1; - } - } - } - } - - // Sum the counter and merge same-depth ranges of positions - let contig = std::str::from_utf8(header.tid2name(tid)).unwrap(); - self.sum_counter(counter, contig, start, stop) - }) - } } -/// Implement [par_io::RegionProcessor] for [SimpleProcessor] -impl RegionProcessor for OnlyDepthProcessor { - /// Objects of [position::Position] will be returned by each call to [SimpleProcessor::process_region] +/// Collect rows for callers of the generic region-processing interface. +/// The CLI uses the same counter but writes each row immediately. +impl RegionProcessor for OnlyDepthProcessor { type P = RangePositions; - /// Process a region by fetching it from a BAM/CRAM, getting a pileup, and then - /// walking the pileup (checking bounds) to create Position objects according to - /// the defined filters fn process_region(&self, tid: u32, start: u32, stop: u32) -> Vec { - trace!("Processing region {}(tid):{}-{}", tid, start, stop); - if self.fast_mode { - self.process_region_fast(tid, start, stop) - } else { - self.process_region(tid, start, stop) - } - } -} - -// A tweaked impl of IterAlignedBlocks from [here](https://github.com/rust-bio/rust-htslib/blob/9175d3ca186baef4f84a7d7ccb27869b43471e36/src/bam/ext.rs#L51) -// Not that this will also hang onto the bam::Record and supplies the qname for each thing returned. -// At the end of the day this shouldn't be the worst since any given read should not have that many splits in it -struct IterAlignedBlocks { - pos: i64, - cigar_index: usize, - cigar: bam::record::CigarStringView, - overlap_status: bool, - record: Rc, -} -impl IterAlignedBlocks { - fn new(record: Rc, check_mate: bool) -> Self { - let overlap = if check_mate { - OnlyDepth::maybe_overlaps_mate(&record) - } else { - false - }; - Self { - pos: record.reference_start(), - cigar_index: 0, - cigar: record.cigar(), - overlap_status: overlap, - record, - } - } -} - -impl Iterator for IterAlignedBlocks { - type Item = (i64, i64, bool, String); - fn next(&mut self) -> Option { - while self.cigar_index < self.cigar.len() { - let entry = self.cigar[self.cigar_index]; - match entry { - Cigar::Match(len) | Cigar::Equal(len) | Cigar::Diff(len) | Cigar::Del(len) => { - let out_pos = self.pos; - self.pos += len as i64; - self.cigar_index += 1; - return Some(( - out_pos, - out_pos + len as i64, - self.overlap_status, - String::from( - std::str::from_utf8(self.record.qname()).expect("Convert qname"), - ), - )); - } - Cigar::RefSkip(len) => self.pos += len as i64, - _ => (), - } - self.cigar_index += 1; - } - None + stream::collect(self, (tid, start, stop)).expect("count region") } } @@ -531,6 +259,83 @@ mod tests { use std::{collections::HashMap, path::PathBuf}; use tempfile::{TempDir, tempdir}; + #[test] + fn streaming_depth_matches_cigar_coverage() { + let directory = tempdir().unwrap(); + let reads = directory.path().join("reads.bam"); + let output = directory.path().join("depth.tsv"); + let mut header = bam::Header::new(); + header.push_record( + bam::header::HeaderRecord::new(b"SQ") + .push_tag(b"SN", "a") + .push_tag(b"LN", 12), + ); + let view = bam::HeaderView::from_header(&header); + let mut writer = bam::Writer::from_path(&reads, &header, bam::Format::Bam).unwrap(); + for sam in [ + "pair\t65\ta\t1\t60\t4M3N4M\t=\t3\t11\tAAAAAAAA\tIIIIIIII", + "pair\t129\ta\t3\t60\t6M\t=\t1\t-11\tAAAAAA\tIIIIII", + "filtered\t512\ta\t4\t60\t2M\t*\t0\t0\tAA\tII", + "single\t0\ta\t6\t60\t2M2D2M\t*\t0\t0\tAAAA\tIIII", + ] { + writer + .write(&Record::from_sam(&view, sam.as_bytes()).unwrap()) + .unwrap(); + } + drop(writer); + bam::index::build(&reads, None, bam::index::Type::Bai, 1).unwrap(); + for (mode, depths) in [ + (None, [1, 1, 2, 2, 1, 2, 2, 3, 2, 2, 2, 0]), + (Some("-m"), [1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 0]), + (Some("-x"), [1, 1, 2, 2, 2, 3, 3, 3, 2, 2, 2, 0]), + ] { + for bed in [false, true] { + for seqair in [false, cfg!(feature = "seqair")] { + let mut args = vec![ + "only-depth", + reads.to_str().unwrap(), + "-n", + "-k", + "-z", + "-F", + "512", + "-c", + "12", + "-t", + "4", + "-o", + output.to_str().unwrap(), + ]; + args.extend(mode); + if bed { + args.push("--bed-format"); + } + if seqair { + args.push("--seqair"); + } + OnlyDepth::from_iter_safe(args).unwrap().run().unwrap(); + let mut expected = if bed { + std::string::String::new() + } else { + "REF\tPOS\tEND\tDEPTH\n".to_owned() + }; + for (pos, depth) in depths.iter().enumerate() { + expected.push_str(&format!( + "a\t{pos}\t{}\t{}{depth}\n", + pos + 1, + if bed { "\t" } else { "" } + )); + } + assert_eq!( + std::fs::read_to_string(&output).unwrap(), + expected, + "mode={mode:?}, bed={bed}, seqair={seqair}" + ); + } + } + } + } + #[fixture] fn read_filter() -> DefaultReadFilter { DefaultReadFilter::new(0, 512, 0) diff --git a/src/commands/only_depth/stream.rs b/src/commands/only_depth/stream.rs new file mode 100644 index 0000000..ac1363f --- /dev/null +++ b/src/commands/only_depth/stream.rs @@ -0,0 +1,357 @@ +//! Reuse readers and coverage arrays; emit depth rows without retaining them. +use super::{OnlyDepth, OnlyDepthProcessor, ReadFilter}; +use crate::commands::stream::{self, Config, Output}; +#[cfg(feature = "seqair")] +use anyhow::Context; +use anyhow::Result; +use perbase_lib::{ + par_granges::Region, + position::{Position, range_positions::RangePositions}, + utils, +}; +use rust_htslib::bam::{self, Read, ext::BamRecordExtensions}; +use std::{collections::HashMap, io::Write}; + +#[cfg(feature = "seqair")] +use seqair::reader::{ + IndexedReader, PartialIndexedReader, PartialRecord, RecordFields, ResolveTid, +}; +#[cfg(feature = "seqair")] +use seqair_types::Pos0; + +enum Reader { + Htslib(bam::IndexedReader, bam::Record), + #[cfg(feature = "seqair")] + Seqair(PartialIndexedReader, PartialRecord), +} + +#[derive(Clone, Copy)] +struct Block { + start: usize, + stop: usize, + no_stop: bool, +} + +#[derive(Default)] +struct Counter { + delta: Vec, + groups: HashMap, Vec>, +} + +impl Counter { + fn reset(&mut self, length: usize) { + self.delta.clear(); + self.delta.resize(length, 0); + self.groups.clear(); + } + + fn count(delta: &mut [i64], block: Block) { + delta[block.start] += 1; + if !block.no_stop { + delta[block.stop] -= 1; + } + } + + fn add(&mut self, name: Option<&[u8]>, begin: i64, end: i64, range: (u32, u32)) { + let (start, stop) = (i64::from(range.0), i64::from(range.1)); + if begin >= stop || end <= start { + return; + } + let block = Block { + start: (begin.max(start) - start) as usize, + stop: (end - start).min(stop - start - 1) as usize, + no_stop: end >= stop, + }; + if let Some(name) = name { + // Borrow for lookup so the second mate does not allocate a name. + if let Some(intervals) = self.groups.get_mut(name) { + intervals.push(block); + } else { + self.groups.insert(name.to_vec(), vec![block]); + } + } else { + Self::count(&mut self.delta, block); + } + } + + fn merge_mates(&mut self) { + for intervals in self.groups.values_mut() { + // Stable order preserves the first interval's clipped-end flag, + // matching rust-lapper even when later intervals extend further. + if intervals.len() > 1 { + intervals.sort_by_key(|block| (block.start, block.stop)); + } + let mut current = intervals[0]; + for block in &intervals[1..] { + if block.start <= current.stop { + current.stop = current.stop.max(block.stop); + } else { + Self::count(&mut self.delta, current); + current = *block; + } + } + Self::count(&mut self.delta, current); + } + } + + fn emit( + &self, + processor: &OnlyDepthProcessor, + start: u32, + stop: u32, + mut row: impl FnMut(u32, u32, u32) -> Result<()>, + ) -> Result<()> { + let mut sum = 0; + let mut begin = start; + let mut depth = self.delta[0]; + for (offset, change) in self.delta.iter().enumerate() { + sum += change; + let pos = start + offset as u32; + if processor.no_merge { + if sum != 0 || processor.keep_zeros { + row(pos, pos + 1, u32::try_from(sum)?)?; + } + } else if depth != sum { + row(begin, pos, u32::try_from(depth)?)?; + begin = pos; + depth = sum; + } + } + if !processor.no_merge { + row(begin, stop, u32::try_from(depth)?)?; + } + Ok(()) + } +} + +#[allow(clippy::too_many_arguments)] +pub(super) fn run( + processor: &OnlyDepthProcessor, + regions: &[Region], + workers: usize, + chunksize: u32, + modifier: f64, + bed: bool, + use_seqair: bool, + output: &mut dyn Write, +) -> Result<()> { + let names = + utils::with_indexed_reader(&processor.reads, processor.ref_fasta.as_deref(), |r| { + r.header() + .target_names() + .iter() + .map(|name| String::from_utf8(name.to_vec())) + .collect::, _>>() + })?; + let longest = names.iter().map(String::len).max().unwrap_or(0); + #[cfg(feature = "seqair")] + let template = if use_seqair { + let fields = if processor.mate_fix { + RecordFields::core().with_names() + } else { + RecordFields::core() + }; + let mut builder = IndexedReader::builder(&processor.reads).fields(fields); + if let Some(reference) = &processor.ref_fasta { + builder = builder.reference(reference); + } + Some(builder.open()?) + } else { + None + }; + #[cfg(not(feature = "seqair"))] + let _ = use_seqair; + stream::run( + regions, + Config { + workers, + chunksize, + modifier, + estimated_row_bytes: longest + 32, + }, + if bed { b"" } else { b"REF\tPOS\tEND\tDEPTH\n" }, + output, + || { + #[cfg(feature = "seqair")] + if let Some(template) = &template { + return Ok(( + Reader::Seqair(template.fork()?, PartialRecord::default()), + Counter::default(), + )); + } + let mut reader = bam::IndexedReader::from_path(&processor.reads)?; + if let Some(reference) = &processor.ref_fasta { + reader.set_reference(reference)?; + } + Ok(( + Reader::Htslib(reader, bam::Record::new()), + Counter::default(), + )) + }, + |(reader, counter), (tid, start, stop), output| { + if start == stop { + return Ok(()); + } + counter.reset((stop - start) as usize); + load(reader, counter, processor, (tid, start, stop))?; + counter.merge_mates(); + let name = stream::quote_name(&names[tid as usize]); + counter.emit(processor, start, stop, |begin, end, depth| { + append_row( + output, + &name, + begin + processor.coord_base, + end + processor.coord_base, + depth, + bed, + ) + }) + }, + ) +} + +fn load( + reader: &mut Reader, + counter: &mut Counter, + processor: &OnlyDepthProcessor, + (tid, start, stop): Region, +) -> Result<()> { + match reader { + Reader::Htslib(reader, record) => { + reader.fetch((tid, start, stop))?; + while let Some(result) = reader.read(record) { + result?; + if !processor.read_filter.filter_read(record, None) { + continue; + } + let mut pos = record.reference_start(); + let name = (processor.mate_fix && OnlyDepth::maybe_overlaps_mate(record)) + .then(|| record.qname()); + if processor.fast_mode { + counter.add(name, pos, record.reference_end(), (start, stop)); + continue; + } + for op in record.cigar().iter() { + let end = pos + i64::from(op.len()); + match op.char() { + 'M' | '=' | 'X' | 'D' => { + counter.add(name, pos, end, (start, stop)); + pos = end; + } + 'N' => pos = end, + _ => (), + } + } + } + } + #[cfg(feature = "seqair")] + Reader::Seqair(reader, record) => { + let tid = tid.resolve_tid(reader.header())?; + let first = + Pos0::new(start).context("region start exceeds Seqair's coordinate range")?; + let last = + Pos0::new(stop - 1).context("region end exceeds Seqair's coordinate range")?; + let mut query = reader.query_raw(tid, (first..=last).into())?; + while query.read_record(record)? { + if !processor + .read_filter + .filter_fields(record.flags().raw(), record.mapq().as_byte()) + { + continue; + } + let mut pos = record + .pos() + .context("indexed record has no position")? + .as_i64(); + let end = record + .end_pos() + .context("indexed record has no end")? + .as_i64() + + 1; + if processor.fast_mode { + counter.add(None, pos, end, (start, stop)); + continue; + } + let mate = record.mate_fields(); + let overlaps = processor.mate_fix + && record.flags().raw() & 1 != 0 + && record.tid() == mate.tid + && (i64::from(mate.template_len.map_or(0, |len| len.get())).abs() / 2 + < end - pos + || (mate.pos.map_or(-1, |pos| pos.as_i64()) - pos).abs() < end - pos); + let name = if overlaps { + Some(record.qname()?) + } else { + None + }; + for op in record.cigar() { + let end = pos + i64::from(op.len()); + match op.op_code() { + 0 | 2 | 7 | 8 => { + counter.add(name, pos, end, (start, stop)); + pos = end; + } + 3 => pos = end, + _ => (), + } + } + } + } + } + Ok(()) +} + +pub(super) fn collect( + processor: &OnlyDepthProcessor, + region: Region, +) -> Result> { + let (tid, start, stop) = region; + if start == stop { + return Ok(Vec::new()); + } + let mut reader = bam::IndexedReader::from_path(&processor.reads)?; + if let Some(reference) = &processor.ref_fasta { + reader.set_reference(reference)?; + } + let name = std::str::from_utf8(reader.header().tid2name(tid))?.to_owned(); + let mut counter = Counter::default(); + counter.reset((stop - start) as usize); + load( + &mut Reader::Htslib(reader, bam::Record::new()), + &mut counter, + processor, + region, + )?; + counter.merge_mates(); + let mut rows = Vec::new(); + counter.emit(processor, start, stop, |begin, end, depth| { + let mut row = RangePositions::new(name.as_str().into(), begin + processor.coord_base); + row.end = end + processor.coord_base; + row.depth = depth; + rows.push(row); + Ok(()) + })?; + Ok(rows) +} + +fn append_row( + output: &mut Output<'_>, + name: &[u8], + start: u32, + stop: u32, + depth: u32, + bed: bool, +) -> Result<()> { + output.text.extend_from_slice(name); + for value in [start, stop] { + output.text.push(b'\t'); + stream::append_unsigned(output.text, value); + } + if bed { + output.text.push(b'\t'); + } + output.text.push(b'\t'); + stream::append_unsigned(output.text, depth); + output.text.push(b'\n'); + output.flush() +} diff --git a/src/commands/stream.rs b/src/commands/stream.rs new file mode 100644 index 0000000..d7387cd --- /dev/null +++ b/src/commands/stream.rs @@ -0,0 +1,185 @@ +//! Bounded, ordered output shared by the depth commands. +use anyhow::{Context, Result, anyhow}; +use crossbeam_channel::{Receiver, Sender, bounded}; +use perbase_lib::par_granges::Region; +use std::{io::Write, panic::AssertUnwindSafe, thread}; + +const BLOCK: usize = 65536; + +pub(super) fn quote_name(name: &str) -> Vec { + let mut quoted = Vec::new(); + if name + .bytes() + .any(|byte| matches!(byte, b'\t' | b'\r' | b'\n' | b'"')) + { + quoted.push(b'"'); + for byte in name.bytes() { + if byte == b'"' { + quoted.push(byte); + } + quoted.push(byte); + } + quoted.push(b'"'); + } else { + quoted.extend_from_slice(name.as_bytes()); + } + quoted +} + +/// Most depth counters fit in two digits. Larger values use itoa's stack buffer. +#[inline] +pub(super) fn append_unsigned(text: &mut Vec, value: u32) { + if value < 10 { + text.push(b'0' + value as u8); + } else if value < 100 { + text.push(b'0' + (value / 10) as u8); + text.push(b'0' + (value % 10) as u8); + } else { + text.extend_from_slice(itoa::Buffer::new().format(value).as_bytes()); + } +} + +pub(super) struct Config { + pub workers: usize, + pub chunksize: u32, + pub modifier: f64, + pub estimated_row_bytes: usize, +} + +/// A worker's reusable output buffer. Full blocks leave immediately; a partial +/// tail stays here until another row fills it or the region finishes. +pub(super) struct Output<'a> { + pub text: &'a mut Vec, + sender: &'a Sender>>, + recycled: &'a Receiver>, +} + +impl Output<'_> { + pub fn flush(&mut self) -> Result<()> { + self.send(false) + } + + fn send(&mut self, finish: bool) -> Result<()> { + let length = if finish { + self.text.len() + } else { + self.text.len() / BLOCK * BLOCK + }; + if length == 0 { + return Ok(()); + } + for block in self.text[..length].chunks(BLOCK) { + let mut bytes = if block.len() == BLOCK { + self.recycled + .try_recv() + .unwrap_or_else(|_| Vec::with_capacity(BLOCK)) + } else { + Vec::with_capacity(block.len()) + }; + bytes.extend_from_slice(block); + self.sender + .send(Ok(bytes)) + .map_err(|_| anyhow!("output consumer stopped"))?; + } + self.text.copy_within(length.., 0); + self.text.truncate(self.text.len() - length); + Ok(()) + } +} + +/// Count and format in workers, then write their blocks in input-region order. +/// A failed worker or output write disconnects the queues before joining the +/// threads, so producers cannot remain blocked behind a stopped consumer. +pub(super) fn run( + regions: &[Region], + config: Config, + header: &[u8], + output: &mut dyn Write, + make_worker: impl Fn() -> Result, + process: impl Fn(&mut W, Region, &mut Output<'_>) -> Result<()> + Sync, +) -> Result<()> { + anyhow::ensure!( + config.modifier.is_finite() && config.modifier >= 0.0, + "invalid channel-size-modifier" + ); + if regions.is_empty() { + return Ok(()); + } + let estimate = f64::from(config.chunksize) * config.estimated_row_bytes as f64; + let budget = config.modifier * 1_000_000_000.0 * config.workers as f64; + // Queues hold blocks, not whole regions. A small output budget must not + // serialize counting: give every worker at least one queued block. + let window = config.workers.max(1).min(regions.len()); + let blocks = ((budget / window as f64).min(estimate) as usize / BLOCK).max(1); + let states = (0..window) + .map(|_| make_worker()) + .collect::>>()?; + let (jobs_tx, jobs_rx) = bounded::(window); + let (senders, receivers): (Vec<_>, Vec<_>) = (0..window) + .map(|_| bounded::>>(blocks)) + .unzip(); + // Retain just one spare allocation per slot, not entire completed regions. + let (returns, recycled): (Vec<_>, Vec<_>) = (0..window).map(|_| bounded::>(1)).unzip(); + + thread::scope(|scope| { + for mut worker in states { + let jobs = jobs_rx.clone(); + let senders = &senders; + let recycled = &recycled; + let process = &process; + scope.spawn(move || { + let mut text = Vec::with_capacity(BLOCK + 256); + for job in jobs { + text.clear(); + let sender = &senders[job % window]; + let mut output = Output { + text: &mut text, + sender, + recycled: &recycled[job % window], + }; + let result = std::panic::catch_unwind(AssertUnwindSafe(|| { + process(&mut worker, regions[job], &mut output)?; + output.send(true) + })) + .unwrap_or_else(|_| Err(anyhow!("counting worker panicked"))); + let failed = result.is_err(); + if sender.send(result.map(|()| Vec::new())).is_err() || failed { + break; + } + } + }); + } + let result = (|| -> Result<()> { + for job in 0..window { + jobs_tx.send(job)?; + } + let mut started = false; + for job in 0..regions.len() { + loop { + let mut bytes = receivers[job % window] + .recv() + .context("counting worker disconnected")??; + if bytes.is_empty() { + break; + } + if !started { + output.write_all(header)?; + started = true; + } + output.write_all(&bytes)?; + if bytes.capacity() == BLOCK { + bytes.clear(); + let _ = returns[job % window].try_send(bytes); + } + } + if job + window < regions.len() { + jobs_tx.send(job + window)?; + } + } + Ok(()) + })(); + drop(receivers); + drop(jobs_tx); + result + }) +} diff --git a/src/lib/par_granges.rs b/src/lib/par_granges.rs index 330aadc..f2b3816 100644 --- a/src/lib/par_granges.rs +++ b/src/lib/par_granges.rs @@ -25,6 +25,9 @@ pub const CHANNEL_SIZE_MODIFIER: f64 = 0.15; /// The ideal number of basepairs each worker will receive. Total bp in memory at one time = `threads` * `chunksize` pub const CHUNKSIZE: u32 = 1_000_000; +/// An ordered job: reference ID, zero-based start, and exclusive end. +pub type Region = (u32, u32, u32); + lazy_static! { /// CHANNEL_SIZE_MODIFIER as a str pub static ref CHANNEL_SIZE_MODIFIER_STR: String = CHANNEL_SIZE_MODIFIER.to_string(); @@ -156,6 +159,49 @@ impl ParGranges { Ok(()) } + fn input_intervals(&self, header: &HeaderView) -> Result>> { + let bed = self + .regions_bed + .as_ref() + .map(|path| Self::bed_to_intervals(header, path, self.merge_regions)) + .transpose()?; + let bcf = self + .regions_bcf + .as_ref() + .map(|path| Self::bcf_to_intervals(header, path, self.merge_regions)) + .transpose()?; + Ok(match (bed, bcf) { + (Some(a), Some(b)) => Self::merge_intervals(a, b, self.merge_regions), + (Some(intervals), None) | (None, Some(intervals)) => intervals, + (None, None) => Self::header_to_intervals(header, self.chunksize)?, + }) + } + + /// Return the processor and its ordered `(tid, start, stop)` jobs without + /// running them. Boundaries match `process`, including overlaps retained + /// with merging disabled. A streaming consumer must not subdivide these + /// jobs: doing so could change maximum-depth read admission. + pub fn into_regions(self) -> Result<(R, Vec)> { + self.validate()?; + anyhow::ensure!(self.chunksize > 0, "chunksize must be positive"); + let reader = IndexedReader::from_path(&self.reads)?; + let header = reader.header(); + let intervals = self.input_intervals(header)?; + let width = self.chunksize.saturating_mul(self.threads as u32); + let mut jobs = Vec::new(); + for (tid, intervals) in intervals.iter().enumerate() { + let tid = tid as u32; + let end: u32 = header.target_len(tid).unwrap().try_into()?; + for start in (0..end).step_by(width as usize) { + let stop = start.saturating_add(width).min(end); + for interval in intervals.find(start, stop) { + jobs.push((tid, interval.start.max(start), interval.stop.min(stop))); + } + } + } + Ok((self.processor, jobs)) + } + /// Process each region. /// /// This method splits the sequences in the BAM/CRAM header into `chunksize` * `self.threads` regions (aka 'super chunks'). @@ -193,38 +239,9 @@ impl ParGranges { // Get a copy of the header let header = reader.header().to_owned(); - // Work out if we are restricted to a subset of sites - let bed_intervals = if let Some(regions_bed) = &self.regions_bed { - Some( - Self::bed_to_intervals(&header, regions_bed, self.merge_regions) - .expect("Parsed BED to intervals"), - ) - } else { - None - }; - let bcf_intervals = if let Some(regions_bcf) = &self.regions_bcf { - Some( - Self::bcf_to_intervals(&header, regions_bcf, self.merge_regions) - .expect("Parsed BCF/VCF to intervals"), - ) - } else { - None - }; - let restricted_ivs = match (bed_intervals, bcf_intervals) { - (Some(bed_ivs), Some(bcf_ivs)) => { - Some(Self::merge_intervals(bed_ivs, bcf_ivs, self.merge_regions)) - } - (Some(bed_ivs), None) => Some(bed_ivs), - (None, Some(bcf_ivs)) => Some(bcf_ivs), - (None, None) => None, - }; - - let intervals = if let Some(restricted) = restricted_ivs { - restricted - } else { - Self::header_to_intervals(&header, self.chunksize) - .expect("Parsed BAM/CRAM header to intervals") - }; + let intervals = self + .input_intervals(&header) + .expect("Parsed input intervals"); // The number positions to try to process in one batch let serial_step_size = self.chunksize.saturating_mul(self.threads as u32); // aka superchunk diff --git a/src/lib/position/mate_fix.rs b/src/lib/position/mate_fix.rs index a7902cf..4f8c15d 100644 --- a/src/lib/position/mate_fix.rs +++ b/src/lib/position/mate_fix.rs @@ -218,6 +218,39 @@ pub enum MateResolutionStrategy { Original, } +// The observations used by mate resolution, independent of the BAM reader. +// Base and quality are only requested when both observations have a query +// position. Keeping these accesses lazy avoids decoding bases for Original. +pub(crate) trait MateRead { + fn flags(&self) -> u16; + fn mapq(&self) -> u8; + fn qpos(&self) -> Option; + fn base(&self) -> Base; + fn base_qual(&self) -> u8; +} + +impl MateRead for (Alignment<'_>, Record) { + fn flags(&self) -> u16 { + self.1.flags() + } + + fn mapq(&self) -> u8 { + self.1.mapq() + } + + fn qpos(&self) -> Option { + self.0.qpos() + } + + fn base(&self) -> Base { + Base::from(self.1.seq()[self.0.qpos().unwrap()] as char) + } + + fn base_qual(&self) -> u8 { + self.1.qual()[self.0.qpos().unwrap()] + } +} + impl MateResolutionStrategy { pub(crate) fn cmp( &self, @@ -228,6 +261,16 @@ impl MateResolutionStrategy { // Handle user-set filters. let a_pass = read_filter.filter_read(&a.1, Some(&a.0)); let b_pass = read_filter.filter_read(&b.1, Some(&b.0)); + self.compare(a, b, a_pass, b_pass) + } + + pub(crate) fn compare( + &self, + a: &impl MateRead, + b: &impl MateRead, + a_pass: bool, + b_pass: bool, + ) -> MateResolution { if a_pass && !b_pass { return MateResolution::new(Ordering::Greater, None); } else if b_pass && !a_pass { @@ -254,48 +297,45 @@ impl MateResolutionStrategy { } pub(crate) fn resolve_base( - a: &(Alignment<'_>, Record), - b: &(Alignment<'_>, Record), + a: &impl MateRead, + b: &impl MateRead, ) -> MateResolution { // First check that we have a base - let a_is_not_base = a.0.qpos().is_none(); - let b_is_not_base = b.0.qpos().is_none(); + let a_is_not_base = a.qpos().is_none(); + let b_is_not_base = b.qpos().is_none(); if a_is_not_base || b_is_not_base { return Self::original(a, b); } - let a_base = Base::from(a.1.seq()[a.0.qpos().unwrap()] as char); - let b_base = Base::from(b.1.seq()[b.0.qpos().unwrap()] as char); + let a_base = a.base(); + let b_base = b.base(); MateResolution::new( Ordering::Greater, Some(Base::either_or::(a_base, b_base)), ) } - pub(crate) fn base_qual_map_qual_n( - a: &(Alignment<'_>, Record), - b: &(Alignment<'_>, Record), - ) -> MateResolution { + pub(crate) fn base_qual_map_qual_n(a: &impl MateRead, b: &impl MateRead) -> MateResolution { // First check that we have a base - let a_is_not_base = a.0.qpos().is_none(); - let b_is_not_base = b.0.qpos().is_none(); + let a_is_not_base = a.qpos().is_none(); + let b_is_not_base = b.qpos().is_none(); if a_is_not_base || b_is_not_base { return Self::original(a, b); } // compare baseq -> mapq -> default to first in pair - let a_qual = a.1.qual()[a.0.qpos().unwrap()]; - let b_qual = b.1.qual()[b.0.qpos().unwrap()]; + let a_qual = a.base_qual(); + let b_qual = b.base_qual(); match a_qual.cmp(&b_qual) { Ordering::Greater => MateResolution::new(Ordering::Greater, None), Ordering::Less => MateResolution::new(Ordering::Less, None), - Ordering::Equal => match a.1.mapq().cmp(&b.1.mapq()) { + Ordering::Equal => match a.mapq().cmp(&b.mapq()) { Ordering::Greater => MateResolution::new(Ordering::Greater, None), Ordering::Less => MateResolution::new(Ordering::Less, None), Ordering::Equal => { - let a_base = Base::from(a.1.seq()[a.0.qpos().unwrap()] as char); - let b_base = Base::from(b.1.seq()[b.0.qpos().unwrap()] as char); + let a_base = a.base(); + let b_base = b.base(); let base = Base::either_or::(a_base, b_base); MateResolution::new(Ordering::Greater, Some(base)) // how to pass this back } @@ -303,30 +343,27 @@ impl MateResolutionStrategy { } } - pub(crate) fn base_qual_map_qual_iupac( - a: &(Alignment<'_>, Record), - b: &(Alignment<'_>, Record), - ) -> MateResolution { + pub(crate) fn base_qual_map_qual_iupac(a: &impl MateRead, b: &impl MateRead) -> MateResolution { // First check that we have a base - let a_is_not_base = a.0.qpos().is_none(); - let b_is_not_base = b.0.qpos().is_none(); + let a_is_not_base = a.qpos().is_none(); + let b_is_not_base = b.qpos().is_none(); if a_is_not_base || b_is_not_base { return Self::original(a, b); } // compare baseq -> mapq -> default to first in pair - let a_qual = a.1.qual()[a.0.qpos().unwrap()]; - let b_qual = b.1.qual()[b.0.qpos().unwrap()]; + let a_qual = a.base_qual(); + let b_qual = b.base_qual(); match a_qual.cmp(&b_qual) { Ordering::Greater => MateResolution::new(Ordering::Greater, None), Ordering::Less => MateResolution::new(Ordering::Less, None), - Ordering::Equal => match a.1.mapq().cmp(&b.1.mapq()) { + Ordering::Equal => match a.mapq().cmp(&b.mapq()) { Ordering::Greater => MateResolution::new(Ordering::Greater, None), Ordering::Less => MateResolution::new(Ordering::Less, None), Ordering::Equal => { - let a_base = Base::from(a.1.seq()[a.0.qpos().unwrap()] as char); - let b_base = Base::from(b.1.seq()[b.0.qpos().unwrap()] as char); + let a_base = a.base(); + let b_base = b.base(); MateResolution::new( Ordering::Greater, Some(Base::either_or::(a_base, b_base)), @@ -337,19 +374,19 @@ impl MateResolutionStrategy { } pub(crate) fn base_qual_map_qual_first_in_pair( - a: &(Alignment<'_>, Record), - b: &(Alignment<'_>, Record), + a: &impl MateRead, + b: &impl MateRead, ) -> MateResolution { // First check that we have a base - let a_is_not_base = a.0.qpos().is_none(); - let b_is_not_base = b.0.qpos().is_none(); + let a_is_not_base = a.qpos().is_none(); + let b_is_not_base = b.qpos().is_none(); if a_is_not_base || b_is_not_base { return Self::original(a, b); } // compare baseq -> mapq -> default to first in pair - let a_qual = a.1.qual()[a.0.qpos().unwrap()]; - let b_qual = b.1.qual()[b.0.qpos().unwrap()]; + let a_qual = a.base_qual(); + let b_qual = b.base_qual(); match a_qual.cmp(&b_qual) { Ordering::Greater => MateResolution::new(Ordering::Greater, None), @@ -358,30 +395,27 @@ impl MateResolutionStrategy { } } - pub(crate) fn map_qual_base_qual_n( - a: &(Alignment<'_>, Record), - b: &(Alignment<'_>, Record), - ) -> MateResolution { + pub(crate) fn map_qual_base_qual_n(a: &impl MateRead, b: &impl MateRead) -> MateResolution { // First check that we have a base - let a_is_not_base = a.0.qpos().is_none(); - let b_is_not_base = b.0.qpos().is_none(); + let a_is_not_base = a.qpos().is_none(); + let b_is_not_base = b.qpos().is_none(); if a_is_not_base || b_is_not_base { return Self::original(a, b); } - match a.1.mapq().cmp(&b.1.mapq()) { + match a.mapq().cmp(&b.mapq()) { Ordering::Greater => MateResolution::new(Ordering::Greater, None), Ordering::Less => MateResolution::new(Ordering::Less, None), Ordering::Equal => { - let a_qual = a.1.qual()[a.0.qpos().unwrap()]; - let b_qual = b.1.qual()[b.0.qpos().unwrap()]; + let a_qual = a.base_qual(); + let b_qual = b.base_qual(); match a_qual.cmp(&b_qual) { Ordering::Greater => MateResolution::new(Ordering::Greater, None), Ordering::Less => MateResolution::new(Ordering::Less, None), Ordering::Equal => { - let a_base = Base::from(a.1.seq()[a.0.qpos().unwrap()] as char); - let b_base = Base::from(b.1.seq()[b.0.qpos().unwrap()] as char); + let a_base = a.base(); + let b_base = b.base(); let base = Base::either_or::(a_base, b_base); MateResolution::new(Ordering::Greater, Some(base)) // how to pass this back } @@ -390,30 +424,27 @@ impl MateResolutionStrategy { } } - pub(crate) fn map_qual_base_qual_iupac( - a: &(Alignment<'_>, Record), - b: &(Alignment<'_>, Record), - ) -> MateResolution { + pub(crate) fn map_qual_base_qual_iupac(a: &impl MateRead, b: &impl MateRead) -> MateResolution { // First check that we have a base - let a_is_not_base = a.0.qpos().is_none(); - let b_is_not_base = b.0.qpos().is_none(); + let a_is_not_base = a.qpos().is_none(); + let b_is_not_base = b.qpos().is_none(); if a_is_not_base || b_is_not_base { return Self::original(a, b); } - match a.1.mapq().cmp(&b.1.mapq()) { + match a.mapq().cmp(&b.mapq()) { Ordering::Greater => MateResolution::new(Ordering::Greater, None), Ordering::Less => MateResolution::new(Ordering::Less, None), Ordering::Equal => { - let a_qual = a.1.qual()[a.0.qpos().unwrap()]; - let b_qual = b.1.qual()[b.0.qpos().unwrap()]; + let a_qual = a.base_qual(); + let b_qual = b.base_qual(); match a_qual.cmp(&b_qual) { Ordering::Greater => MateResolution::new(Ordering::Greater, None), Ordering::Less => MateResolution::new(Ordering::Less, None), Ordering::Equal => { - let a_base = Base::from(a.1.seq()[a.0.qpos().unwrap()] as char); - let b_base = Base::from(b.1.seq()[b.0.qpos().unwrap()] as char); + let a_base = a.base(); + let b_base = b.base(); MateResolution::new( Ordering::Greater, Some(Base::either_or::(a_base, b_base)), @@ -425,30 +456,30 @@ impl MateResolutionStrategy { } pub(crate) fn map_qual_base_qual_first_in_pair( - a: &(Alignment<'_>, Record), - b: &(Alignment<'_>, Record), + a: &impl MateRead, + b: &impl MateRead, ) -> MateResolution { // First check that we have a base - let a_is_not_base = a.0.qpos().is_none(); - let b_is_not_base = b.0.qpos().is_none(); + let a_is_not_base = a.qpos().is_none(); + let b_is_not_base = b.qpos().is_none(); if a_is_not_base || b_is_not_base { return Self::original(a, b); } - match a.1.mapq().cmp(&b.1.mapq()) { + match a.mapq().cmp(&b.mapq()) { Ordering::Greater => MateResolution::new(Ordering::Greater, None), Ordering::Less => MateResolution::new(Ordering::Less, None), Ordering::Equal => { - let a_qual = a.1.qual()[a.0.qpos().unwrap()]; - let b_qual = b.1.qual()[b.0.qpos().unwrap()]; + let a_qual = a.base_qual(); + let b_qual = b.base_qual(); match a_qual.cmp(&b_qual) { Ordering::Greater => MateResolution::new(Ordering::Greater, None), Ordering::Less => MateResolution::new(Ordering::Less, None), Ordering::Equal => { - if a.1.flags() & 64 != 0 { + if a.flags() & 64 != 0 { MateResolution::new(Ordering::Greater, None) - } else if b.1.flags() & 64 != 0 { + } else if b.flags() & 64 != 0 { MateResolution::new(Ordering::Less, None) } else { // Default to `a` in the event that there is no first in pair for some reason @@ -461,18 +492,15 @@ impl MateResolutionStrategy { } /// Whichever has higher MAPQ, or if equal, whichever is first in pair - pub(crate) fn original( - a: &(Alignment<'_>, Record), - b: &(Alignment<'_>, Record), - ) -> MateResolution { - match a.1.mapq().cmp(&b.1.mapq()) { + pub(crate) fn original(a: &impl MateRead, b: &impl MateRead) -> MateResolution { + match a.mapq().cmp(&b.mapq()) { Ordering::Greater => MateResolution::new(Ordering::Greater, None), Ordering::Less => MateResolution::new(Ordering::Less, None), Ordering::Equal => { // Check if a is first in pair - if a.1.flags() & 64 != 0 { + if a.flags() & 64 != 0 { MateResolution::new(Ordering::Greater, None) - } else if b.1.flags() & 64 != 0 { + } else if b.flags() & 64 != 0 { MateResolution::new(Ordering::Less, None) } else { // Default to `a` in the event that there is no first in pair for some reason diff --git a/src/lib/position/mod.rs b/src/lib/position/mod.rs index 8e63bf0..58edd4c 100644 --- a/src/lib/position/mod.rs +++ b/src/lib/position/mod.rs @@ -2,6 +2,8 @@ pub mod mate_fix; pub mod pileup_position; pub mod range_positions; +#[cfg(feature = "seqair")] +pub mod seqair; use serde::Serialize; use smartstring::alias::String; diff --git a/src/lib/position/pileup_position.rs b/src/lib/position/pileup_position.rs index a7476c6..35d022a 100644 --- a/src/lib/position/pileup_position.rs +++ b/src/lib/position/pileup_position.rs @@ -1,7 +1,6 @@ //! An implementation of `Position` for dealing with pileups. use crate::position::Position; use crate::read_filter::ReadFilter; -use itertools::Itertools; use rust_htslib::bam::{ self, HeaderView, pileup::{Alignment, Pileup}, @@ -9,14 +8,14 @@ use rust_htslib::bam::{ }; use serde::Serialize; use smartstring::{LazyCompact, SmartString, alias::String}; -use std::default; +use std::{collections::HashMap, default}; use super::mate_fix::{Base, MateResolutionStrategy}; /// Hold all information about a position. // NB: The max depth that htslib will return is i32::MAX, and the type of pos for htlib is u32 // There is no reason to go bigger, for now at least -#[derive(Debug, Serialize, Default)] +#[derive(Debug, Clone, Serialize, Default)] #[serde(rename_all = "SCREAMING_SNAKE_CASE")] pub struct PileupPosition { /// Reference sequence name. @@ -195,6 +194,9 @@ impl PileupPosition { /// first in pair that passes filters. In the event of both failing filters or not being first in pair, the first /// read encountered is kept. /// + /// When processing many positions in a region, use [`MateCounter`] to + /// reuse read-name groups instead of sorting the names at each position. + /// /// # Arguments /// /// * `pileup` - a pileup at a genomic position @@ -210,58 +212,50 @@ impl PileupPosition { mate_fix_strat: MateResolutionStrategy, ) -> Self { let name = Self::compact_refseq(header, pileup.tid()); - // make output 1-based let mut pos = Self::new(name, pileup.pos()); pos.depth = pileup.depth(); - - // Group records by qname - let grouped_by_qname = pileup + let mut reads: Vec<_> = pileup .alignments() - .map(|aln| { - let record = aln.record(); - (aln, record) + .map(|alignment| { + let record = alignment.record(); + (alignment, record) }) - .sorted_by(|a, b| Ord::cmp(a.1.qname(), b.1.qname())) - // TODO: I'm not sure there is a good way to remove this allocation - .chunk_by(|a| a.1.qname().to_owned()); - - for (_qname, reads) in grouped_by_qname.into_iter() { - let mut total_reads = 0; // count how many reads there were - - let mut reads = reads - .inspect(|_| total_reads += 1) - .sorted_by(|a, b| mate_fix_strat.cmp(a, b, read_filter).ordering.reverse()); - let best = &reads.next().unwrap(); - - if let Some(second_best) = &reads.next().as_ref() { - // Deal explicitly with mate overlap, they are already ordered correctly - let result = mate_fix_strat.cmp(best, second_best, read_filter); - pos.depth -= total_reads - 1; - Self::update( - &mut pos, - &best.0, - &best.1, - read_filter, - base_filter, - result.recommended_base, - true, - ); - } else { - pos.depth -= total_reads - 1; - Self::update( - &mut pos, - &best.0, - &best.1, - read_filter, - base_filter, - None, - false, - ); - } + .collect(); + reads.sort_by(|a, b| a.1.qname().cmp(b.1.qname())); + for group in reads.chunk_by_mut(|a, b| a.1.qname() == b.1.qname()) { + pos.update_mate_group(group, read_filter, base_filter, mate_fix_strat); } pos } + // Count one group of same-name records. Keep the original stable sort: + // some strategies resolve ties by encounter order, including groups with + // more than two alignments (for example, secondary alignments). + fn update_mate_group( + &mut self, + reads: &mut [(Alignment<'_>, Record)], + read_filter: &F, + base_filter: Option, + mate_fix_strat: MateResolutionStrategy, + ) { + reads.sort_by(|a, b| mate_fix_strat.cmp(a, b, read_filter).ordering.reverse()); + let best = &reads[0]; + let recommended_base = reads.get(1).and_then(|second_best| { + mate_fix_strat + .cmp(best, second_best, read_filter) + .recommended_base + }); + self.depth -= reads.len() as u32 - 1; + self.update( + &best.0, + &best.1, + read_filter, + base_filter, + recommended_base, + reads.len() > 1, + ); + } + /// Convert a tid to a [`SmartString`]. #[inline] pub fn compact_refseq(header: &HeaderView, tid: u32) -> SmartString { @@ -270,6 +264,84 @@ impl PileupPosition { } } +/// Count bases without counting overlapping same-name reads more than once. +/// +/// Reuse one counter for the positions in a queried region. It copies each +/// distinct read name once and reuses the lists that group alignments by name, +/// instead of sorting names and allocating those lists at every position. +/// Names remain stored until the counter is dropped, so create a new counter +/// for each region rather than retaining one for an entire genome. +#[derive(Default)] +pub struct MateCounter { + names: HashMap, usize>, + groups: Vec>, + active_groups: Vec, +} + +impl MateCounter { + /// Count one position using the requested filters and mate strategy. + /// Only names and reusable index lists survive the call; BAM records and + /// alignments are borrowed from this pileup and are not retained. + pub fn count( + &mut self, + pileup: Pileup, + header: &HeaderView, + read_filter: &F, + base_filter: Option, + mate_fix_strat: MateResolutionStrategy, + ) -> PileupPosition { + let name = PileupPosition::compact_refseq(header, pileup.tid()); + let mut pos = PileupPosition::new(name, pileup.pos()); + pos.depth = pileup.depth(); + self.active_groups.clear(); + + for (index, alignment) in pileup.alignments().enumerate() { + let record = alignment.record(); + let group = match self.names.get(record.qname()) { + Some(&group) => group, + None => { + let group = self.groups.len(); + self.names.insert(record.qname().to_owned(), group); + self.groups.push(Vec::new()); + group + } + }; + if self.groups[group].is_empty() { + self.active_groups.push(group); + } + self.groups[group].push(index); + } + + // The iterator is backed by a slice, so nth() indexes directly without + // walking earlier alignments. Reconstructing a Record here only borrows + // its BAM data; it does not copy the sequence or allocate storage. + let entry = |index| { + let alignment = pileup.alignments().nth(index).unwrap(); + let record = alignment.record(); + (alignment, record) + }; + for &group in &self.active_groups { + let indices = &mut self.groups[group]; + match *indices.as_slice() { + [index] => { + let (alignment, record) = entry(index); + pos.update(&alignment, &record, read_filter, base_filter, None, false); + } + [first, second] => { + let mut reads = [entry(first), entry(second)]; + pos.update_mate_group(&mut reads, read_filter, base_filter, mate_fix_strat); + } + _ => { + let mut reads: Vec<_> = indices.iter().map(|&index| entry(index)).collect(); + pos.update_mate_group(&mut reads, read_filter, base_filter, mate_fix_strat); + } + } + indices.clear(); + } + pos + } +} + #[cfg(test)] mod tests { use super::*; @@ -310,6 +382,8 @@ mod tests { b"TESTPAIR\t147\tchr1\t15\t40\t10M\tchr1\t10\t30\tCCCCCCCCCC\t##########", ) .unwrap(), + // Reuse the name after both mates have left the pileup. + Record::from_sam(&view, b"TESTPAIR\t0\tchr1\t40\t20\t3M\t*\t0\t0\tGGG\t###").unwrap(), ]; // Write BAM file @@ -359,6 +433,39 @@ mod tests { } assert!(found_position, "Should have found pileup at position 14"); + + // Reusing the counter must handle one read, overlapping mates, the + // remaining read, and a later alignment with the same name. Counts + // from earlier positions must never leak into a subsequent position. + reader.fetch(("chr1", 0, 100)).unwrap(); + let mut counter = MateCounter::default(); + let observed: Vec<_> = reader + .pileup() + .map(|pileup| { + let position = counter.count( + pileup.unwrap(), + &header_view, + &read_filter, + None, + MateResolutionStrategy::Original, + ); + ( + position.pos, + position.depth, + position.a, + position.c, + position.g, + position.count_of_mate_resolutions, + ) + }) + .collect(); + let expected: Vec<_> = (9..14) + .map(|pos| (pos, 1, 1, 0, 0, 0)) + .chain((14..19).map(|pos| (pos, 1, 1, 0, 0, 1))) + .chain((19..24).map(|pos| (pos, 1, 0, 1, 0, 0))) + .chain((39..42).map(|pos| (pos, 1, 0, 0, 1, 0))) + .collect(); + assert_eq!(observed, expected); } /// Test different MateResolutionStrategy behaviors diff --git a/src/lib/position/seqair.rs b/src/lib/position/seqair.rs new file mode 100644 index 0000000..75fdbd7 --- /dev/null +++ b/src/lib/position/seqair.rs @@ -0,0 +1,778 @@ +//! Seqair-backed base counting with Perbase's filtering and mate rules. +use super::{ + Position, + mate_fix::{Base, MateRead, MateResolutionStrategy}, + pileup_position::PileupPosition, +}; +use crate::read_filter::DefaultReadFilter; +use ::seqair::{ + bam::{ + pileup::{PileupAlignment, PileupEngine, PileupOp}, + record_idx::RecordIdx, + record_store::{CustomizeRecordStore, FilterRawFields, RecordStore, SlimRecord}, + }, + reader::{AuxTags, IndexedReader, PartialIndexedReader, RecordFields, ResolveTid}, +}; +use anyhow::{Context, Result}; +use seqair_types::Pos0; +use std::{cmp::Reverse, collections::BinaryHeap, path::Path, sync::Mutex}; + +/// Reads indexed regions with Seqair and counts their bases. Workers share +/// the parsed header and index, but own their file handles and read buffers. +pub struct SeqairCounter { + template: Mutex, + filter: DefaultReadFilter, + strategy: Option, + max_depth: u32, + min_base_quality: Option, +} + +impl SeqairCounter { + /// Open the alignment and its index. Supplying a reference also enables + /// CRAM files whose bases must be reconstructed from that reference. + pub fn open( + path: &Path, + reference: Option<&Path>, + filter: DefaultReadFilter, + strategy: Option, + max_depth: u32, + min_base_quality: Option, + ) -> Result { + let mut builder = + IndexedReader::builder(path).fields(RecordFields::all().with_aux(AuxTags::None)); + if let Some(reference) = reference { + builder = builder.reference(reference); + } + let reader = builder.open()?; + Ok(Self { + template: Mutex::new(reader), + filter, + strategy, + max_depth, + min_base_quality, + }) + } + + /// Count the half-open interval `[start, stop)`. Positions returned here + /// are zero-based; the caller adds coordinate conversion, uncovered rows + /// and reference annotations just as it does for HTSlib results. + pub fn count_region(&self, tid: u32, start: u32, stop: u32) -> Result> { + let mut positions = Vec::new(); + self.worker()? + .for_each_position(tid, start, stop, |position| { + positions.push(position); + Ok(()) + })?; + Ok(positions) + } + + /// Create a worker whose reader and allocations can be reused across regions. + pub fn worker(&self) -> Result> { + Ok(SeqairWorker { + counter: self, + reader: self.template.lock().expect("Seqair reader lock").fork()?, + store: RecordStore::new(), + ends: BinaryHeap::new(), + info: Vec::new(), + names: Vec::new(), + heads: Vec::new(), + links: Vec::new(), + active: Vec::new(), + members: Vec::new(), + group_ends: Vec::new(), + overlaps: Vec::new(), + }) + } +} + +/// A single counting worker. No output rows are retained between callbacks. +pub struct SeqairWorker<'a> { + counter: &'a SeqairCounter, + reader: PartialIndexedReader, + store: RecordStore, + ends: BinaryHeap>, + info: Vec, + names: Vec>, + heads: Vec, + links: Vec, + active: Vec, + members: Vec, + group_ends: Vec>, + overlaps: Vec, +} + +impl SeqairWorker<'_> { + /// Count `[start, stop)` and immediately pass each zero-based row to `emit`. + /// The callback may stop processing by returning an error, such as a failed + /// output write. Read storage is reused on the next successful region. + pub fn for_each_position( + &mut self, + tid: u32, + start: u32, + stop: u32, + mut emit: impl FnMut(PileupPosition) -> Result<()>, + ) -> Result<()> { + self.for_each_position_ref(tid, start, stop, |position| emit(position.clone())) + } + + /// Count `[start, stop)` without copying the reference name for every row. + /// The callback borrows a row that is reused at the next position. Clone + /// it only when retaining rows rather than immediately writing them. + pub fn for_each_position_ref( + &mut self, + tid: u32, + start: u32, + stop: u32, + mut emit: impl FnMut(&PileupPosition) -> Result<()>, + ) -> Result<()> { + anyhow::ensure!(start <= stop, "region start must not exceed its end"); + if start == stop { + return Ok(()); + } + let start = Pos0::new(start).context("region start exceeds Seqair's coordinate range")?; + let last = Pos0::new(stop - 1).context("region end exceeds Seqair's coordinate range")?; + let tid = tid.resolve_tid(self.reader.header())?; + let name = self + .reader + .header() + .target_name(tid) + .context("unknown Seqair reference")? + .to_owned(); + let mut reads = CountingReads::default(); + self.reader.fetch_into_customized( + tid, + (start..=last).into(), + &mut self.store, + &mut reads, + )?; + anyhow::ensure!( + !reads.long_cigar, + "Seqair 0.10.1 pileup storage does not support BAM long CIGARs (CG tags); rerun base-depth without --seqair" + ); + self.info.clear(); + self.info.extend(self.store.records().map(|read| { + ReadInfo { + group: 0, + passes: self + .counter + .filter + .filter_fields(read.flags.raw(), read.mapq.as_byte()), + admitted: true, + } + })); + let restricted = self.admit()?; + let input = std::mem::take(&mut self.store).prepare_for_pileup().input; + let store = input.store(); + if self.counter.strategy.is_some() { + // Store record indices in the name table, not copies of names. + // Exact byte comparison resolves collisions and includes unpaired + // and secondary reads, unlike primary-mate links. + let capacity = (store.len() * 2).next_power_of_two(); + self.names.clear(); + self.names.resize(capacity, None); + self.heads.clear(); + self.heads.resize(store.len(), usize::MAX); + self.group_ends.clear(); + self.group_ends.resize(store.len(), None); + self.overlaps.clear(); + self.overlaps.resize(store.len(), false); + for index in store.indices() { + let record = store.record(index).unwrap(); + let name = record.qname(); + let mut slot = + record.qname_hash().map_or(0, |hash| hash.get()) as usize & (capacity - 1); + while let Some(other) = self.names[slot] { + if store.record(other).unwrap().qname() == name { + break; + } + slot = (slot + 1) & (capacity - 1); + } + let group = *self.names[slot].get_or_insert(index); + let group = group.as_usize(); + self.info[index.as_usize()].group = group; + let start = record.pos.as_u32(); + let end = record.end_pos.as_u32(); + // Sorted starts and inclusive ends let us prove that no two + // records in this name group can contribute to one column. + if let Some(previous_end) = self.group_ends[group] { + self.overlaps[group] |= start <= previous_end; + self.group_ends[group] = Some(previous_end.max(end)); + } else { + self.group_ends[group] = Some(end); + } + } + } + let mut engine = PileupEngine::new(input, (start..=last).into()); + // Seqair's per-position cap would discard different bases. Whole-read + // admission above implements the HTSlib cap, so leave this uncapped. + let cutoff = self.counter.max_depth - (self.counter.max_depth as f64 * 0.01) as u32; + let mut position = PileupPosition::new(name.as_str().into(), start.as_u32()); + while let Some(column) = engine.pileups() { + let depth = if restricted { + column + .raw_alignments() + .filter(|read| self.info[read.record_idx().as_usize()].admitted) + .count() + } else { + column.depth() + } as u32; + if depth == 0 { + continue; + } + position = PileupPosition { + ref_seq: position.ref_seq, + pos: column.pos().as_u32(), + depth, + near_max_depth: depth > cutoff, + ..Default::default() + }; + if let Some(strategy) = self.counter.strategy { + self.active.clear(); + self.links.clear(); + self.members.clear(); + self.active.reserve(column.depth()); + self.links.reserve(column.depth()); + self.members.reserve(column.depth()); + for (index, alignment) in column.raw_alignments().enumerate() { + let info = self.info[alignment.record_idx().as_usize()]; + if restricted && !info.admitted { + self.links.push(usize::MAX); + continue; + } + let group = info.group; + if !self.overlaps[group] { + self.links.push(usize::MAX); + update( + &mut position, + alignment, + info.passes, + self.counter.min_base_quality, + None, + false, + ); + continue; + } + if self.heads[group] == usize::MAX { + self.active.push(group); + } + self.links.push(self.heads[group]); + self.heads[group] = index; + } + let alignment = |index| column.alignment_at(index).unwrap().alignment(); + let passes = + |read: &PileupAlignment| self.info[read.record_idx().as_usize()].passes; + for &group in self.active.iter() { + let last = self.heads[group]; + let previous = self.links[last]; + if previous == usize::MAX { + let best = alignment(last); + update( + &mut position, + best, + passes(best), + self.counter.min_base_quality, + None, + false, + ); + self.heads[group] = usize::MAX; + continue; + } + if self.links[previous] == usize::MAX { + // Preserve the two-element stable sort's comparison + // order, including ties and recommended-base selection. + let (a, b) = (alignment(previous), alignment(last)); + let swap = strategy.compare(b, a, passes(b), passes(a)).ordering + == std::cmp::Ordering::Greater; + let (best, second) = if swap { (b, a) } else { (a, b) }; + let base = strategy + .compare(best, second, passes(best), passes(second)) + .recommended_base; + position.depth -= 1; + update( + &mut position, + best, + passes(best), + self.counter.min_base_quality, + base, + true, + ); + self.heads[group] = usize::MAX; + continue; + } + self.members.clear(); + let mut current = self.heads[group]; + while current != usize::MAX { + self.members.push(current); + current = self.links[current]; + } + self.members.reverse(); + let reads = &mut self.members; + reads.sort_by(|&a, &b| { + let (a, b) = (alignment(a), alignment(b)); + strategy + .compare(a, b, passes(a), passes(b)) + .ordering + .reverse() + }); + let best = alignment(reads[0]); + let base = reads.get(1).and_then(|&second| { + let second = alignment(second); + strategy + .compare(best, second, passes(best), passes(second)) + .recommended_base + }); + position.depth -= reads.len() as u32 - 1; + update( + &mut position, + best, + passes(best), + self.counter.min_base_quality, + base, + reads.len() > 1, + ); + self.heads[group] = usize::MAX; + } + } else { + for read in column.raw_alignments() { + if restricted && !self.info[read.record_idx().as_usize()].admitted { + continue; + } + update( + &mut position, + read, + self.info[read.record_idx().as_usize()].passes, + self.counter.min_base_quality, + None, + false, + ); + } + } + emit(&position)?; + } + self.store = engine.reclaim_allocation().unwrap_or_default(); + Ok(()) + } + + /// Mark whole reads excluded by HTSlib's depth-admission rule. Small + /// regions cannot hit the limit and need no heap or CIGAR scan. + /// Reads ending exactly at the next start still occupy a slot. The first + /// read at a new start bypasses the cap, except at reference 0, position 0. + /// Like Goose, this runs after loading the region: the depth limit does + /// not limit how much read data is retained in memory. + fn admit(&mut self) -> Result { + let limit = self.counter.max_depth.min(i32::MAX as u32) as usize; + if limit > 0 && self.store.len() <= limit { + return Ok(false); + } + self.ends.clear(); + let mut previous = None; + let mut restricted = false; + for (read, info) in self.store.records().zip(&mut self.info) { + let start = read.pos.as_i64(); + while self.ends.peek().is_some_and(|end| end.0 < start) { + self.ends.pop(); + } + let at_cursor = previous == Some(start) + || (previous.is_none() && read.tid.unwrap().as_u32() == 0 && start == 0); + if at_cursor && self.ends.len() >= limit { + info.admitted = false; + restricted = true; + continue; + } + // Seqair normalizes both zero- and one-base spans to end == pos. + // Only that ambiguous case needs the CIGAR; longer spans are cached. + let span = if read.end_pos == read.pos { + read.cigar(&self.store)? + .iter() + .filter(|op| op.consumes_ref()) + .map(|op| i64::from(op.len())) + .sum() + } else { + read.end_pos.as_i64() - start + 1 + }; + if span > 0 || !at_cursor { + self.ends.push(Reverse(start + span)); + } + previous = Some(start); + } + Ok(restricted) + } +} + +#[derive(Clone, Copy)] +struct ReadInfo { + group: usize, + passes: bool, + admitted: bool, +} + +#[derive(Clone, Default)] +struct CountingReads { + long_cigar: bool, +} + +impl CustomizeRecordStore for CountingReads { + type Extra = (); + + fn filter_raw(&mut self, fields: &FilterRawFields<'_>) -> bool { + if fields.flags.is_unmapped() || fields.tid.is_none() { + return false; + } + // BAM stores long CIGARs as two placeholder operations and a CG tag. + // The upstream raw reader expands them, but RecordStore does not yet. + if fields.n_cigar_ops == 2 { + let mut ops = fields + .cigar + .as_chunks::<4>() + .0 + .iter() + .map(|bytes| u32::from_le_bytes(*bytes)); + let first = ops.next().unwrap(); + let second = ops.next().unwrap(); + if first & 15 == 4 + && first >> 4 == fields.seq_len + && second & 15 == 3 + && ::seqair::bam::aux::find_tag(fields.aux_bytes, *b"CG").is_some() + { + self.long_cigar = true; + return false; + } + } + true + } + + fn compute(&mut self, _: &SlimRecord, _: &RecordStore) {} +} + +impl MateRead for PileupAlignment { + fn flags(&self) -> u16 { + self.flags.raw() + } + fn mapq(&self) -> u8 { + self.mapq.as_byte() + } + fn qpos(&self) -> Option { + match self.op { + PileupOp::Match { qpos, .. } + | PileupOp::Insertion { qpos, .. } + | PileupOp::SoftClip { qpos, .. } => Some(qpos.as_usize()), + _ => None, + } + } + fn base(&self) -> Base { + let base = match self.op { + PileupOp::Match { base, .. } + | PileupOp::Insertion { base, .. } + | PileupOp::SoftClip { base, .. } => base, + _ => seqair_types::Base::Unknown, + }; + match base { + seqair_types::Base::A => Base::A, + seqair_types::Base::C => Base::C, + seqair_types::Base::G => Base::G, + seqair_types::Base::T => Base::T, + _ => Base::N, + } + } + fn base_qual(&self) -> u8 { + match self.op { + PileupOp::Match { qual, .. } + | PileupOp::Insertion { qual, .. } + | PileupOp::SoftClip { qual, .. } => qual.as_byte(), + _ => 255, + } + } +} + +#[inline] +fn update( + position: &mut PileupPosition, + read: &PileupAlignment, + passes: bool, + min_quality: Option, + recommended: Option, + mates_resolved: bool, +) { + if !passes { + position.depth -= 1; + position.fail += 1; + return; + } + if matches!( + read.op, + PileupOp::RefSkip + | PileupOp::ComplexIndel { + is_refskip: true, + .. + } + ) { + position.ref_skip += 1; + position.depth -= 1; + } else if matches!( + read.op, + PileupOp::Deletion { .. } | PileupOp::ComplexIndel { .. } + ) { + position.del += 1; + } else { + let base = if min_quality.is_some_and(|q| read.seq_len == 0 || read.base_qual() < q) { + Base::N + } else { + recommended.unwrap_or_else(|| MateRead::base(read)) + }; + match base { + Base::A => position.a += 1, + Base::C => position.c += 1, + Base::G => position.g += 1, + Base::T => position.t += 1, + Base::R => position.r += 1, + Base::Y => position.y += 1, + Base::S => position.s += 1, + Base::W => position.w += 1, + Base::K => position.k += 1, + Base::M => position.m += 1, + _ => position.n += 1, + } + if matches!(read.op, PileupOp::Insertion { insert_len, .. } if insert_len > 0) { + position.ins += 1; + } + } + if mates_resolved { + position.count_of_mate_resolutions += 1; + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::position::pileup_position::MateCounter; + use proptest::prelude::*; + use rust_htslib::bam::{self, Read}; + + // name, contig, start, CIGAR shape, flags, MAPQ, sequence offset, quality + type ReadSpec = (u8, u8, u8, u8, u8, u8, u8, u8); + + fn fixture(specs: &[ReadSpec]) -> (tempfile::TempDir, std::path::PathBuf) { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("reads.bam"); + let mut header = bam::Header::new(); + for name in ["a", "b"] { + header.push_record( + bam::header::HeaderRecord::new(b"SQ") + .push_tag(b"SN", name) + .push_tag(b"LN", 64), + ); + } + let view = bam::HeaderView::from_header(&header); + let mut records = Vec::new(); + for &(name, tid, start, shape, flags, mapq, base, quality) in specs { + let qname = if name == 0 { + "*".to_owned() + } else { + format!("name-{name}") + }; + let sequence: String = (0..5) + .map(|i| b"ACGTN"[(usize::from(base) + i) % 5] as char) + .collect(); + let qualities = if quality == 3 { + "*".to_owned() + } else { + std::iter::repeat_n((33 + [0u8, 20, 40][quality as usize]) as char, 5).collect() + }; + let cigar = ["5M", "2M2D3M", "2M3N3M", "1S2M1I1M", "5I"][shape as usize]; + let flags = [0, 4, 65, 129, 256, 2048][flags as usize]; + let mapq = [0, 20, 60, 255][mapq as usize]; + let sam = format!( + "{qname}\t{flags}\t{}\t{}\t{mapq}\t{cigar}\t*\t0\t0\t{sequence}\t{qualities}", + ["a", "b"][tid as usize], + start + 1 + ); + records.push(bam::Record::from_sam(&view, sam.as_bytes()).unwrap()); + } + records.sort_by_key(|read| (read.tid(), read.pos())); + let mut writer = bam::Writer::from_path(&path, &header, bam::Format::Bam).unwrap(); + for record in records { + writer.write(&record).unwrap(); + } + drop(writer); + bam::index::build(&path, None, bam::index::Type::Bai, 1).unwrap(); + (directory, path) + } + + fn tsv(positions: &[PileupPosition]) -> Vec { + let mut writer = csv::WriterBuilder::new() + .delimiter(b'\t') + .from_writer(Vec::new()); + for position in positions { + writer.serialize(position).unwrap(); + } + writer.into_inner().unwrap() + } + + fn compare( + path: &Path, + strategy: Option, + depth: u32, + filter: DefaultReadFilter, + baseq: Option, + ) { + let counter = SeqairCounter::open(path, None, filter, strategy, depth, baseq).unwrap(); + let mut reader = bam::IndexedReader::from_path(path).unwrap(); + let header = reader.header().clone(); + let mut worker = counter.worker().unwrap(); + for (tid, start, stop) in [(0, 0, 64), (0, 5, 18), (1, 0, 64), (0, 5, 18), (1, 8, 8)] { + reader.fetch((tid, start, stop)).unwrap(); + let mut pileups = reader.pileup(); + pileups.set_max_depth(depth); + let mut mates = MateCounter::default(); + let mut expected = Vec::new(); + for pileup in pileups { + let pileup = pileup.unwrap(); + if pileup.pos() < start || pileup.pos() >= stop { + continue; + } + let raw_depth = pileup.depth(); + let mut position = match strategy { + Some(strategy) => mates.count(pileup, &header, &filter, baseq, strategy), + None => PileupPosition::from_pileup(pileup, &header, &filter, baseq), + }; + position.near_max_depth = raw_depth > depth - (depth as f64 * 0.01) as u32; + expected.push(position); + } + let mut actual = Vec::new(); + worker + .for_each_position(tid, start, stop, |position| { + actual.push(position); + Ok(()) + }) + .unwrap(); + assert_eq!( + String::from_utf8(tsv(&actual)).unwrap(), + String::from_utf8(tsv(&expected)).unwrap(), + "strategy={strategy:?}, depth={depth}, region={tid}:{start}-{stop}" + ); + } + } + + const STRATEGIES: [MateResolutionStrategy; 9] = [ + MateResolutionStrategy::Original, + MateResolutionStrategy::IUPAC, + MateResolutionStrategy::N, + MateResolutionStrategy::BaseQualMapQualFirstInPair, + MateResolutionStrategy::BaseQualMapQualIUPAC, + MateResolutionStrategy::BaseQualMapQualN, + MateResolutionStrategy::MapQualBaseQualFirstInPair, + MateResolutionStrategy::MapQualBaseQualIUPAC, + MateResolutionStrategy::MapQualBaseQualN, + ]; + + proptest! { + #![proptest_config(ProptestConfig::with_cases(16))] + #[test] + fn matches_htslib_for_overlaps_filters_and_depth_caps( + reads in prop::collection::vec((0u8..7, 0u8..2, 0u8..20, 0u8..5, 0u8..6, 0u8..4, 0u8..5, 0u8..4), 0..45) + ) { + let (_directory, path) = fixture(&reads); + for strategy in [None].into_iter().chain(STRATEGIES.map(Some)) { + for depth in [0, 1, 2, 7, 100000] { + compare(&path, strategy, depth, DefaultReadFilter::new(0, 0, 0), None); + compare(&path, strategy, depth, DefaultReadFilter::new(1, 256, 20), Some(20)); + } + } + } + } + + #[test] + fn disjoint_name_groups_and_end_boundaries_match_htslib() { + // Abutting spans, a one-base overlap, and a later overlap after a + // disjoint prefix. The same worker is reused by compare(). + for starts in [[0, 5, 10], [0, 4, 10], [0, 5, 9], [0, 0, 10]] { + for shape in 0..5 { + let reads: Vec<_> = starts + .into_iter() + .enumerate() + .map(|(i, start)| (1, 0, start, shape, i as u8 + 2, 2, i as u8, 2)) + .collect(); + let (_directory, path) = fixture(&reads); + for strategy in STRATEGIES { + for depth in [1, 7, 100000] { + compare( + &path, + Some(strategy), + depth, + DefaultReadFilter::new(0, 0, 0), + None, + ); + compare( + &path, + Some(strategy), + depth, + DefaultReadFilter::new(0, 256, 20), + Some(20), + ); + } + } + } + } + } + + #[test] + fn rejects_long_cigars_instead_of_counting_the_placeholder() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("long.bam"); + let mut header = bam::Header::new(); + header.push_record( + bam::header::HeaderRecord::new(b"SQ") + .push_tag(b"SN", "a") + .push_tag(b"LN", 100000), + ); + let mut record = bam::Record::new(); + let cigar = bam::record::CigarString( + [bam::record::Cigar::Match(1), bam::record::Cigar::Ins(1)].repeat(32768), + ); + record.set( + b"long", + Some(&cigar), + &b"AC".repeat(32768), + &vec![40; 65536], + ); + record.set_tid(0); + record.set_pos(1); + record.set_flags(0); + let mut writer = bam::Writer::from_path(&path, &header, bam::Format::Bam).unwrap(); + writer.write(&record).unwrap(); + drop(writer); + bam::index::build(&path, None, bam::index::Type::Bai, 1).unwrap(); + let counter = SeqairCounter::open( + &path, + None, + DefaultReadFilter::new(0, 0, 0), + None, + 100000, + None, + ) + .unwrap(); + let error = counter.count_region(0, 1, 8).unwrap_err(); + assert!( + error + .to_string() + .contains("rerun base-depth without --seqair") + ); + } + + #[test] + fn preserves_ties_in_large_same_name_groups() { + for size in [2, 3, 16, 32, 80] { + let reads: Vec<_> = (0..size).map(|i| (1, 0, 0, 0, 0, 2, i % 5, 2)).collect(); + let (_directory, path) = fixture(&reads); + for strategy in STRATEGIES { + compare( + &path, + Some(strategy), + 100000, + DefaultReadFilter::new(0, 0, 0), + None, + ); + } + } + } +} diff --git a/src/lib/read_filter.rs b/src/lib/read_filter.rs index c0a6a82..aa5adf2 100644 --- a/src/lib/read_filter.rs +++ b/src/lib/read_filter.rs @@ -9,6 +9,7 @@ pub trait ReadFilter { } /// A straightforward read filter. +#[derive(Clone, Copy)] pub struct DefaultReadFilter { include_flags: u16, exclude_flags: u16, @@ -24,15 +25,19 @@ impl DefaultReadFilter { min_mapq, } } + + /// Apply the SAM flag and mapping-quality filters without an HTSlib record. + pub fn filter_fields(&self, flags: u16, mapq: u8) -> bool { + (!flags) & self.include_flags == 0 + && flags & self.exclude_flags == 0 + && mapq >= self.min_mapq + } } impl ReadFilter for DefaultReadFilter { /// Filter reads based SAM flags and mapping quality #[inline(always)] fn filter_read(&self, read: &Record, _alignment: Option<&Alignment>) -> bool { - let flags = read.flags(); - (!flags) & self.include_flags == 0 - && flags & self.exclude_flags == 0 - && read.mapq() >= self.min_mapq + self.filter_fields(read.flags(), read.mapq()) } }