diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c8b17f..90c2284 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,9 @@ - Extended `BitVectorBuilder` with `push_bits` and `set_bit` APIs. - Added `from_bit` constructor on `BitVectorBuilder` for repeating a single bit. - `DacsByte` now stores level data as zero-copy `View<[u8]>` values. +- Added `to_bytes` and `from_bytes` on `DacsByte` for zero-copy serialization. +- Documented the byte layout produced by `DacsByte::to_bytes` with ASCII art. +- Flags are serialized before level data to eliminate padding. - Added `get_bits` methods to `BitVectorData` and `BitVector`. - Removed deprecated `size_in_bytes` helpers. - Added `scripts/devtest.sh` and `scripts/preflight.sh` for testing and diff --git a/INVENTORY.md b/INVENTORY.md index 117ad8e..996d892 100644 --- a/INVENTORY.md +++ b/INVENTORY.md @@ -9,6 +9,7 @@ - Investigate alternative dense-select index strategies to replace removed `DArrayIndex`. - Explore additional index implementations leveraging the new generic `DacsByte`. - Demonstrate the generic `from_slice` usage in examples and docs. +- Showcase `DacsByte` byte serialization in an example. - Provide serialization helpers for additional structures beyond `WaveletMatrix`. - Show `CompactVector::to_bytes` and `from_bytes` in examples. diff --git a/README.md b/README.md index 9ac37e4..e01008d 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,20 @@ is backed by `anybytes::View`. The data can be serialized with `BitVectorData::to_bytes` and reconstructed using `BitVectorData::from_bytes`, allowing zero-copy loading from an mmap or any other source by passing the byte region to `Bytes::from_source`. + +`DacsByte` sequences support a similar interface with `to_bytes` returning +metadata alongside the byte slice and `from_bytes` rebuilding the sequence +using that metadata. + +```text +Bytes layout from `DacsByte::to_bytes`: + +| flag[0] words | flag[1] words | ... | flag[n-2] words | level[0] data | level[1] data | ... | level[n-1] data | + +The flag vectors come first and store native-endian `usize` words. The level +data immediately follows without any padding. +``` + `CompactVector` offers similar helpers: `CompactVector::to_bytes` returns a metadata struct along with the raw bytes, and `CompactVector::from_bytes` reconstructs the vector from that information. diff --git a/src/int_vectors/dacs_byte.rs b/src/int_vectors/dacs_byte.rs index f08e16d..4e91393 100644 --- a/src/int_vectors/dacs_byte.rs +++ b/src/int_vectors/dacs_byte.rs @@ -13,6 +13,8 @@ use anybytes::{Bytes, View}; const LEVEL_WIDTH: usize = 8; const LEVEL_MASK: usize = (1 << LEVEL_WIDTH) - 1; +/// Maximum possible number of levels for a [`usize`] value. +const MAX_LEVELS: usize = (usize::BITS as usize + LEVEL_WIDTH - 1) / LEVEL_WIDTH; /// Compressed integer sequence using Directly Addressable Codes (DACs) in a simple bytewise scheme. /// @@ -60,6 +62,45 @@ pub struct DacsByte { flags: Vec>, } +/// Metadata required to reconstruct a `DacsByte` from bytes. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DacsByteMeta { + /// Number of valid levels stored. + pub num_levels: usize, + /// Byte length for each level in order. + pub level_lens: Vec, + /// Metadata for each flag bit vector between levels. + pub flag_meta: Vec, +} + +/// Metadata describing a flag bit vector stored inside a `DacsByte`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct FlagMeta { + /// Number of bits stored in the bit vector. + pub len_bits: usize, + /// Number of machine words used to hold the bits. + pub num_words: usize, +} + +impl Default for FlagMeta { + fn default() -> Self { + Self { + len_bits: 0, + num_words: 0, + } + } +} + +impl Default for DacsByteMeta { + fn default() -> Self { + Self { + num_levels: 0, + level_lens: Vec::new(), + flag_meta: Vec::new(), + } + } +} + impl DacsByte { /// Builds DACs by assigning 8 bits to represent each level. /// @@ -179,6 +220,95 @@ impl DacsByte { pub fn widths(&self) -> Vec { self.data.iter().map(|_| LEVEL_WIDTH).collect() } + + /// Serializes the sequence into a [`Bytes`] buffer. + /// + /// Returns the metadata necessary for [`from_bytes`]. + pub fn to_bytes(&self) -> (DacsByteMeta, Bytes) { + let level_lens = self.data.iter().map(|v| v.len()).collect::>(); + let flag_meta = self + .flags + .iter() + .map(|f| FlagMeta { + len_bits: f.data.len, + num_words: f.data.num_words(), + }) + .collect::>(); + + let mut buf: Vec = Vec::new(); + for flag in &self.flags { + for &word in flag.data.words.as_ref() { + buf.extend_from_slice(&word.to_ne_bytes()); + } + } + + for level in &self.data { + buf.extend_from_slice(level.as_ref()); + } + + ( + DacsByteMeta { + num_levels: self.data.len(), + level_lens, + flag_meta, + }, + Bytes::from_source(buf), + ) + } + + /// Reconstructs the sequence from zero-copy [`Bytes`]. + /// + /// The `meta` argument should come from [`to_bytes`]. + pub fn from_bytes(meta: DacsByteMeta, bytes: Bytes) -> Result { + use std::mem::size_of; + + let usize_size = size_of::(); + let mut cursor = 0; + let slice = bytes.as_ref(); + + if meta.num_levels == 0 + || meta.num_levels > MAX_LEVELS + || meta.level_lens.len() != meta.num_levels + || meta.flag_meta.len() != meta.num_levels.saturating_sub(1) + { + return Err(anyhow!("invalid metadata")); + } + + let mut flags = Vec::with_capacity(meta.flag_meta.len()); + for fm in &meta.flag_meta { + let bytes_len = fm.num_words * usize_size; + if cursor + bytes_len > slice.len() { + return Err(anyhow!("insufficient bytes")); + } + let words_view = bytes + .slice_to_bytes(&slice[cursor..cursor + bytes_len]) + .ok_or_else(|| anyhow!("invalid slice"))? + .view::<[usize]>() + .map_err(|e| anyhow!(e))?; + cursor += bytes_len; + let data = bit_vector::BitVectorData { + words: words_view, + len: fm.len_bits, + }; + let index = I::build(&data); + flags.push(bit_vector::BitVector { data, index }); + } + + let mut data = Vec::with_capacity(meta.num_levels); + for &len in &meta.level_lens { + if cursor + len > slice.len() { + return Err(anyhow!("insufficient bytes")); + } + let view_bytes = bytes + .slice_to_bytes(&slice[cursor..cursor + len]) + .ok_or_else(|| anyhow!("invalid slice"))?; + let view = view_bytes.view::<[u8]>().map_err(|e| anyhow!(e))?; + data.push(view); + cursor += len; + } + + Ok(Self { data, flags }) + } } impl Default for DacsByte { @@ -372,6 +502,14 @@ mod tests { assert_eq!(seq.to_vec(), vec![5, 7]); } + #[test] + fn bytes_roundtrip() { + let seq = DacsByte::::from_slice(&[5, 0, 100000, 334]).unwrap(); + let (meta, bytes) = seq.to_bytes(); + let other = DacsByte::::from_bytes(meta, bytes).unwrap(); + assert_eq!(seq, other); + } + #[test] fn test_from_slice_uncastable() { let e = DacsByte::::from_slice(&[u128::MAX]);