Skip to content

refactor: simplify xor construction and local naming #326

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
tisonkun merged 6 commits into from
Nov 5, 2026
Merged

refactor: simplify xor construction and local naming #326

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
d729a06
refactor(xor): simplify raw parts and construction
tisonkun Nov 5, 2026
b3ede56
refactor(xor): replace builders and parts with direct constructors
tisonkun Nov 5, 2026
d09c823
refactor: shorten names for local values and parameters
tisonkun Nov 5, 2026
e3b508e
refactor(xor): unify hash construction and encapsulate serialization
tisonkun Nov 5, 2026
b96d88f
refactor(frequencies): retain descriptive error type parameters
tisonkun Nov 5, 2026
11de76f
refactor(xor): keep serialization sizing private
tisonkun Nov 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ All significant changes to this project will be documented in this file.
### New features

* Add `HllUnion::serialize` to serialize union results directly.
* Add immutable `XorFilter` and `XorFilterBuilder` behind the `xor` feature, with 8-, 16-, and 32-bit fingerprints, precomputed-hash queries and borrowed distinct-hash construction, bounded seeded retries and compact temporary storage, optional cardinality metadata for imported filters, standard serialization for all three widths (Java-compatible for 8/16-bit filters), and raw parts for application-defined storage.
* Add immutable `XorFilter` behind the `xor` feature, with 8-, 16-, and 32-bit fingerprints, construction from values with automatic deduplication or borrowed distinct precomputed hashes, bounded seeded retries and compact temporary storage, optional cardinality metadata for imported filters, standard serialization for all three widths (Java-compatible for 8/16-bit filters), and validated imports for application-defined storage.

### Improvements

Expand Down
62 changes: 22 additions & 40 deletions benchmarks/xor/build.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ use std::hint::black_box;

use benchmarks::Bencher;
use benchmarks::Runner;
use datasketches::xor::XorFilterBuilder;
use datasketches::xor::XorFilter;
use datasketches::xor::XorFilterType;

use crate::xor::hashes;
Expand All @@ -31,13 +31,8 @@ fn repeated_hashes(rows: usize) -> Vec<u64> {

fn distinct_8(bencher: Bencher, cardinality: usize) {
let keys = hashes(cardinality);
bencher.bench_local(|| {
black_box(
XorFilterBuilder::new(XorFilterType::Xor8)
.build_from_distinct_hashes(&keys)
.unwrap(),
)
});
bencher
.bench_local(|| black_box(XorFilter::from_hashes(&keys, XorFilterType::Xor8, 0).unwrap()));
}

fn xorf_distinct_8(bencher: Bencher, cardinality: usize) {
Expand All @@ -48,18 +43,17 @@ fn xorf_distinct_8(bencher: Bencher, cardinality: usize) {
fn deduplicate_8(bencher: Bencher, rows: usize) {
let keys = repeated_hashes(rows);
bencher.bench_local(|| {
black_box(
XorFilterBuilder::new(XorFilterType::Xor8)
.build_from_hashes(keys.iter().copied())
.unwrap(),
)
let mut distinct = keys.clone();
distinct.sort_unstable();
distinct.dedup();
black_box(XorFilter::from_hashes(&distinct, XorFilterType::Xor8, 0).unwrap())
});
}

fn xorf_deduplicate_8(bencher: Bencher, rows: usize) {
let keys = repeated_hashes(rows);
bencher.bench_local(|| {
// xorf requires distinct input, so include the caller's preprocessing.
// Include the caller's deduplication, as for DataSketches.
let mut distinct = keys.clone();
distinct.sort_unstable();
distinct.dedup();
Expand All @@ -69,13 +63,8 @@ fn xorf_deduplicate_8(bencher: Bencher, rows: usize) {

fn distinct_16(bencher: Bencher, cardinality: usize) {
let keys = hashes(cardinality);
bencher.bench_local(|| {
black_box(
XorFilterBuilder::new(XorFilterType::Xor16)
.build_from_distinct_hashes(&keys)
.unwrap(),
)
});
bencher
.bench_local(|| black_box(XorFilter::from_hashes(&keys, XorFilterType::Xor16, 0).unwrap()));
}

fn xorf_distinct_16(bencher: Bencher, cardinality: usize) {
Expand All @@ -86,18 +75,17 @@ fn xorf_distinct_16(bencher: Bencher, cardinality: usize) {
fn deduplicate_16(bencher: Bencher, rows: usize) {
let keys = repeated_hashes(rows);
bencher.bench_local(|| {
black_box(
XorFilterBuilder::new(XorFilterType::Xor16)
.build_from_hashes(keys.iter().copied())
.unwrap(),
)
let mut distinct = keys.clone();
distinct.sort_unstable();
distinct.dedup();
black_box(XorFilter::from_hashes(&distinct, XorFilterType::Xor16, 0).unwrap())
});
}

fn xorf_deduplicate_16(bencher: Bencher, rows: usize) {
let keys = repeated_hashes(rows);
bencher.bench_local(|| {
// xorf requires distinct input, so include the caller's preprocessing.
// Include the caller's deduplication, as for DataSketches.
let mut distinct = keys.clone();
distinct.sort_unstable();
distinct.dedup();
Expand All @@ -107,13 +95,8 @@ fn xorf_deduplicate_16(bencher: Bencher, rows: usize) {

fn distinct_32(bencher: Bencher, cardinality: usize) {
let keys = hashes(cardinality);
bencher.bench_local(|| {
black_box(
XorFilterBuilder::new(XorFilterType::Xor32)
.build_from_distinct_hashes(&keys)
.unwrap(),
)
});
bencher
.bench_local(|| black_box(XorFilter::from_hashes(&keys, XorFilterType::Xor32, 0).unwrap()));
}

fn xorf_distinct_32(bencher: Bencher, cardinality: usize) {
Expand All @@ -124,18 +107,17 @@ fn xorf_distinct_32(bencher: Bencher, cardinality: usize) {
fn deduplicate_32(bencher: Bencher, rows: usize) {
let keys = repeated_hashes(rows);
bencher.bench_local(|| {
black_box(
XorFilterBuilder::new(XorFilterType::Xor32)
.build_from_hashes(keys.iter().copied())
.unwrap(),
)
let mut distinct = keys.clone();
distinct.sort_unstable();
distinct.dedup();
black_box(XorFilter::from_hashes(&distinct, XorFilterType::Xor32, 0).unwrap())
});
}

fn xorf_deduplicate_32(bencher: Bencher, rows: usize) {
let keys = repeated_hashes(rows);
bencher.bench_local(|| {
// xorf requires distinct input, so include the caller's preprocessing.
// Include the caller's deduplication, as for DataSketches.
let mut distinct = keys.clone();
distinct.sort_unstable();
distinct.dedup();
Expand Down
20 changes: 5 additions & 15 deletions benchmarks/xor/build_batches.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,14 +19,13 @@ use std::hint::black_box;

use benchmarks::Bencher;
use benchmarks::Runner;
use datasketches::xor::XorFilterBuilder;
use datasketches::xor::XorFilter;
use datasketches::xor::XorFilterType;

use crate::xor::hashes_with_offset;

fn build<F>(bencher: Bencher, cardinality: usize, build: impl Fn(&[u64]) -> F) {
// Multiple independent sets keep a particular seed's retry count on one set
// from dominating the comparison of default construction policies.
// Multiple independent sets keep one set's retry count from dominating the comparison.
let inputs: Vec<_> = (0..32)
.map(|i| hashes_with_offset(cardinality, i * cardinality as u64))
.collect();
Expand All @@ -38,24 +37,15 @@ fn build<F>(bencher: Bencher, cardinality: usize, build: impl Fn(&[u64]) -> F) {
}

pub fn register(runner: &mut Runner) {
// Use xorf's initial seed for each constructor.
for (cardinality, rounds) in [(128, 10_000), (16_384, 40), (50_000, 12)] {
runner.compare(
&format!("xor::build_batches::8/{cardinality}"),
rounds,
&[
("default_seed", &|b| {
("distinct", &|b| {
build(b, cardinality, |keys| {
XorFilterBuilder::new(XorFilterType::Xor8)
.build_from_distinct_hashes(keys)
.unwrap()
})
}),
("matched_seed", &|b| {
build(b, cardinality, |keys| {
XorFilterBuilder::new(XorFilterType::Xor8)
.seed(1)
.build_from_distinct_hashes(keys)
.unwrap()
XorFilter::from_hashes(keys, XorFilterType::Xor8, 1).unwrap()
})
}),
("xorf", &|b| {
Expand Down
26 changes: 7 additions & 19 deletions benchmarks/xor/query.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,17 +19,15 @@ use std::hint::black_box;

use benchmarks::Bencher;
use benchmarks::Runner;
use datasketches::xor::XorFilterBuilder;
use datasketches::xor::XorFilter;
use datasketches::xor::XorFilterType;
use xorf::Filter as _;

use crate::xor::hashes;

fn datasketches_8_present(bencher: Bencher, cardinality: usize) {
let keys = hashes(cardinality * 2);
let filter = XorFilterBuilder::new(XorFilterType::Xor8)
.build_from_distinct_hashes(&keys[..cardinality])
.unwrap();
let filter = XorFilter::from_hashes(&keys[..cardinality], XorFilterType::Xor8, 0).unwrap();
bencher.bench_local(|| {
for key in &keys[..cardinality] {
black_box(filter.contains_hash(black_box(*key)));
Expand All @@ -39,9 +37,7 @@ fn datasketches_8_present(bencher: Bencher, cardinality: usize) {

fn datasketches_8_absent(bencher: Bencher, cardinality: usize) {
let keys = hashes(cardinality * 2);
let filter = XorFilterBuilder::new(XorFilterType::Xor8)
.build_from_distinct_hashes(&keys[..cardinality])
.unwrap();
let filter = XorFilter::from_hashes(&keys[..cardinality], XorFilterType::Xor8, 0).unwrap();
bencher.bench_local(|| {
for key in &keys[cardinality..] {
black_box(filter.contains_hash(black_box(*key)));
Expand Down Expand Up @@ -71,9 +67,7 @@ fn xorf_8_absent(bencher: Bencher, cardinality: usize) {

fn datasketches_16_present(bencher: Bencher, cardinality: usize) {
let keys = hashes(cardinality * 2);
let filter = XorFilterBuilder::new(XorFilterType::Xor16)
.build_from_distinct_hashes(&keys[..cardinality])
.unwrap();
let filter = XorFilter::from_hashes(&keys[..cardinality], XorFilterType::Xor16, 0).unwrap();
bencher.bench_local(|| {
for key in &keys[..cardinality] {
black_box(filter.contains_hash(black_box(*key)));
Expand All @@ -83,9 +77,7 @@ fn datasketches_16_present(bencher: Bencher, cardinality: usize) {

fn datasketches_16_absent(bencher: Bencher, cardinality: usize) {
let keys = hashes(cardinality * 2);
let filter = XorFilterBuilder::new(XorFilterType::Xor16)
.build_from_distinct_hashes(&keys[..cardinality])
.unwrap();
let filter = XorFilter::from_hashes(&keys[..cardinality], XorFilterType::Xor16, 0).unwrap();
bencher.bench_local(|| {
for key in &keys[cardinality..] {
black_box(filter.contains_hash(black_box(*key)));
Expand Down Expand Up @@ -115,9 +107,7 @@ fn xorf_16_absent(bencher: Bencher, cardinality: usize) {

fn datasketches_32_present(bencher: Bencher, cardinality: usize) {
let keys = hashes(cardinality * 2);
let filter = XorFilterBuilder::new(XorFilterType::Xor32)
.build_from_distinct_hashes(&keys[..cardinality])
.unwrap();
let filter = XorFilter::from_hashes(&keys[..cardinality], XorFilterType::Xor32, 0).unwrap();
bencher.bench_local(|| {
for key in &keys[..cardinality] {
black_box(filter.contains_hash(black_box(*key)));
Expand All @@ -127,9 +117,7 @@ fn datasketches_32_present(bencher: Bencher, cardinality: usize) {

fn datasketches_32_absent(bencher: Bencher, cardinality: usize) {
let keys = hashes(cardinality * 2);
let filter = XorFilterBuilder::new(XorFilterType::Xor32)
.build_from_distinct_hashes(&keys[..cardinality])
.unwrap();
let filter = XorFilter::from_hashes(&keys[..cardinality], XorFilterType::Xor32, 0).unwrap();
bencher.bench_local(|| {
for key in &keys[cardinality..] {
black_box(filter.contains_hash(black_box(*key)));
Expand Down
3 changes: 1 addition & 2 deletions benchmarks/xor/query_shards.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ use std::hint::black_box;
use benchmarks::Bencher;
use benchmarks::Runner;
use datasketches::xor::XorFilter;
use datasketches::xor::XorFilterParts;
use datasketches::xor::XorFilterType;
use datasketches::xor::XorFingerprints;
use xorf::Filter;
Expand Down Expand Up @@ -74,7 +73,7 @@ impl Reference {
XorFingerprints::Xor32(f.fingerprints),
),
};
XorFilter::from_parts(XorFilterParts::new(seed, segment, fingerprints)).unwrap()
XorFilter::from_parts(seed, segment, fingerprints, None).unwrap()
}
}

Expand Down
14 changes: 7 additions & 7 deletions datasketches/src/cpc/sketch.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -368,18 +368,18 @@ impl CpcSketch {

self.surprising_value_table_mut().clear(); // the new number of surprises will be about the same

let mask_for_clearing_window = (0xFF << new_offset) ^ u64::MAX;
let mask_for_flipping_early_zone = (1u64 << new_offset) - 1;
let clear_mask = (0xFF << new_offset) ^ u64::MAX;
let flip_mask = (1u64 << new_offset) - 1;

let mut all_surprises_ored = 0u64;
let mut surprises = 0u64;
for i in 0..k {
let mut pattern = bit_matrix[i];
self.sliding_window[i] = ((pattern >> new_offset) & 0xff) as u8;
pattern &= mask_for_clearing_window;
pattern &= clear_mask;
// The following line converts surprising 0's to 1's in the "early zone",
// (and vice versa, which is essential for this procedure's O(k) time cost).
pattern ^= mask_for_flipping_early_zone;
all_surprises_ored |= pattern; // a cheap way to recalculate first_interesting_column
pattern ^= flip_mask;
surprises |= pattern; // a cheap way to recalculate first_interesting_column
while pattern != 0 {
let col = pattern.trailing_zeros();
pattern ^= 1 << col; // erase the 1
Expand All @@ -390,7 +390,7 @@ impl CpcSketch {
}

self.window_offset = new_offset;
self.first_interesting_column = all_surprises_ored.trailing_zeros() as u8;
self.first_interesting_column = surprises.trailing_zeros() as u8;
if self.first_interesting_column > new_offset {
self.first_interesting_column = new_offset; // corner case
}
Expand Down
14 changes: 7 additions & 7 deletions datasketches/src/cpc/union.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -171,18 +171,18 @@ impl CpcUnion {
let mut table = PairTable::new(new_table_lg_size, 6 + lg_k);

// the following should work even when the offset is zero
let mask_for_clearing_window = (0xFFu64 << offset) ^ u64::MAX;
let mask_for_flipping_early_zone = (1u64 << offset) - 1;
let mut all_surprises_ored = 0;
let clear_mask = (0xFFu64 << offset) ^ u64::MAX;
let flip_mask = (1u64 << offset) - 1;
let mut surprises = 0;

// The snowplow effect was caused by processing the rows in order,
// but we have fixed it by using a sufficiently large hash table.
for i in 0..k {
let mut pattern = matrix[i];
sliding_window[i] = ((pattern >> offset) & 0xFF) as u8;
pattern &= mask_for_clearing_window;
pattern ^= mask_for_flipping_early_zone; // this flipping converts surprising 0's to 1's
all_surprises_ored |= pattern;
pattern &= clear_mask;
pattern ^= flip_mask; // this flipping converts surprising 0's to 1's
surprises |= pattern;
while pattern != 0 {
let col = pattern.trailing_zeros();
pattern ^= 1u64 << col; // erase the 1
Expand All @@ -194,7 +194,7 @@ impl CpcUnion {

// at this point we could shrink an oversized hash table, but the relative waste
// isn't very big
sketch.first_interesting_column = all_surprises_ored.trailing_zeros() as u8;
sketch.first_interesting_column = surprises.trailing_zeros() as u8;
if sketch.first_interesting_column > offset {
sketch.first_interesting_column = offset; // corner case
}
Expand Down
14 changes: 7 additions & 7 deletions datasketches/src/hll/union.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -555,9 +555,9 @@ fn merge_array_with_downsample(dst: &mut Array8, dst_lg_k: u8, src_mode: &Mode,

/// Changing representation preserves the estimate state; replaying registers alone would change
/// HIP.
fn convert_array8_to_type(src: &Array8, lg_config_k: u8, target_type: HllType) -> HllSketch {
let estimate_state = src.estimate_state();
match target_type {
fn convert_array8_to_type(src: &Array8, lg_config_k: u8, ty: HllType) -> HllSketch {
let state = src.estimate_state();
match ty {
HllType::Hll8 => HllSketch::from_mode(lg_config_k, Mode::Array8(src.clone())),
HllType::Hll6 => {
let mut array6 = Array6::new(lg_config_k);
Expand All @@ -569,7 +569,7 @@ fn convert_array8_to_type(src: &Array8, lg_config_k: u8, target_type: HllType) -
array6.update(coupon);
}
}
array6.restore_estimate_state(estimate_state);
array6.restore_estimate_state(state);

HllSketch::from_mode(lg_config_k, Mode::Array6(array6))
}
Expand All @@ -582,7 +582,7 @@ fn convert_array8_to_type(src: &Array8, lg_config_k: u8, target_type: HllType) -
array4.update(coupon);
}
}
array4.restore_estimate_state(estimate_state);
array4.restore_estimate_state(state);

HllSketch::from_mode(lg_config_k, Mode::Array4(array4))
}
Expand All @@ -601,7 +601,7 @@ fn copy_array46_via_coupons(dst: &mut Array8, num_registers: usize, get_value: i

/// Preserves the source estimate state because the result represents the same logical sketch.
fn copy_or_downsample(src_mode: &Mode, src_lg_k: u8, tgt_lg_k: u8) -> Array8 {
let estimate_state = get_array_estimate_state(src_mode);
let state = get_array_estimate_state(src_mode);

let mut result = if src_lg_k <= tgt_lg_k {
match src_mode {
Expand All @@ -628,6 +628,6 @@ fn copy_or_downsample(src_mode: &Mode, src_lg_k: u8, tgt_lg_k: u8) -> Array8 {
result
};

result.restore_estimate_state(estimate_state);
result.restore_estimate_state(state);
result
}
Loading
Loading