From ce2bad36417ad89e00652f09c05bd08c91d8532b Mon Sep 17 00:00:00 2001 From: tison Date: Tue, 27 Oct 2026 12:29:00 +0800 Subject: [PATCH 1/6] feat(xor): add immutable filters with portable serialization --- CHANGELOG.md | 4 + CONTRIBUTING.md | 2 + Cargo.lock | 8 + README.md | 3 +- benchmarks/Cargo.toml | 2 + benchmarks/main.rs | 1 + benchmarks/xor/build.rs | 108 +++ benchmarks/xor/matched_build.rs | 104 +++ benchmarks/xor/mod.rs | 36 + benchmarks/xor/query.rs | 187 +++++ datasketches/Cargo.toml | 1 + datasketches/src/codec/family.rs | 9 + datasketches/src/codec/mod.rs | 2 + datasketches/src/error.rs | 3 + datasketches/src/hash/mod.rs | 7 +- datasketches/src/lib.rs | 5 +- datasketches/src/xor/filter.rs | 765 +++++++++++++++++++++ datasketches/src/xor/mod.rs | 79 +++ datasketches/src/xor/serialization.rs | 232 +++++++ tests-integration/Cargo.toml | 2 + tests-integration/tests/serde_tests.rs | 3 + tests-integration/tests/serde_tests/xor.rs | 283 ++++++++ tests-integration/tests/xor_test/filter.rs | 154 +++++ tests-integration/tests/xor_test/main.rs | 20 + tests-integration/tests/xor_test/parts.rs | 223 ++++++ 25 files changed, 2238 insertions(+), 5 deletions(-) create mode 100644 benchmarks/xor/build.rs create mode 100644 benchmarks/xor/matched_build.rs create mode 100644 benchmarks/xor/mod.rs create mode 100644 benchmarks/xor/query.rs create mode 100644 datasketches/src/xor/filter.rs create mode 100644 datasketches/src/xor/mod.rs create mode 100644 datasketches/src/xor/serialization.rs create mode 100644 tests-integration/tests/serde_tests/xor.rs create mode 100644 tests-integration/tests/xor_test/filter.rs create mode 100644 tests-integration/tests/xor_test/main.rs create mode 100644 tests-integration/tests/xor_test/parts.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index af0429a4..742aad27 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ All significant changes to this project will be documented in this file. ## Unreleased +### New features + +* Add immutable `XorFilter` and `XorFilterBuilder` behind the `xor` feature, with 8-, 16-, and 32-bit fingerprints, bounded seeded construction with recoverable builder failures, optional cardinality metadata for imported filters, Java-compatible standard serialization for 8/16-bit filters, and raw parts for application-defined storage. + ## v0.6.0 ### Breaking changes diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c02d8d16..2798be7c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -168,6 +168,8 @@ cargo x prepare-testdata java If no language is specified, all languages are prepared. These directories are not stored in Git. Run the command before the first test run and again whenever the pinned TCK revision changes. It requires network access and replaces the selected generated directories. +The pinned TCK revision does not yet contain XOR filter snapshots. Until it does, `serde_tests/xor.rs` retains small hexadecimal fixtures generated from the merged Java implementation, with the source revision and inputs documented alongside them. Keep these separate from the synchronized `*_generated_files` directories; replace them with TCK fixtures when available. + ## Code of Conduct We expect all community members to follow our [Code of Conduct](https://www.apache.org/foundation/policies/conduct.html). diff --git a/Cargo.lock b/Cargo.lock index 81862f6d..2421b061 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -86,6 +86,7 @@ dependencies = [ "datasketches", "divan", "rand", + "xorf", ] [[package]] @@ -782,6 +783,7 @@ dependencies = [ "googletest", "insta", "quickcheck", + "xorf", ] [[package]] @@ -1003,6 +1005,12 @@ dependencies = [ "which", ] +[[package]] +name = "xorf" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f13846c9009c0e550697ddde851e934b5dfafbce757a5ef5edfdb27ea20803e" + [[package]] name = "zeroize" version = "1.9.0" diff --git a/README.md b/README.md index 7ec87caf..4bcc3be3 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,7 @@ Enable multiple algorithms by listing their features together, such as `features | `tdigest` | `TDigestMut`, `TDigest` | Quantile and rank estimation, with high accuracy near distribution tails. | | `theta` | `ThetaSketch` and set operations | Distinct counts, set expressions, and Jaccard similarity. | | `tuple` | `TupleSketch` and set operations | Theta-style keys with user-defined summaries attached to retained entries. | +| `xor` | `XorFilter`, `XorFilterBuilder` | Immutable probabilistic membership with 8-, 16-, or 32-bit fingerprints. | See the [API documentation](https://docs.rs/datasketches) for configuration, accuracy guarantees, serialization, and examples for each algorithm. Runnable end-to-end scenarios live in [`datasketches/examples`](datasketches/examples). @@ -82,7 +83,7 @@ See the [API documentation](https://docs.rs/datasketches) for configuration, acc The minimum supported Rust version is 1.86.0. The crate currently supports little-endian targets only. -Supported serialization formats are tested with fixtures produced by Apache DataSketches Java, C++, and Go through the [DataSketches TCK](https://github.com/apache/datasketches-tck). +Supported serialization formats are tested with fixtures produced by Apache DataSketches Java, C++, and Go through the [DataSketches TCK](https://github.com/apache/datasketches-tck). XOR additionally uses fixtures generated from the merged Java implementation until TCK snapshots are available. Its standard format supports 8/16-bit fingerprints; 32-bit fingerprints and formats without an item count use the raw-parts API. Serialization compatibility does not imply that an ordinary Rust `Hash` implementation produces the same update bytes across languages. When sketches must represent the same inputs across implementations, use `hash::value::{raw_bytes, canonical_float, sign_extend, natural_extend}` (and the constructors within those modules) to select matching input encodings. When interoperating with an implementation that skips empty strings, skip them before updating here as well. diff --git a/benchmarks/Cargo.toml b/benchmarks/Cargo.toml index 25af32a6..49c3b9b9 100644 --- a/benchmarks/Cargo.toml +++ b/benchmarks/Cargo.toml @@ -23,6 +23,7 @@ edition.workspace = true rust-version.workspace = true [dev-dependencies] +xorf = { version = "=0.13.0", default-features = false } datasketches = { workspace = true, features = [ "bloom", "countmin", @@ -34,6 +35,7 @@ datasketches = { workspace = true, features = [ "tdigest", "theta", "tuple", + "xor", ] } divan = { workspace = true } rand = { workspace = true } diff --git a/benchmarks/main.rs b/benchmarks/main.rs index ee288034..202f8e02 100644 --- a/benchmarks/main.rs +++ b/benchmarks/main.rs @@ -31,6 +31,7 @@ mod req; mod tdigest; mod theta; mod tuple; +mod xor; fn main() { divan::main(); diff --git a/benchmarks/xor/build.rs b/benchmarks/xor/build.rs new file mode 100644 index 00000000..8d212007 --- /dev/null +++ b/benchmarks/xor/build.rs @@ -0,0 +1,108 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use datasketches::xor::XorFilter; +use datasketches::xor::XorFilterType; +use divan::Bencher; +use divan::black_box; +use divan::counter::ItemsCount; + +use crate::xor::hashes; + +#[divan::bench(args = [1_000, 16_384, 49_999, 50_000, 50_001, 100_000])] +fn datasketches_distinct_8(bencher: Bencher, cardinality: usize) { + let keys = hashes(cardinality); + bencher + .counter(ItemsCount::new(cardinality)) + .bench_local(|| { + black_box(XorFilter::from_distinct_hashes(&keys, XorFilterType::Xor8).unwrap()) + }); +} + +#[divan::bench(args = [1_000, 16_384, 49_999, 50_000, 50_001, 100_000])] +fn datasketches_deduplicate_8(bencher: Bencher, cardinality: usize) { + let keys = hashes(cardinality); + bencher + .counter(ItemsCount::new(cardinality)) + .bench_local(|| { + black_box(XorFilter::from_hashes(keys.iter().copied(), XorFilterType::Xor8).unwrap()) + }); +} + +#[divan::bench(args = [1_000, 16_384, 49_999, 50_000, 50_001, 100_000])] +fn xorf_8(bencher: Bencher, cardinality: usize) { + let keys = hashes(cardinality); + bencher + .counter(ItemsCount::new(cardinality)) + .bench_local(|| black_box(xorf::Xor8::from_iterator(keys.iter().copied()))); +} + +#[divan::bench(args = [1_000, 16_384, 49_999, 50_000, 50_001, 100_000])] +fn datasketches_distinct_16(bencher: Bencher, cardinality: usize) { + let keys = hashes(cardinality); + bencher + .counter(ItemsCount::new(cardinality)) + .bench_local(|| { + black_box(XorFilter::from_distinct_hashes(&keys, XorFilterType::Xor16).unwrap()) + }); +} + +#[divan::bench(args = [1_000, 16_384, 49_999, 50_000, 50_001, 100_000])] +fn datasketches_deduplicate_16(bencher: Bencher, cardinality: usize) { + let keys = hashes(cardinality); + bencher + .counter(ItemsCount::new(cardinality)) + .bench_local(|| { + black_box(XorFilter::from_hashes(keys.iter().copied(), XorFilterType::Xor16).unwrap()) + }); +} + +#[divan::bench(args = [1_000, 16_384, 49_999, 50_000, 50_001, 100_000])] +fn xorf_16(bencher: Bencher, cardinality: usize) { + let keys = hashes(cardinality); + bencher + .counter(ItemsCount::new(cardinality)) + .bench_local(|| black_box(xorf::Xor16::from_iterator(keys.iter().copied()))); +} + +#[divan::bench(args = [1_000, 16_384, 49_999, 50_000, 50_001, 100_000])] +fn datasketches_distinct_32(bencher: Bencher, cardinality: usize) { + let keys = hashes(cardinality); + bencher + .counter(ItemsCount::new(cardinality)) + .bench_local(|| { + black_box(XorFilter::from_distinct_hashes(&keys, XorFilterType::Xor32).unwrap()) + }); +} + +#[divan::bench(args = [1_000, 16_384, 49_999, 50_000, 50_001, 100_000])] +fn datasketches_deduplicate_32(bencher: Bencher, cardinality: usize) { + let keys = hashes(cardinality); + bencher + .counter(ItemsCount::new(cardinality)) + .bench_local(|| { + black_box(XorFilter::from_hashes(keys.iter().copied(), XorFilterType::Xor32).unwrap()) + }); +} + +#[divan::bench(args = [1_000, 16_384, 49_999, 50_000, 50_001, 100_000])] +fn xorf_32(bencher: Bencher, cardinality: usize) { + let keys = hashes(cardinality); + bencher + .counter(ItemsCount::new(cardinality)) + .bench_local(|| black_box(xorf::Xor32::from_iterator(keys.iter().copied()))); +} diff --git a/benchmarks/xor/matched_build.rs b/benchmarks/xor/matched_build.rs new file mode 100644 index 00000000..8fbb1add --- /dev/null +++ b/benchmarks/xor/matched_build.rs @@ -0,0 +1,104 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use datasketches::xor::XorFilter; +use datasketches::xor::XorFilterType; +use divan::Bencher; +use divan::black_box; +use divan::counter::ItemsCount; + +use crate::xor::hashes_with_offset; + +// xorf 0.13 starts SplitMix at 1. Both builders use the same seed sequence; +// equal winning seeds therefore imply equal attempt counts. The offset 917504 +// case requires two attempts, while the other listed cases require one. +fn verify_seed(keys: &[u64], kind: XorFilterType) { + let filter = XorFilter::from_distinct_hashes_with_seed(keys, kind, 1).unwrap(); + assert_eq!(filter.seed(), xorf::Xor8::from(keys).seed); +} + +#[divan::bench(args = [(1_000, 0), (16_384, 0), (16_384, 917_504), (49_999, 0), (50_000, 0), (50_001, 0), (100_000, 0)])] +fn datasketches_8(bencher: Bencher, case: (usize, u64)) { + let (cardinality, offset) = case; + let keys = hashes_with_offset(cardinality, offset); + verify_seed(&keys, XorFilterType::Xor8); + bencher + .counter(ItemsCount::new(cardinality)) + .bench_local(|| { + black_box( + XorFilter::from_distinct_hashes_with_seed(&keys, XorFilterType::Xor8, 1).unwrap(), + ) + }); +} + +#[divan::bench(args = [(1_000, 0), (16_384, 0), (16_384, 917_504), (49_999, 0), (50_000, 0), (50_001, 0), (100_000, 0)])] +fn xorf_8(bencher: Bencher, case: (usize, u64)) { + let (cardinality, offset) = case; + let keys = hashes_with_offset(cardinality, offset); + verify_seed(&keys, XorFilterType::Xor8); + bencher + .counter(ItemsCount::new(cardinality)) + .bench_local(|| black_box(xorf::Xor8::from_iterator(keys.iter().copied()))); +} + +#[divan::bench(args = [(1_000, 0), (16_384, 0), (16_384, 917_504), (49_999, 0), (50_000, 0), (50_001, 0), (100_000, 0)])] +fn datasketches_16(bencher: Bencher, case: (usize, u64)) { + let (cardinality, offset) = case; + let keys = hashes_with_offset(cardinality, offset); + verify_seed(&keys, XorFilterType::Xor16); + bencher + .counter(ItemsCount::new(cardinality)) + .bench_local(|| { + black_box( + XorFilter::from_distinct_hashes_with_seed(&keys, XorFilterType::Xor16, 1).unwrap(), + ) + }); +} + +#[divan::bench(args = [(1_000, 0), (16_384, 0), (16_384, 917_504), (49_999, 0), (50_000, 0), (50_001, 0), (100_000, 0)])] +fn xorf_16(bencher: Bencher, case: (usize, u64)) { + let (cardinality, offset) = case; + let keys = hashes_with_offset(cardinality, offset); + verify_seed(&keys, XorFilterType::Xor16); + bencher + .counter(ItemsCount::new(cardinality)) + .bench_local(|| black_box(xorf::Xor16::from_iterator(keys.iter().copied()))); +} + +#[divan::bench(args = [(1_000, 0), (16_384, 0), (16_384, 917_504), (49_999, 0), (50_000, 0), (50_001, 0), (100_000, 0)])] +fn datasketches_32(bencher: Bencher, case: (usize, u64)) { + let (cardinality, offset) = case; + let keys = hashes_with_offset(cardinality, offset); + verify_seed(&keys, XorFilterType::Xor32); + bencher + .counter(ItemsCount::new(cardinality)) + .bench_local(|| { + black_box( + XorFilter::from_distinct_hashes_with_seed(&keys, XorFilterType::Xor32, 1).unwrap(), + ) + }); +} + +#[divan::bench(args = [(1_000, 0), (16_384, 0), (16_384, 917_504), (49_999, 0), (50_000, 0), (50_001, 0), (100_000, 0)])] +fn xorf_32(bencher: Bencher, case: (usize, u64)) { + let (cardinality, offset) = case; + let keys = hashes_with_offset(cardinality, offset); + verify_seed(&keys, XorFilterType::Xor32); + bencher + .counter(ItemsCount::new(cardinality)) + .bench_local(|| black_box(xorf::Xor32::from_iterator(keys.iter().copied()))); +} diff --git a/benchmarks/xor/mod.rs b/benchmarks/xor/mod.rs new file mode 100644 index 00000000..cf9fbfc9 --- /dev/null +++ b/benchmarks/xor/mod.rs @@ -0,0 +1,36 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +mod build; +mod matched_build; +mod query; + +// SplitMix is a permutation, so these pseudorandom precomputed hashes are distinct. +fn hashes(count: usize) -> Vec { + hashes_with_offset(count, 0) +} + +fn hashes_with_offset(count: usize, offset: u64) -> Vec { + (offset..offset + count as u64) + .map(|i| { + let mut z = i.wrapping_add(0x9e3779b97f4a7c15); + z = (z ^ (z >> 30)).wrapping_mul(0xbf58476d1ce4e5b9); + z = (z ^ (z >> 27)).wrapping_mul(0x94d049bb133111eb); + z ^ (z >> 31) + }) + .collect() +} diff --git a/benchmarks/xor/query.rs b/benchmarks/xor/query.rs new file mode 100644 index 00000000..9e30e8a7 --- /dev/null +++ b/benchmarks/xor/query.rs @@ -0,0 +1,187 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use datasketches::xor::XorFilter; +use datasketches::xor::XorFilterType; +use divan::Bencher; +use divan::black_box; +use divan::counter::ItemsCount; +use xorf::Filter as _; + +use crate::xor::hashes; + +#[divan::bench(args = [16_384, 50_000])] +fn datasketches_8_present(bencher: Bencher, cardinality: usize) { + let keys = hashes(cardinality * 2); + let filter = + XorFilter::from_distinct_hashes(&keys[..cardinality], XorFilterType::Xor8).unwrap(); + bencher + .counter(ItemsCount::new(cardinality)) + .bench_local(|| { + for key in &keys[..cardinality] { + black_box(filter.contains_hash(black_box(*key))); + } + }); +} + +#[divan::bench(args = [16_384, 50_000])] +fn datasketches_8_absent(bencher: Bencher, cardinality: usize) { + let keys = hashes(cardinality * 2); + let filter = + XorFilter::from_distinct_hashes(&keys[..cardinality], XorFilterType::Xor8).unwrap(); + bencher + .counter(ItemsCount::new(cardinality)) + .bench_local(|| { + for key in &keys[cardinality..] { + black_box(filter.contains_hash(black_box(*key))); + } + }); +} + +#[divan::bench(args = [16_384, 50_000])] +fn xorf_8_present(bencher: Bencher, cardinality: usize) { + let keys = hashes(cardinality * 2); + let filter = xorf::Xor8::from(&keys[..cardinality]); + bencher + .counter(ItemsCount::new(cardinality)) + .bench_local(|| { + for key in &keys[..cardinality] { + black_box(filter.contains(black_box(key))); + } + }); +} + +#[divan::bench(args = [16_384, 50_000])] +fn xorf_8_absent(bencher: Bencher, cardinality: usize) { + let keys = hashes(cardinality * 2); + let filter = xorf::Xor8::from(&keys[..cardinality]); + bencher + .counter(ItemsCount::new(cardinality)) + .bench_local(|| { + for key in &keys[cardinality..] { + black_box(filter.contains(black_box(key))); + } + }); +} + +#[divan::bench(args = [16_384, 50_000])] +fn datasketches_16_present(bencher: Bencher, cardinality: usize) { + let keys = hashes(cardinality * 2); + let filter = + XorFilter::from_distinct_hashes(&keys[..cardinality], XorFilterType::Xor16).unwrap(); + bencher + .counter(ItemsCount::new(cardinality)) + .bench_local(|| { + for key in &keys[..cardinality] { + black_box(filter.contains_hash(black_box(*key))); + } + }); +} + +#[divan::bench(args = [16_384, 50_000])] +fn datasketches_16_absent(bencher: Bencher, cardinality: usize) { + let keys = hashes(cardinality * 2); + let filter = + XorFilter::from_distinct_hashes(&keys[..cardinality], XorFilterType::Xor16).unwrap(); + bencher + .counter(ItemsCount::new(cardinality)) + .bench_local(|| { + for key in &keys[cardinality..] { + black_box(filter.contains_hash(black_box(*key))); + } + }); +} + +#[divan::bench(args = [16_384, 50_000])] +fn xorf_16_present(bencher: Bencher, cardinality: usize) { + let keys = hashes(cardinality * 2); + let filter = xorf::Xor16::from(&keys[..cardinality]); + bencher + .counter(ItemsCount::new(cardinality)) + .bench_local(|| { + for key in &keys[..cardinality] { + black_box(filter.contains(black_box(key))); + } + }); +} + +#[divan::bench(args = [16_384, 50_000])] +fn xorf_16_absent(bencher: Bencher, cardinality: usize) { + let keys = hashes(cardinality * 2); + let filter = xorf::Xor16::from(&keys[..cardinality]); + bencher + .counter(ItemsCount::new(cardinality)) + .bench_local(|| { + for key in &keys[cardinality..] { + black_box(filter.contains(black_box(key))); + } + }); +} + +#[divan::bench(args = [16_384, 50_000])] +fn datasketches_32_present(bencher: Bencher, cardinality: usize) { + let keys = hashes(cardinality * 2); + let filter = + XorFilter::from_distinct_hashes(&keys[..cardinality], XorFilterType::Xor32).unwrap(); + bencher + .counter(ItemsCount::new(cardinality)) + .bench_local(|| { + for key in &keys[..cardinality] { + black_box(filter.contains_hash(black_box(*key))); + } + }); +} + +#[divan::bench(args = [16_384, 50_000])] +fn datasketches_32_absent(bencher: Bencher, cardinality: usize) { + let keys = hashes(cardinality * 2); + let filter = + XorFilter::from_distinct_hashes(&keys[..cardinality], XorFilterType::Xor32).unwrap(); + bencher + .counter(ItemsCount::new(cardinality)) + .bench_local(|| { + for key in &keys[cardinality..] { + black_box(filter.contains_hash(black_box(*key))); + } + }); +} + +#[divan::bench(args = [16_384, 50_000])] +fn xorf_32_present(bencher: Bencher, cardinality: usize) { + let keys = hashes(cardinality * 2); + let filter = xorf::Xor32::from(&keys[..cardinality]); + bencher + .counter(ItemsCount::new(cardinality)) + .bench_local(|| { + for key in &keys[..cardinality] { + black_box(filter.contains(black_box(key))); + } + }); +} + +#[divan::bench(args = [16_384, 50_000])] +fn xorf_32_absent(bencher: Bencher, cardinality: usize) { + let keys = hashes(cardinality * 2); + let filter = xorf::Xor32::from(&keys[..cardinality]); + bencher + .counter(ItemsCount::new(cardinality)) + .bench_local(|| { + for key in &keys[cardinality..] { + black_box(filter.contains(black_box(key))); + } + }); +} diff --git a/datasketches/Cargo.toml b/datasketches/Cargo.toml index 2c931fc7..adbbb84b 100644 --- a/datasketches/Cargo.toml +++ b/datasketches/Cargo.toml @@ -48,6 +48,7 @@ req = [] tdigest = [] theta = [] tuple = [] +xor = [] [[example]] name = "bloom_difference" diff --git a/datasketches/src/codec/family.rs b/datasketches/src/codec/family.rs index 734d5291..06a8f3d1 100644 --- a/datasketches/src/codec/family.rs +++ b/datasketches/src/codec/family.rs @@ -124,6 +124,15 @@ impl Family { min_pre_longs: 3, max_pre_longs: 4, }; + + /// Immutable XOR Filter. + #[cfg(feature = "xor")] + pub const XORFILTER: Family = Family { + id: 22, + name: "XORFILTER", + min_pre_longs: 3, + max_pre_longs: 3, + }; } impl Family { diff --git a/datasketches/src/codec/mod.rs b/datasketches/src/codec/mod.rs index d6b6a06c..7c50da39 100644 --- a/datasketches/src/codec/mod.rs +++ b/datasketches/src/codec/mod.rs @@ -33,6 +33,7 @@ pub use self::encode::SketchBytes; feature = "tdigest", feature = "theta", feature = "tuple", + feature = "xor", ))] #[allow(dead_code)] // some utilities are only used for certain sketches pub(crate) mod assert; @@ -48,5 +49,6 @@ pub(crate) mod assert; feature = "tdigest", feature = "theta", feature = "tuple", + feature = "xor", ))] pub(crate) mod family; diff --git a/datasketches/src/error.rs b/datasketches/src/error.rs index 110b8c31..6d2bd4ec 100644 --- a/datasketches/src/error.rs +++ b/datasketches/src/error.rs @@ -27,6 +27,8 @@ pub enum ErrorKind { InvalidArgument, /// The serialized sketch data is malformed. InvalidData, + /// No valid construction was found within the allowed attempts. + ConstructionFailed, } impl ErrorKind { @@ -35,6 +37,7 @@ impl ErrorKind { match self { ErrorKind::InvalidArgument => "InvalidArgument", ErrorKind::InvalidData => "InvalidData", + ErrorKind::ConstructionFailed => "ConstructionFailed", } } } diff --git a/datasketches/src/hash/mod.rs b/datasketches/src/hash/mod.rs index 2084902b..f2dde4f4 100644 --- a/datasketches/src/hash/mod.rs +++ b/datasketches/src/hash/mod.rs @@ -38,9 +38,9 @@ mod murmurhash; ))] pub(crate) use self::murmurhash::*; -#[cfg(feature = "bloom")] +#[cfg(any(feature = "bloom", feature = "xor"))] mod xxhash; -#[cfg(feature = "bloom")] +#[cfg(any(feature = "bloom", feature = "xor"))] pub(crate) use self::xxhash::*; #[cfg(any( @@ -82,7 +82,7 @@ pub(crate) use self::seed::*; ))] pub(crate) const DEFAULT_UPDATE_SEED: u64 = 9001; -#[cfg(feature = "bloom")] +#[cfg(any(feature = "bloom", feature = "xor"))] #[inline(always)] fn read_u32_le(bytes: &[u8]) -> u32 { u32::from_le_bytes(bytes.try_into().expect("four-byte hash input")) @@ -96,6 +96,7 @@ fn read_u32_le(bytes: &[u8]) -> u32 { feature = "hll", feature = "theta", feature = "tuple", + feature = "xor", ))] #[inline(always)] fn read_u64_le(bytes: &[u8]) -> u64 { diff --git a/datasketches/src/lib.rs b/datasketches/src/lib.rs index 813a841b..18cd2bc4 100644 --- a/datasketches/src/lib.rs +++ b/datasketches/src/lib.rs @@ -33,7 +33,7 @@ //! //! ## Choosing a sketch //! -//! * Use `bloom` for probabilistic membership queries. +//! * Use `bloom` for incremental probabilistic membership queries or `xor` for immutable sets. //! * Use `countmin` for point-frequency estimates and `frequencies` for discovering heavy hitters. //! * Use `hll` for fast distinct counts, `cpc` for compact serialized distinct counts, or `theta` //! when set operations are required. @@ -92,6 +92,9 @@ pub use self::thetafamily::theta; #[cfg(feature = "tuple")] pub use self::thetafamily::tuple; +#[cfg(feature = "xor")] +pub mod xor; + // common modules pub mod codec; pub mod common; diff --git a/datasketches/src/xor/filter.rs b/datasketches/src/xor/filter.rs new file mode 100644 index 00000000..5fe95540 --- /dev/null +++ b/datasketches/src/xor/filter.rs @@ -0,0 +1,765 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::hash::Hash; +use std::hash::Hasher; +use std::ops::BitXor; + +use crate::error::Error; +use crate::error::ErrorKind; +use crate::hash::XxHash64; + +const NUM_HASHES: u8 = 3; +const HASH_SEED: u64 = 0; +const DEFAULT_CONSTRUCTION_SEED: u64 = 0; +const LOAD_FACTOR: f64 = 1.23; +const CAPACITY_OFFSET: u64 = 32; +const MAX_CONSTRUCTION_ATTEMPTS: usize = 100; + +const MURMUR_C1: u64 = 0xff51_afd7_ed55_8ccd; +const MURMUR_C2: u64 = 0xc4ce_b9fe_1a85_ec53; + +const SPLITMIX_GAMMA: u64 = 0x9e37_79b9_7f4a_7c15; +const SPLITMIX_MUL1: u64 = 0xbf58_476d_1ce4_e5b9; +const SPLITMIX_MUL2: u64 = 0x94d0_49bb_1331_11eb; + +/// Fingerprint representation used by an [`XorFilter`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum XorFilterType { + /// Uses 8-bit fingerprints and has an expected false-positive probability of about `1 / 256`. + Xor8, + /// Uses 16-bit fingerprints and has an expected false-positive probability of about + /// `1 / 65_536`. + Xor16, + /// Uses 32-bit fingerprints and has an expected false-positive probability of about `2^-32`. + /// + /// This variant supports application-defined storage through [`XorFilter::from_parts`] and + /// [`XorFilter::into_parts`]. Standard serialization only supports 8- and 16-bit fingerprints. + Xor32, +} + +impl XorFilterType { + /// Returns the number of bits in each fingerprint. + pub const fn bits_per_fingerprint(self) -> u8 { + match self { + Self::Xor8 => 8, + Self::Xor16 => 16, + Self::Xor32 => 32, + } + } + + pub(crate) const fn bytes_per_fingerprint(self) -> usize { + (self.bits_per_fingerprint() / 8) as usize + } +} + +/// Owned fingerprint slots for importing and exporting an application-defined format. +/// +/// Slots are ordered as three consecutive, equally sized segments. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum XorFingerprints { + /// 8-bit fingerprint slots. + Xor8(Box<[u8]>), + /// 16-bit fingerprint slots. + Xor16(Box<[u16]>), + /// 32-bit fingerprint slots; not part of the Java serialization format. + Xor32(Box<[u32]>), +} + +impl XorFingerprints { + fn len(&self) -> usize { + match self { + Self::Xor8(values) => values.len(), + Self::Xor16(values) => values.len(), + Self::Xor32(values) => values.len(), + } + } + + pub(crate) fn byte_len(&self) -> usize { + match self { + Self::Xor8(values) => size_of_val::<[u8]>(values), + Self::Xor16(values) => size_of_val::<[u16]>(values), + Self::Xor32(values) => size_of_val::<[u32]>(values), + } + } +} + +/// An immutable xor filter for probabilistic set membership. +/// +/// A query that returns `false` proves that the item was not in the input set. A query that returns +/// `true` may be a false positive, with probability determined by [`XorFilterType`]. A filter +/// known to be empty always returns `false`. Values cannot +/// be added after construction; rebuild the filter when the set changes. +/// +/// Ordinary values passed to [`contains`](Self::contains) are reduced to 64-bit hashes with +/// xxHash64 and seed `0`. Use [`contains_hash`](Self::contains_hash) only with the same hashes that +/// were supplied to [`XorFilter::from_hashes`] or [`XorFilterBuilder::update_hash`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct XorFilter { + pub(crate) segment_length: usize, + pub(crate) num_items: Option, + pub(crate) seed: u64, + pub(crate) fingerprints: XorFingerprints, +} + +impl XorFilter { + /// Builds a filter from precomputed 64-bit hashes. + /// + /// Duplicate hashes are removed before construction. The same hash values must be passed to + /// [`contains_hash`](Self::contains_hash) when querying the result. + /// + /// # Errors + /// + /// Returns [`ErrorKind::InvalidArgument`] if the fingerprint slot count exceeds `u32::MAX` + /// or a scratch allocation would exceed `isize::MAX` bytes. Returns + /// [`ErrorKind::ConstructionFailed`] if no peelable construction is found within 100 attempts. + /// A successful build may still exceed the limits of [`XorFilter::serialize`]. + /// + /// # Examples + /// + /// ``` + /// use datasketches::xor::XorFilter; + /// use datasketches::xor::XorFilterType; + /// + /// let filter = XorFilter::from_hashes([10, 20, 30], XorFilterType::Xor8).unwrap(); + /// assert!(filter.contains_hash(20)); + /// ``` + pub fn from_hashes( + hashes: impl IntoIterator, + filter_type: XorFilterType, + ) -> Result { + let mut builder = XorFilterBuilder::new(filter_type); + builder.extend_hashes(hashes); + builder.build() + } + + /// Builds a filter from distinct precomputed hashes without copying or sorting them. + /// + /// This is useful when the caller already owns a deduplicated set of hashes. The default + /// construction seed is `0`; input order does not affect the result. + /// + /// # Errors + /// + /// Returns [`ErrorKind::InvalidArgument`] for the size limits described by + /// [`from_hashes`](Self::from_hashes), or [`ErrorKind::ConstructionFailed`] if no peelable + /// construction is found within 100 attempts. Duplicate hashes prevent peeling but are not + /// checked separately; they produce the same error as an exhausted search over valid input. + /// Changing the seed does not guarantee success. Use [`from_hashes`](Self::from_hashes) + /// to remove duplicates. + pub fn from_distinct_hashes(hashes: &[u64], filter_type: XorFilterType) -> Result { + Self::from_distinct_hashes_with_seed(hashes, filter_type, DEFAULT_CONSTRUCTION_SEED) + } + + /// Builds a filter from distinct hashes with an explicit base construction seed. + /// + /// This has the same input contract as [`from_distinct_hashes`](Self::from_distinct_hashes). + /// A different seed can be used to retry a failed construction without copying or sorting. + /// + /// # Errors + /// + /// Returns an error under the same conditions as + /// [`from_distinct_hashes`](Self::from_distinct_hashes). + pub fn from_distinct_hashes_with_seed( + hashes: &[u64], + filter_type: XorFilterType, + seed: u64, + ) -> Result { + Self::build(hashes, filter_type, seed, MAX_CONSTRUCTION_ATTEMPTS) + } + + /// Imports raw parts without requiring a DataSketches serialization header. + /// + /// The caller must preserve the original construction seed and fingerprint order, and query + /// with the original input hashes. A supplied item count must describe the original set. + /// A valid layout alone cannot establish these invariants. + /// Pass `None` when the storage format does not record the distinct item count. No count is + /// inferred from the payload, and standard serialization requires a known count. + /// + /// # Errors + /// + /// Returns an error unless the segment length is positive, the payload has exactly three + /// segments, and a supplied item count fits the capacity. Segment lengths above `u32::MAX` + /// are rejected because the index reduction uses 32-bit hash words. + pub fn from_parts( + seed: u64, + segment_length: usize, + fingerprints: XorFingerprints, + num_items: Option, + ) -> Result { + if segment_length == 0 + || segment_length > u32::MAX as usize + || segment_length.checked_mul(3) != Some(fingerprints.len()) + { + return Err(Error::invalid_argument( + "invalid xor filter segment or payload length", + )); + } + if num_items.is_some_and(|count| count > fingerprints.len()) { + return Err(Error::invalid_argument( + "item count exceeds xor filter capacity", + )); + } + Ok(Self { + segment_length, + num_items, + seed, + fingerprints, + }) + } + + /// Returns the number of fingerprint slots in each of the three segments. + pub fn segment_length(&self) -> usize { + self.segment_length + } + + /// Returns the fingerprint slots for application-defined serialization. + pub fn fingerprints(&self) -> &XorFingerprints { + &self.fingerprints + } + + /// Returns the seed, segment length, fingerprint slots, and optional item count without + /// copying. + pub fn into_parts(self) -> (u64, usize, XorFingerprints, Option) { + ( + self.seed, + self.segment_length, + self.fingerprints, + self.num_items, + ) + } + + /// Returns `true` if an item is possibly in the input set. + /// + /// A `false` result means the item was definitely absent; a `true` result may be a false + /// positive. + #[inline] + pub fn contains(&self, item: &T) -> bool { + self.contains_hash(hash_item(item)) + } + + /// Returns `true` if a precomputed 64-bit hash is possibly in the input set. + /// + /// This method bypasses xxHash64. The caller must use the same hash value during construction + /// and queries; mixing raw values with precomputed hashes can introduce false negatives. + #[inline] + pub fn contains_hash(&self, hash: u64) -> bool { + if self.num_items == Some(0) { + return false; + } + let mixed = mix(hash, self.seed); + let [h0, h1, h2] = indexes(mixed, self.segment_length); + + match &self.fingerprints { + XorFingerprints::Xor8(fingerprints) => { + fingerprint(mixed) as u8 == fingerprints[h0] ^ fingerprints[h1] ^ fingerprints[h2] + } + XorFingerprints::Xor16(fingerprints) => { + fingerprint(mixed) as u16 == fingerprints[h0] ^ fingerprints[h1] ^ fingerprints[h2] + } + XorFingerprints::Xor32(fingerprints) => { + fingerprint(mixed) as u32 == fingerprints[h0] ^ fingerprints[h1] ^ fingerprints[h2] + } + } + } + + /// Returns whether the input set was empty, or `None` if imported without an item count. + /// + /// Fingerprint contents cannot reliably establish whether the original set was empty. + pub fn is_empty(&self) -> Option { + self.num_items.map(|count| count == 0) + } + + /// Returns the number of distinct input hashes, or `None` if imported without a count. + pub fn num_items(&self) -> Option { + self.num_items + } + + /// Returns the fingerprint representation. + pub fn filter_type(&self) -> XorFilterType { + match &self.fingerprints { + XorFingerprints::Xor8(_) => XorFilterType::Xor8, + XorFingerprints::Xor16(_) => XorFilterType::Xor16, + XorFingerprints::Xor32(_) => XorFilterType::Xor32, + } + } + + /// Returns the number of bits in each fingerprint. + pub fn bits_per_fingerprint(&self) -> u8 { + self.filter_type().bits_per_fingerprint() + } + + /// Returns the number of hash locations used by non-empty filter queries. + pub fn num_hashes(&self) -> u8 { + NUM_HASHES + } + + /// Returns the number of fingerprint slots in the filter. + pub fn capacity(&self) -> usize { + self.fingerprints.len() + } + + /// Returns the construction seed stored with the filter. + pub fn seed(&self) -> u64 { + self.seed + } + + /// Returns the number of fingerprint bits allocated per distinct input hash. + /// + /// Returns `None` for an unknown item count and `Some(0.0)` for an empty filter. For + /// sufficiently large inputs the value approaches `9.84` for [`XorFilterType::Xor8`], + /// `19.68` for [`XorFilterType::Xor16`], and `39.36` for [`XorFilterType::Xor32`]. + pub fn bits_per_item(&self) -> Option { + self.num_items.map(|count| { + if count == 0 { + 0.0 + } else { + self.capacity() as f64 * f64::from(self.bits_per_fingerprint()) / count as f64 + } + }) + } + + /// Returns the estimated in-memory size of the filter in bytes. + pub fn estimated_size(&self) -> usize { + size_of::() + self.fingerprints.byte_len() + } + + fn build( + keys: &[u64], + filter_type: XorFilterType, + base_seed: u64, + max_attempts: usize, + ) -> Result { + debug_assert!(max_attempts > 0); + let num_items = keys.len(); + let capacity = compute_capacity(num_items)?; + let segment_length = capacity / usize::from(NUM_HASHES); + let mut xor_mask = vec![0_u64; capacity]; + let mut count = vec![0_u32; capacity]; + let mut queue = vec![0_u32; capacity]; + let mut stack_hash = vec![0_u64; num_items]; + let mut stack_index = vec![0_u32; num_items]; + + let mut rng_state = base_seed; + let mut construction_seed = 0; + let mut stack_size = 0; + for attempt in 0..max_attempts { + if attempt != 0 { + xor_mask.fill(0); + count.fill(0); + } + rng_state = rng_state.wrapping_add(SPLITMIX_GAMMA); + construction_seed = splitmix64(rng_state); + stack_size = map( + keys, + construction_seed, + segment_length, + &mut xor_mask, + &mut count, + &mut queue, + &mut stack_hash, + &mut stack_index, + ); + if stack_size == num_items { + break; + } + } + + if stack_size != num_items { + return Err(Error::new( + ErrorKind::ConstructionFailed, + format!("xor filter construction failed after {max_attempts} attempts"), + ) + .with_context("num_items", num_items)); + } + + let fingerprints = match filter_type { + XorFilterType::Xor8 => { + let mut values = vec![0_u8; capacity].into_boxed_slice(); + assign_fingerprints( + &mut values, + segment_length, + &stack_hash[..stack_size], + &stack_index[..stack_size], + |hash| fingerprint(hash) as u8, + ); + XorFingerprints::Xor8(values) + } + XorFilterType::Xor16 => { + let mut values = vec![0_u16; capacity].into_boxed_slice(); + assign_fingerprints( + &mut values, + segment_length, + &stack_hash[..stack_size], + &stack_index[..stack_size], + |hash| fingerprint(hash) as u16, + ); + XorFingerprints::Xor16(values) + } + XorFilterType::Xor32 => { + let mut values = vec![0_u32; capacity].into_boxed_slice(); + assign_fingerprints( + &mut values, + segment_length, + &stack_hash[..stack_size], + &stack_index[..stack_size], + |hash| fingerprint(hash) as u32, + ); + XorFingerprints::Xor32(values) + } + }; + + Ok(Self { + segment_length, + num_items: Some(num_items), + seed: construction_seed, + fingerprints, + }) + } +} + +/// Builder for accumulating values and creating an immutable [`XorFilter`]. +/// +/// Values are reduced to 64-bit hashes as they are added, so the builder retains one `u64` per +/// update regardless of the original value size. Duplicate hashes are removed by +/// [`build`](Self::build). +#[derive(Debug, Clone)] +pub struct XorFilterBuilder { + filter_type: XorFilterType, + seed: u64, + hashes: Vec, +} + +impl XorFilterBuilder { + /// Creates a builder for the requested fingerprint representation. + /// + /// The default base construction seed is `0`. + pub fn new(filter_type: XorFilterType) -> Self { + Self { + filter_type, + seed: DEFAULT_CONSTRUCTION_SEED, + hashes: Vec::new(), + } + } + + /// Sets the base seed used to derive construction attempts. + /// + /// A fixed seed makes construction deterministic for a given set of hashes. The filter stores + /// the successful derived seed, which is returned by [`XorFilter::seed`]. + pub fn seed(mut self, seed: u64) -> Self { + self.seed = seed; + self + } + + /// Updates the builder with a value hashed by xxHash64 with seed `0`. + pub fn update(&mut self, item: T) { + self.hashes.push(hash_item(&item)); + } + + /// Updates the builder with a precomputed 64-bit hash. + /// + /// This method bypasses xxHash64. Query the resulting filter with + /// [`XorFilter::contains_hash`] using hashes from the same source. + pub fn update_hash(&mut self, hash: u64) { + self.hashes.push(hash); + } + + /// Extends the builder with values hashed by xxHash64 with seed `0`. + pub fn extend(&mut self, items: impl IntoIterator) { + self.hashes + .extend(items.into_iter().map(|item| hash_item(&item))); + } + + /// Extends the builder with precomputed 64-bit hashes. + pub fn extend_hashes(&mut self, hashes: impl IntoIterator) { + self.hashes.extend(hashes); + } + + /// Returns the number of retained hashes, including duplicates added since the last build. + /// + /// Each build removes duplicates in place, even when construction fails. + pub fn num_items(&self) -> usize { + self.hashes.len() + } + + /// Returns `true` if no updates have been accumulated. + pub fn is_empty(&self) -> bool { + self.hashes.is_empty() + } + + /// Builds an immutable filter after removing duplicate hashes. + /// + /// Retains the deduplicated input on success and failure. The builder can accept more values + /// or be moved through [`seed`](Self::seed) to retry with a different base seed; success is not + /// guaranteed. Drop the builder when its retained hashes are no longer needed. + /// + /// # Errors + /// + /// Returns [`ErrorKind::InvalidArgument`] if the fingerprint slot count exceeds `u32::MAX` + /// or a scratch allocation would exceed `isize::MAX` bytes. Returns + /// [`ErrorKind::ConstructionFailed`] if no peelable construction is found within 100 attempts. + /// A successful build may still exceed the limits of [`XorFilter::serialize`]. + pub fn build(&mut self) -> Result { + self.build_with_attempts(MAX_CONSTRUCTION_ATTEMPTS) + } + + // A private attempt bound makes exhaustion and recovery deterministic in unit tests. + fn build_with_attempts(&mut self, max_attempts: usize) -> Result { + self.hashes.sort_unstable(); + self.hashes.dedup(); + XorFilter::build(&self.hashes, self.filter_type, self.seed, max_attempts) + } +} + +fn hash_item(item: &T) -> u64 { + let mut hasher = XxHash64::with_seed(HASH_SEED); + item.hash(&mut hasher); + hasher.finish() +} + +fn compute_capacity(num_items: usize) -> Result { + if num_items > u32::MAX as usize { + return Err(Error::invalid_argument(format!( + "xor filter item count exceeds construction limit: {num_items}" + ))); + } + + let scaled = (LOAD_FACTOR * num_items as f64) as u64; + let capacity = CAPACITY_OFFSET + scaled; + let capacity = capacity / u64::from(NUM_HASHES) * u64::from(NUM_HASHES); + if capacity > u32::MAX as u64 || capacity > (isize::MAX as u64 / size_of::() as u64) { + return Err(Error::invalid_argument(format!( + "xor filter capacity exceeds construction limit: {capacity}" + ))); + } + Ok(capacity as usize) +} + +fn map( + keys: &[u64], + seed: u64, + segment_length: usize, + xor_mask: &mut [u64], + count: &mut [u32], + queue: &mut [u32], + stack_hash: &mut [u64], + stack_index: &mut [u32], +) -> usize { + // Separate queues keep each drain within one segment and make the two neighbor + // segments statically known. Scratch space remains one slot per fingerprint. + let mut masks = split_segments(xor_mask, segment_length); + let mut counts = split_segments(count, segment_length); + let mut queues = split_segments(queue, segment_length); + for &key in keys { + let hash = mix(key, seed); + let indexes = [ + reduce(hash as u32, segment_length), + reduce(hash.rotate_left(21) as u32, segment_length), + reduce(hash.rotate_left(42) as u32, segment_length), + ]; + for block in 0..3 { + let index = indexes[block]; + masks[block][index] ^= hash; + counts[block][index] += 1; + } + } + let mut queue_sizes = [0; 3]; + for block in 0..3 { + for (index, &value) in counts[block].iter().enumerate() { + // Avoid a poorly predicted singleton branch. Before this write, the number + // retained cannot exceed the current scan index, so the queue has room. + queues[block][queue_sizes[block]] = index as u32; + queue_sizes[block] += usize::from(value == 1); + } + } + let mut stack_size = 0; + while queue_sizes != [0; 3] { + peel_block::<0>( + &mut masks, + &mut counts, + &mut queues, + &mut queue_sizes, + segment_length, + stack_hash, + stack_index, + &mut stack_size, + ); + peel_block::<1>( + &mut masks, + &mut counts, + &mut queues, + &mut queue_sizes, + segment_length, + stack_hash, + stack_index, + &mut stack_size, + ); + peel_block::<2>( + &mut masks, + &mut counts, + &mut queues, + &mut queue_sizes, + segment_length, + stack_hash, + stack_index, + &mut stack_size, + ); + } + stack_size +} + +fn split_segments(values: &mut [T], segment_length: usize) -> [&mut [T]; 3] { + let (first, rest) = values.split_at_mut(segment_length); + let (second, third) = rest.split_at_mut(segment_length); + [first, second, third] +} + +#[inline] +fn peel_block( + masks: &mut [&mut [u64]; 3], + counts: &mut [&mut [u32]; 3], + queues: &mut [&mut [u32]; 3], + queue_sizes: &mut [usize; 3], + segment_length: usize, + stack_hash: &mut [u64], + stack_index: &mut [u32], + stack_size: &mut usize, +) { + while queue_sizes[BLOCK] > 0 { + queue_sizes[BLOCK] -= 1; + let index = queues[BLOCK][queue_sizes[BLOCK]] as usize; + if counts[BLOCK][index] == 0 { + continue; + } + let hash = masks[BLOCK][index]; + stack_hash[*stack_size] = hash; + stack_index[*stack_size] = (index + BLOCK * segment_length) as u32; + *stack_size += 1; + // The owned slot has no other live edge and is queued at most once. Only + // the other two slots need updating; stale entries there are skipped above. + for other in [(BLOCK + 1) % 3, (BLOCK + 2) % 3] { + let index = reduce(hash.rotate_left(other as u32 * 21) as u32, segment_length); + masks[other][index] ^= hash; + counts[other][index] -= 1; + if counts[other][index] == 1 { + queues[other][queue_sizes[other]] = index as u32; + queue_sizes[other] += 1; + } + } + } +} + +fn assign_fingerprints>( + fingerprints: &mut [T], + segment_length: usize, + stack_hash: &[u64], + stack_index: &[u32], + fingerprint_of: impl Fn(u64) -> T, +) { + for (&hash, &index) in stack_hash.iter().zip(stack_index).rev() { + let index = index as usize; + let [h0, h1, h2] = indexes(hash, segment_length); + fingerprints[index] = + fingerprint_of(hash) ^ fingerprints[h0] ^ fingerprints[h1] ^ fingerprints[h2]; + } +} + +#[inline] +fn indexes(hash: u64, segment_length: usize) -> [usize; 3] { + [ + reduce(hash as u32, segment_length), + reduce(hash.rotate_left(21) as u32, segment_length) + segment_length, + reduce(hash.rotate_left(42) as u32, segment_length) + 2 * segment_length, + ] +} + +#[inline] +fn reduce(hash: u32, range: usize) -> usize { + ((u64::from(hash) * range as u64) >> 32) as usize +} + +#[inline] +fn fingerprint(hash: u64) -> u64 { + hash ^ (hash >> 32) +} + +#[inline] +fn mix(key: u64, seed: u64) -> u64 { + let mut hash = key.wrapping_add(seed); + hash ^= hash >> 33; + hash = hash.wrapping_mul(MURMUR_C1); + hash ^= hash >> 33; + hash = hash.wrapping_mul(MURMUR_C2); + hash ^ (hash >> 33) +} + +fn splitmix64(state: u64) -> u64 { + let mut value = state; + value = (value ^ (value >> 30)).wrapping_mul(SPLITMIX_MUL1); + value = (value ^ (value >> 27)).wrapping_mul(SPLITMIX_MUL2); + value ^ (value >> 31) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn oversized_input_is_rejected_before_allocating() { + for count in [usize::MAX, u32::MAX as usize] { + assert_eq!( + compute_capacity(count).unwrap_err().kind(), + ErrorKind::InvalidArgument + ); + } + } + + #[test] + fn builder_retains_input_after_exhaustion_and_can_retry() { + // This synthetic set fails its first peeling attempt with base seed 1. + // Limiting the private engine to one attempt exercises the public build failure path + // without relying on a rare failure across all 100 production attempts. + let keys: Vec<_> = (917_504..917_504 + 16_384u64) + .map(|i| splitmix64(i.wrapping_add(SPLITMIX_GAMMA))) + .collect(); + for kind in [ + XorFilterType::Xor8, + XorFilterType::Xor16, + XorFilterType::Xor32, + ] { + let mut builder = XorFilterBuilder::new(kind).seed(1); + builder.extend_hashes(keys.iter().copied().chain(keys.iter().copied())); + let error = builder.build_with_attempts(1).unwrap_err(); + assert_eq!(error.kind(), ErrorKind::ConstructionFailed); + assert_eq!(builder.num_items(), keys.len()); + assert!(!builder.is_empty()); + + builder = builder.seed(1_u64.wrapping_add(SPLITMIX_GAMMA)); + let filter = builder.build().unwrap(); + assert_eq!(filter.num_items(), Some(keys.len())); + for &key in &keys { + assert!(filter.contains_hash(key)); + } + assert_eq!(builder.build().unwrap(), filter); + builder.update_hash(42); + let extended = builder.build().unwrap(); + assert!(extended.contains_hash(42)); + for &key in &keys { + assert!(extended.contains_hash(key)); + } + } + } +} diff --git a/datasketches/src/xor/mod.rs b/datasketches/src/xor/mod.rs new file mode 100644 index 00000000..aa6175b5 --- /dev/null +++ b/datasketches/src/xor/mod.rs @@ -0,0 +1,79 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Xor filters for immutable probabilistic set membership. +//! +//! An xor filter is built from the complete input set and cannot be updated afterward. Compared +//! with a Bloom filter at the same false-positive probability, it generally needs less space and +//! performs exactly three fingerprint reads per query. Construction needs temporary space linear +//! in the number of distinct hashes. +//! +//! The [`XorFilterBuilder`] hashes ordinary Rust values with xxHash64. Call +//! [`XorFilterBuilder::update_hash`] or [`XorFilter::from_hashes`] when hashes have already been +//! computed and must not be hashed again. +//! +//! # Cross-language hashing +//! +//! Standard serialization supports 8- and 16-bit fingerprints using family `22`, version `1`. +//! The 32-bit variant uses application-defined storage through raw parts only. +//! Construction may choose a different valid fingerprint array from another implementation; +//! byte-for-byte equality of independently constructed filters is not guaranteed. Rust's +//! [`Hash`](std::hash::Hash) implementations do not always encode values like other languages. Use +//! the strategies in [`crate::hash::value`] when the input hashes must match another DataSketches +//! implementation. A filter built through the precomputed-hash APIs remains portable only when +//! every reader uses the same external hash function. +//! +//! Java hashes integer scalars as 64-bit little-endian values and strings as UTF-8 bytes with +//! xxHash64 seed `0`. Use `u64`/`i64` and [`crate::hash::value::raw_bytes`] accordingly. Java +//! skips empty strings; callers must do so explicitly here. External hashes (for example XXH3) +//! must use the precomputed-hash APIs throughout; Java's typed query API will not match them. +//! +//! # Application-defined storage +//! +//! [`XorFilter::from_parts`] and [`XorFilter::into_parts`] preserve the construction seed, segment +//! length, and typed fingerprint slots without adding metadata to an existing disk format. +//! Imported parts may omit the item count. Raw parts use the same query algorithm as `xorf` +//! 0.13 for all three widths, but builders need not produce identical fingerprint arrays. +//! Filters with a known zero item count always reject queries; imported parts with an unknown +//! count retain the ordinary fingerprint comparison, including possible false positives. +//! +//! # Examples +//! +//! ``` +//! use datasketches::xor::XorFilterBuilder; +//! use datasketches::xor::XorFilterType; +//! +//! let mut builder = XorFilterBuilder::new(XorFilterType::Xor8); +//! builder.update("apple"); +//! builder.update("banana"); +//! let filter = builder.build().unwrap(); +//! +//! assert!(filter.contains(&"apple")); +//! ``` +//! +//! # References +//! +//! * Graf and Lemire, "Xor Filters: Faster and Smaller Than Bloom and Cuckoo Filters," ACM Journal +//! of Experimental Algorithmics 25 (2020). + +mod filter; +mod serialization; + +pub use self::filter::XorFilter; +pub use self::filter::XorFilterBuilder; +pub use self::filter::XorFilterType; +pub use self::filter::XorFingerprints; diff --git a/datasketches/src/xor/serialization.rs b/datasketches/src/xor/serialization.rs new file mode 100644 index 00000000..0d437d81 --- /dev/null +++ b/datasketches/src/xor/serialization.rs @@ -0,0 +1,232 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::codec::SketchBytes; +use crate::codec::SketchSlice; +use crate::codec::assert::ensure_preamble_longs_in_range; +use crate::codec::assert::ensure_serial_version_is; +use crate::codec::assert::insufficient_data; +use crate::codec::family::Family; +use crate::error::Error; +use crate::xor::filter::XorFilter; +use crate::xor::filter::XorFilterType; +use crate::xor::filter::XorFingerprints; + +const SERIAL_VERSION: u8 = 1; +const NUM_HASHES: u8 = 3; +const PREAMBLE_BYTES: usize = 3 * size_of::(); + +impl XorFilter { + /// Serializes the filter using DataSketches family `22`, version `1`. + /// + /// The format is compatible with Java for 8- and 16-bit fingerprints when input hashing + /// agrees. Reserved fields are written as zero; integers and fingerprints are little-endian. + /// + /// # Errors + /// + /// Returns [`InvalidArgument`](crate::error::ErrorKind::InvalidArgument) for 32-bit + /// fingerprints, an unknown item count, or an item count or segment length above `i32::MAX`. + /// The total number of slots must also fit `i32::MAX` so that Java can index all three + /// segments; this is an additional interoperability limit, not a header field. Construction + /// and raw parts allow larger capacities, so successful construction does not guarantee + /// standard serialization. Use raw parts for other storage formats. + pub fn serialize(&self) -> Result, Error> { + let size = self.serialized_size()?; + let num_items = self + .num_items + .expect("serialized_size checked the item count"); + let mut bytes = SketchBytes::with_capacity(size); + + bytes.write_u8(Family::XORFILTER.min_pre_longs); + bytes.write_u8(SERIAL_VERSION); + bytes.write_u8(Family::XORFILTER.id); + bytes.write_u8(0); // flags + bytes.write_u8(self.bits_per_fingerprint()); + bytes.write_u8(NUM_HASHES); + bytes.write_u16_le(0); // unused + bytes.write_u64_le(self.seed); + bytes.write_i32_le(self.segment_length as i32); + bytes.write_i32_le(num_items as i32); + + match &self.fingerprints { + XorFingerprints::Xor8(fingerprints) => bytes.write(fingerprints), + XorFingerprints::Xor16(fingerprints) => { + for &fingerprint in fingerprints.iter() { + bytes.write_u16_le(fingerprint); + } + } + XorFingerprints::Xor32(_) => { + unreachable!("serialized_size rejected 32-bit fingerprints") + } + } + + Ok(bytes.into_bytes()) + } + + /// Deserializes an owned filter from bytes. + /// + /// Accepts 8- and 16-bit images within the limits of [`serialize`](Self::serialize). Reserved + /// fields and trailing bytes are ignored. + /// + /// # Errors + /// + /// Returns an error if the image is truncated, belongs to another sketch family or version, or + /// contains metadata that could make a membership query index outside the fingerprint payload. + pub fn deserialize(bytes: &[u8]) -> Result { + let mut cursor = SketchSlice::new(bytes); + + let preamble_longs = cursor + .read_u8() + .map_err(insufficient_data("preamble_longs"))?; + let serial_version = cursor + .read_u8() + .map_err(insufficient_data("serial_version"))?; + let family_id = cursor.read_u8().map_err(insufficient_data("family_id"))?; + let _flags = cursor.read_u8().map_err(insufficient_data("flags"))?; + let bits_per_fingerprint = cursor + .read_u8() + .map_err(insufficient_data("bits_per_fingerprint"))?; + let num_hashes = cursor.read_u8().map_err(insufficient_data("num_hashes"))?; + let _unused = cursor.read_u16_le().map_err(insufficient_data("unused"))?; + + Family::XORFILTER.validate_id(family_id)?; + ensure_serial_version_is(SERIAL_VERSION, serial_version)?; + ensure_preamble_longs_in_range( + Family::XORFILTER.min_pre_longs..=Family::XORFILTER.max_pre_longs, + preamble_longs, + )?; + let filter_type = match bits_per_fingerprint { + 8 => XorFilterType::Xor8, + 16 => XorFilterType::Xor16, + _ => { + return Err(Error::deserial( + "standard xor format requires 8- or 16-bit fingerprints", + )); + } + }; + if num_hashes != NUM_HASHES { + return Err(Error::deserial(format!( + "invalid number of hashes: expected {NUM_HASHES}, got {num_hashes}" + ))); + } + + let seed = cursor.read_u64_le().map_err(insufficient_data("seed"))?; + let segment_length = cursor + .read_i32_le() + .map_err(insufficient_data("segment_length"))?; + if segment_length <= 0 { + return Err(Error::deserial(format!( + "invalid segment length: expected a positive value, got {segment_length}" + ))); + } + let segment_length = segment_length as usize; + + let num_items = cursor + .read_i32_le() + .map_err(insufficient_data("num_items"))?; + if num_items < 0 { + return Err(Error::deserial(format!( + "invalid item count: expected a non-negative value, got {num_items}" + ))); + } + let num_items = num_items as usize; + + let capacity = segment_length + .checked_mul(usize::from(NUM_HASHES)) + .ok_or_else(|| Error::deserial("xor filter capacity overflow"))?; + if capacity > i32::MAX as usize { + return Err(Error::deserial(format!( + "invalid xor filter capacity: maximum is {}, got {capacity}", + i32::MAX + ))); + } + if num_items > capacity { + return Err(Error::deserial(format!( + "invalid item count: capacity is {capacity}, got {num_items}" + ))); + } + + let fingerprint_bytes = capacity + .checked_mul(filter_type.bytes_per_fingerprint()) + .ok_or_else(|| Error::deserial("xor filter fingerprint size overflow"))?; + if cursor.remaining().len() < fingerprint_bytes { + return Err(Error::insufficient_data_of( + "fingerprints", + format!( + "expected {fingerprint_bytes} bytes, got {}", + cursor.remaining().len() + ), + )); + } + + let fingerprints = match filter_type { + XorFilterType::Xor8 => { + let mut values = vec![0_u8; capacity].into_boxed_slice(); + cursor + .read_exact(&mut values) + .map_err(insufficient_data("fingerprints"))?; + XorFingerprints::Xor8(values) + } + XorFilterType::Xor16 => { + let mut values = Vec::with_capacity(capacity); + for _ in 0..capacity { + values.push( + cursor + .read_u16_le() + .map_err(insufficient_data("fingerprints"))?, + ); + } + XorFingerprints::Xor16(values.into_boxed_slice()) + } + XorFilterType::Xor32 => unreachable!("header validation rejected 32-bit fingerprints"), + }; + + Ok(Self { + segment_length, + num_items: Some(num_items), + seed, + fingerprints, + }) + } + + /// Returns the standard serialized size in bytes. + /// + /// # Errors + /// + /// Returns an error under the same conditions as [`serialize`](Self::serialize). + pub fn serialized_size(&self) -> Result { + if self.filter_type() == XorFilterType::Xor32 { + return Err(Error::invalid_argument( + "standard xor format does not support 32-bit fingerprints", + )); + } + let count = self + .num_items + .ok_or_else(|| Error::invalid_argument("standard xor format requires an item count"))?; + if count > i32::MAX as usize + || self.segment_length > i32::MAX as usize + || self.capacity() > i32::MAX as usize + { + return Err(Error::invalid_argument( + "xor filter exceeds standard format limits", + )); + } + PREAMBLE_BYTES + .checked_add(self.fingerprints.byte_len()) + .ok_or_else(|| Error::invalid_argument("xor filter serialized size overflow")) + } +} diff --git a/tests-integration/Cargo.toml b/tests-integration/Cargo.toml index 79021d6c..78d9eecd 100644 --- a/tests-integration/Cargo.toml +++ b/tests-integration/Cargo.toml @@ -23,6 +23,7 @@ edition.workspace = true rust-version.workspace = true [dev-dependencies] +xorf = { version = "=0.13.0", default-features = false } datasketches = { workspace = true, features = [ "bloom", "countmin", @@ -34,6 +35,7 @@ datasketches = { workspace = true, features = [ "tdigest", "theta", "tuple", + "xor", ] } googletest = { workspace = true } insta = { workspace = true } diff --git a/tests-integration/tests/serde_tests.rs b/tests-integration/tests/serde_tests.rs index bd723168..bfa66cff 100644 --- a/tests-integration/tests/serde_tests.rs +++ b/tests-integration/tests/serde_tests.rs @@ -71,3 +71,6 @@ mod theta; #[path = "serde_tests/tuple.rs"] mod tuple; + +#[path = "serde_tests/xor.rs"] +mod xor; diff --git a/tests-integration/tests/serde_tests/xor.rs b/tests-integration/tests/serde_tests/xor.rs new file mode 100644 index 00000000..f900a4c7 --- /dev/null +++ b/tests-integration/tests/serde_tests/xor.rs @@ -0,0 +1,283 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use datasketches::error::ErrorKind; +use datasketches::xor::XorFilter; +use datasketches::xor::XorFilterBuilder; +use datasketches::xor::XorFilterType; + +// Generated from datasketches-java commit 1034fb33b6e013334ff870f130b9c388cae5dfa9. +// XorFilterBuilder.create([1, 2, 3, Long.MIN_VALUE], bits, 42).toByteArray(). +const JAVA_XOR8_IMAGE: &str = concat!( + "0301160008030000956eeb2f2632d7bd0c00000004000000", + "00000000000000000000c500000000000000000000000000dd000000040000000b000000" +); +const JAVA_XOR16_IMAGE: &str = concat!( + "0301160010030000", + "956eeb2f2632d7bd", + "0c00000004000000", + "0000000000000000", + "0000000000000000", + "00000000c5b60000", + "0000000000000000", + "0000000000000000", + "0000000000000000", + "dd6e000000000000", + "04e8000000000000", + "0b23000000000000", +); + +#[test] +fn java_images_round_trip_byte_for_byte() { + let values = [1_u64, 2, 3, 1_u64 << 63]; + + for (filter_type, image) in [ + (XorFilterType::Xor8, JAVA_XOR8_IMAGE), + (XorFilterType::Xor16, JAVA_XOR16_IMAGE), + ] { + let expected = decode_hex(image); + let mut builder = XorFilterBuilder::new(filter_type).seed(42); + builder.extend(values); + let built = builder.build().unwrap(); + + let restored = XorFilter::deserialize(&expected).unwrap(); + assert_eq!(restored.serialize().unwrap(), expected); + assert_eq!(restored.filter_type(), filter_type); + assert_eq!(restored.num_items(), Some(values.len())); + for value in values { + assert!(built.contains(&value)); + assert!( + restored.contains(&value), + "{filter_type:?} image did not contain {value}" + ); + } + } +} + +#[test] +fn serialization_round_trips_standard_fingerprint_types() { + for filter_type in [XorFilterType::Xor8, XorFilterType::Xor16] { + let original = XorFilter::from_hashes(0..10_000_u64, filter_type).unwrap(); + let bytes = original.serialize().unwrap(); + assert_eq!(bytes.len(), original.serialized_size().unwrap()); + + let restored = XorFilter::deserialize(&bytes).unwrap(); + assert_eq!(restored, original); + for hash in 0..10_000_u64 { + assert!(restored.contains_hash(hash)); + } + } +} + +#[test] +fn every_truncated_image_is_rejected() { + for filter_type in [XorFilterType::Xor8, XorFilterType::Xor16] { + let bytes = XorFilter::from_hashes(0..100_u64, filter_type) + .unwrap() + .serialize() + .unwrap(); + for end in 0..bytes.len() { + let error = XorFilter::deserialize(&bytes[..end]).unwrap_err(); + assert_eq!(error.kind(), ErrorKind::InvalidData, "length {end}"); + } + assert!(XorFilter::deserialize(&bytes).is_ok()); + } +} + +#[test] +fn invalid_preamble_fields_are_rejected() { + let valid = XorFilter::from_hashes(0..100_u64, XorFilterType::Xor8) + .unwrap() + .serialize() + .unwrap(); + + for (offset, value) in [ + (0, 2), + (0, 4), + (1, 2), + (2, 21), + (4, 7), + (4, 32), + (4, 64), + (5, 4), + ] { + let mut corrupted = valid.clone(); + corrupted[offset] = value; + let error = XorFilter::deserialize(&corrupted).unwrap_err(); + assert_eq!(error.kind(), ErrorKind::InvalidData, "offset {offset}"); + } +} + +#[test] +fn unsafe_lengths_are_rejected_before_indexing() { + let valid = XorFilter::from_hashes(0..100_u64, XorFilterType::Xor8) + .unwrap() + .serialize() + .unwrap(); + + for segment_length in [0_i32, -1, i32::MAX] { + let mut corrupted = valid.clone(); + corrupted[16..20].copy_from_slice(&segment_length.to_le_bytes()); + assert_eq!( + XorFilter::deserialize(&corrupted).unwrap_err().kind(), + ErrorKind::InvalidData + ); + } + + for num_items in [-1_i32, i32::MAX] { + let mut corrupted = valid.clone(); + corrupted[20..24].copy_from_slice(&num_items.to_le_bytes()); + assert_eq!( + XorFilter::deserialize(&corrupted).unwrap_err().kind(), + ErrorKind::InvalidData + ); + } +} + +#[test] +fn trailing_storage_is_ignored() { + let original = XorFilter::from_hashes(0..100_u64, XorFilterType::Xor16).unwrap(); + let mut bytes = original.serialize().unwrap(); + bytes.extend_from_slice(&[0xaa; 16]); + + assert_eq!(XorFilter::deserialize(&bytes).unwrap(), original); +} + +fn decode_hex(input: &str) -> Vec { + assert_eq!(input.len() % 2, 0); + input + .as_bytes() + .chunks_exact(2) + .map(|digits| { + let digits = std::str::from_utf8(digits).unwrap(); + u8::from_str_radix(digits, 16).unwrap() + }) + .collect() +} + +#[test] +fn java_utf8_and_raw_byte_inputs_match() { + use datasketches::hash::value::raw_bytes; + // The same merged Java implementation, using update(String) and update(byte[]). + let images = [ + ( + XorFilterType::Xor8, + "0301160008030000956eeb2f2632d7bd0c0000000400000000000000000000000000000000df00000000000000000000cc0000003000000091000000", + ), + ( + XorFilterType::Xor16, + "0301160010030000956eeb2f2632d7bd0c000000040000000000000000000000000000000000000000000000000000000000df500000000000000000000000000000000000000000ccb5000000000000301200000000000091b8000000000000", + ), + ]; + for (kind, image) in images { + let mut builder = XorFilterBuilder::new(kind).seed(42); + builder.update(raw_bytes::from_str("hello")); + builder.update(raw_bytes::from_str("数据🔎")); + builder.update(raw_bytes::from_slice(&[])); + builder.update(raw_bytes::from_slice(&[0, 1, 255])); + let expected = decode_hex(image); + let built = builder.build().unwrap(); + let restored = XorFilter::deserialize(&expected).unwrap(); + assert_eq!(restored.serialize().unwrap(), expected); + for bytes in [b"hello".as_slice(), "数据🔎".as_bytes(), &[], &[0, 1, 255]] { + assert!(built.contains(&raw_bytes::from_slice(bytes))); + assert!(restored.contains(&raw_bytes::from_slice(bytes))); + } + } +} + +#[test] +fn nonstandard_filters_require_application_defined_storage() { + let filter = XorFilter::from_hashes(0..10, XorFilterType::Xor32).unwrap(); + assert_eq!( + filter.serialize().unwrap_err().kind(), + ErrorKind::InvalidArgument + ); + assert_eq!( + filter.serialized_size().unwrap_err().kind(), + ErrorKind::InvalidArgument + ); +} + +#[test] +fn raw_parts_preserve_an_application_storage_layout() { + use datasketches::codec::SketchSlice; + use datasketches::xor::XorFingerprints; + // Synthetic storage fixtures with public xorf 0.13.0 payloads for [0, 1, 2, u64::MAX]. + // This test-only envelope omits cardinality: magic, width:u8, segment:u32, seed:u64, + // and little-endian slots. It is not a production storage format. + let images = [ + concat!( + "54455354080c000000c15c0289ec2d0a91ad000000000000000000ca000000000000000000dc00300000000000000000", + "0000000000", + ), + concat!( + "54455354100c000000c15c0289ec2d0a91aded000000000000000000000000000000000000ca18000000000000000000", + "000000000000000000dcb9000030fc0000000000000000000000000000000000000000000000000000", + ), + concat!( + "54455354200c000000c15c0289ec2d0a91adedfc84000000000000000000000000000000000000000000000000000000", + "000000000000000000ca187b8e0000000000000000000000000000000000000000000000000000000000000000000000", + "00dcb969ee0000000030fcf3490000000000000000000000000000000000000000000000000000000000000000000000", + "0000000000000000000000000000000000", + ), + ]; + for image in images { + let bytes = decode_hex(image); + let mut cursor = SketchSlice::new(&bytes); + assert_eq!(&bytes[..4], b"TEST"); + let mut magic = [0; 4]; + cursor.read_exact(&mut magic).unwrap(); + let width = cursor.read_u8().unwrap(); + let segment = cursor.read_u32_le().unwrap() as usize; + let seed = cursor.read_u64_le().unwrap(); + let slots = cursor.remaining().len() / usize::from(width / 8); + let payload = match width { + 8 => XorFingerprints::Xor8(cursor.remaining().into()), + 16 => { + XorFingerprints::Xor16((0..slots).map(|_| cursor.read_u16_le().unwrap()).collect()) + } + 32 => { + XorFingerprints::Xor32((0..slots).map(|_| cursor.read_u32_le().unwrap()).collect()) + } + _ => unreachable!(), + }; + let filter = XorFilter::from_parts(seed, segment, payload, None).unwrap(); + for hash in [0, 1, 2, u64::MAX] { + assert!(filter.contains_hash(hash)); + } + assert_eq!(filter.capacity(), slots); + assert_eq!(filter.num_items(), None); + + let (seed, segment, payload, count) = filter.into_parts(); + assert_eq!(count, None); + let mut encoded = b"TEST".to_vec(); + encoded.push(width); + encoded.extend((segment as u32).to_le_bytes()); + encoded.extend(seed.to_le_bytes()); + match payload { + XorFingerprints::Xor8(values) => encoded.extend_from_slice(&values), + XorFingerprints::Xor16(values) => { + encoded.extend(values.iter().flat_map(|v| v.to_le_bytes())) + } + XorFingerprints::Xor32(values) => { + encoded.extend(values.iter().flat_map(|v| v.to_le_bytes())) + } + } + assert_eq!(encoded, bytes); + } +} diff --git a/tests-integration/tests/xor_test/filter.rs b/tests-integration/tests/xor_test/filter.rs new file mode 100644 index 00000000..1610ac74 --- /dev/null +++ b/tests-integration/tests/xor_test/filter.rs @@ -0,0 +1,154 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use datasketches::xor::XorFilter; +use datasketches::xor::XorFilterBuilder; +use datasketches::xor::XorFilterType; + +#[test] +fn values_have_no_false_negatives() { + let mut builder = XorFilterBuilder::new(XorFilterType::Xor8).seed(8123); + for value in 0..10_000_u64 { + builder.update(value); + } + let filter = builder.build().unwrap(); + + for value in 0..10_000_u64 { + assert!(filter.contains(&value), "false negative for {value}"); + } +} + +#[test] +fn precomputed_hashes_have_no_false_negatives() { + let hashes = (0..10_000_u64) + .map(|value| value.wrapping_mul(0x9e37_79b9_7f4a_7c15)) + .collect::>(); + + for filter_type in [ + XorFilterType::Xor8, + XorFilterType::Xor16, + XorFilterType::Xor32, + ] { + let filter = XorFilter::from_hashes(hashes.iter().copied(), filter_type).unwrap(); + for &hash in &hashes { + assert!(filter.contains_hash(hash), "false negative for {hash}"); + } + } +} + +#[test] +fn builder_accepts_mixed_value_types() { + let mut builder = XorFilterBuilder::new(XorFilterType::Xor16); + builder.update("datasketches"); + builder.update(42_u64); + builder.update([1_u8, 2, 3, 4]); + let filter = builder.build().unwrap(); + + assert!(filter.contains("datasketches")); + assert!(filter.contains(&42_u64)); + assert!(filter.contains(&[1_u8, 2, 3, 4])); +} + +#[test] +fn duplicates_are_removed_before_construction() { + let mut builder = XorFilterBuilder::new(XorFilterType::Xor8); + builder.extend_hashes([7, 7, 11, 11, 11]); + assert_eq!(builder.num_items(), 5); + + let filter = builder.build().unwrap(); + assert_eq!(filter.num_items(), Some(2)); + assert!(filter.contains_hash(7)); + assert!(filter.contains_hash(11)); +} + +#[test] +fn construction_is_independent_of_input_order() { + let ascending = XorFilterBuilder::new(XorFilterType::Xor8).seed(424_242); + let descending = ascending.clone(); + + let mut ascending = ascending; + ascending.extend_hashes(0..50_000_u64); + let mut descending = descending; + descending.extend_hashes((0..50_000_u64).rev()); + + assert_eq!( + ascending.build().unwrap().serialize().unwrap(), + descending.build().unwrap().serialize().unwrap() + ); +} + +#[test] +fn metadata_describes_the_compact_payload() { + let filter = XorFilter::from_hashes(0..10_000_u64, XorFilterType::Xor8).unwrap(); + + assert_eq!(filter.is_empty(), Some(false)); + assert_eq!(filter.num_items(), Some(10_000)); + assert_eq!(filter.filter_type(), XorFilterType::Xor8); + assert_eq!(filter.bits_per_fingerprint(), 8); + assert_eq!(filter.num_hashes(), 3); + assert_eq!(filter.capacity() % 3, 0); + assert!((9.5..10.5).contains(&filter.bits_per_item().unwrap())); + assert!(filter.estimated_size() >= filter.capacity()); + assert_eq!(filter.serialized_size().unwrap(), 24 + filter.capacity()); +} + +#[test] +fn empty_filter_is_well_formed() { + let mut builder = XorFilterBuilder::new(XorFilterType::Xor16); + assert!(builder.is_empty()); + let filter = builder.build().unwrap(); + + assert_eq!(filter.is_empty(), Some(true)); + assert_eq!(filter.num_items(), Some(0)); + assert_eq!(filter.capacity(), 30); + assert_eq!(filter.bits_per_item().unwrap(), 0.0); + assert_eq!(filter.serialized_size().unwrap(), 24 + 60); +} + +#[test] +fn false_positive_rates_follow_fingerprint_width() { + const NUM_ITEMS: u64 = 50_000; + const NUM_QUERIES: u64 = 100_000; + + let xor8 = XorFilter::from_hashes(0..NUM_ITEMS, XorFilterType::Xor8).unwrap(); + let xor16 = XorFilter::from_hashes(0..NUM_ITEMS, XorFilterType::Xor16).unwrap(); + let xor32 = XorFilter::from_hashes(0..NUM_ITEMS, XorFilterType::Xor32).unwrap(); + + let false_positives8 = (NUM_ITEMS..NUM_ITEMS + NUM_QUERIES) + .filter(|&hash| xor8.contains_hash(hash)) + .count(); + let false_positives16 = (NUM_ITEMS..NUM_ITEMS + NUM_QUERIES) + .filter(|&hash| xor16.contains_hash(hash)) + .count(); + let false_positives32 = (NUM_ITEMS..NUM_ITEMS + NUM_QUERIES) + .filter(|&hash| xor32.contains_hash(hash)) + .count(); + + assert!( + false_positives8 < 1_000, + "8-bit false-positive count was {false_positives8}" + ); + assert!( + false_positives16 < 100, + "16-bit false-positive count was {false_positives16}" + ); + assert!( + false_positives32 < 10, + "32-bit false-positive count was {false_positives32}" + ); + assert!(false_positives16 < false_positives8); +} diff --git a/tests-integration/tests/xor_test/main.rs b/tests-integration/tests/xor_test/main.rs new file mode 100644 index 00000000..656a5053 --- /dev/null +++ b/tests-integration/tests/xor_test/main.rs @@ -0,0 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +mod filter; + +mod parts; diff --git a/tests-integration/tests/xor_test/parts.rs b/tests-integration/tests/xor_test/parts.rs new file mode 100644 index 00000000..2869fdc7 --- /dev/null +++ b/tests-integration/tests/xor_test/parts.rs @@ -0,0 +1,223 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use datasketches::error::ErrorKind; +use datasketches::xor::XorFilter; +use datasketches::xor::XorFilterType; +use datasketches::xor::XorFingerprints; +use xorf::Filter; + +#[test] +fn raw_parts_interoperate_with_xorf_in_both_directions() { + let keys: Vec<_> = (0..16_384u64) + .map(|n| n.wrapping_mul(0x9e3779b97f4a7c15)) + .collect(); + for kind in [ + XorFilterType::Xor8, + XorFilterType::Xor16, + XorFilterType::Xor32, + ] { + let (seed, segment, payload) = match kind { + XorFilterType::Xor8 => { + let f = xorf::Xor8::from(&keys); + ( + f.seed, + f.block_length, + XorFingerprints::Xor8(f.fingerprints), + ) + } + XorFilterType::Xor16 => { + let f = xorf::Xor16::from(&keys); + ( + f.seed, + f.block_length, + XorFingerprints::Xor16(f.fingerprints), + ) + } + XorFilterType::Xor32 => { + let f = xorf::Xor32::from(&keys); + ( + f.seed, + f.block_length, + XorFingerprints::Xor32(f.fingerprints), + ) + } + }; + let imported = XorFilter::from_parts(seed, segment, payload, None).unwrap(); + assert_eq!(imported.num_items(), None); + assert_eq!(imported.bits_per_item(), None); + assert_eq!(imported.is_empty(), None); + assert_eq!( + imported.serialize().unwrap_err().kind(), + ErrorKind::InvalidArgument + ); + for &key in &keys { + assert!(imported.contains_hash(key)); + } + + let built = XorFilter::from_distinct_hashes(&keys, kind).unwrap(); + let (seed, block_length, payload, count) = built.clone().into_parts(); + assert_eq!(count, Some(keys.len())); + // Compare all answers, including false positives, after transferring ownership. + let reference: Box> = match payload { + XorFingerprints::Xor8(fingerprints) => Box::new(xorf::Xor8 { + seed, + block_length, + fingerprints, + }), + XorFingerprints::Xor16(fingerprints) => Box::new(xorf::Xor16 { + seed, + block_length, + fingerprints, + }), + XorFingerprints::Xor32(fingerprints) => Box::new(xorf::Xor32 { + seed, + block_length, + fingerprints, + }), + }; + for key in keys.iter().copied().chain(0..100_000) { + assert_eq!(built.contains_hash(key), reference.contains(&key)); + } + } +} + +#[test] +fn raw_parts_reject_invalid_layouts_and_counts() { + for (segment, length, count) in [ + (0, 0, None), + (1, 2, None), + (1, 4, None), + (usize::MAX, 3, None), + (1, 3, Some(4)), + ] { + assert_eq!( + XorFilter::from_parts( + 0, + segment, + XorFingerprints::Xor8(vec![0; length].into()), + count + ) + .unwrap_err() + .kind(), + ErrorKind::InvalidArgument + ); + } +} + +#[test] +fn distinct_construction_is_bounded_and_does_not_deduplicate() { + for kind in [ + XorFilterType::Xor8, + XorFilterType::Xor16, + XorFilterType::Xor32, + ] { + let error = XorFilter::from_distinct_hashes(&[1, 2, 1], kind).unwrap_err(); + assert_eq!(error.kind(), ErrorKind::ConstructionFailed); + assert!(error.message().contains("100 attempts")); + for keys in [vec![], vec![0], vec![u64::MAX], vec![0, u64::MAX]] { + let f = XorFilter::from_distinct_hashes(&keys, kind).unwrap(); + assert_eq!(f.num_items(), Some(keys.len())); + for key in keys { + assert!(f.contains_hash(key)); + } + } + } +} + +#[test] +fn empty_filters_never_match_even_when_the_fingerprint_is_zero() { + for kind in [ + XorFilterType::Xor8, + XorFilterType::Xor16, + XorFilterType::Xor32, + ] { + let f = XorFilter::from_hashes([], kind).unwrap(); + // This key mixes to zero under the filter's construction seed. + assert!(!f.contains_hash(f.seed().wrapping_neg())); + } +} + +#[test] +fn construction_recovers_after_an_unpeelable_first_attempt() { + // This distinct set needs two attempts with base seed 1. All fingerprint widths + // share the peeling graph, so each must reset the failed attempt's accumulators. + let keys: Vec<_> = (917_504..917_504 + 16_384u64) + .map(|i| { + let mut z = i.wrapping_add(0x9e3779b97f4a7c15); + z = (z ^ (z >> 30)).wrapping_mul(0xbf58476d1ce4e5b9); + z = (z ^ (z >> 27)).wrapping_mul(0x94d049bb133111eb); + z ^ (z >> 31) + }) + .collect(); + let first_seed = XorFilter::from_distinct_hashes_with_seed(&[], XorFilterType::Xor8, 1) + .unwrap() + .seed(); + let reference = xorf::Xor8::from(&keys); + assert_ne!(reference.seed, first_seed); + for kind in [ + XorFilterType::Xor8, + XorFilterType::Xor16, + XorFilterType::Xor32, + ] { + let filter = XorFilter::from_distinct_hashes_with_seed(&keys, kind, 1).unwrap(); + assert_eq!(filter.seed(), reference.seed); + for &key in &keys { + assert!(filter.contains_hash(key)); + } + let mut reversed = keys.clone(); + reversed.reverse(); + assert_eq!( + filter, + XorFilter::from_distinct_hashes_with_seed(&reversed, kind, 1).unwrap() + ); + } +} + +#[test] +fn unknown_count_does_not_infer_emptiness_from_zero_fingerprints() { + for kind in [ + XorFilterType::Xor8, + XorFilterType::Xor16, + XorFilterType::Xor32, + ] { + let empty = XorFilter::from_hashes([], kind).unwrap(); + let zero_hash = empty.seed().wrapping_neg(); + assert_eq!(empty.is_empty(), Some(true)); + assert!(!empty.contains_hash(zero_hash)); + let (seed, segment, payload, _) = empty.into_parts(); + let unknown = XorFilter::from_parts(seed, segment, payload, None).unwrap(); + assert_eq!(unknown.is_empty(), None); + assert_eq!(unknown.num_items(), None); + assert_eq!(unknown.bits_per_item(), None); + // An omitted count must preserve the raw fingerprint comparison, even for all-zero slots. + assert!(unknown.contains_hash(zero_hash)); + assert_eq!( + unknown.serialized_size().unwrap_err().kind(), + ErrorKind::InvalidArgument + ); + assert_eq!( + unknown.serialize().unwrap_err().kind(), + ErrorKind::InvalidArgument + ); + let (seed, segment, payload, count) = unknown.into_parts(); + assert_eq!(count, None); + let restored = XorFilter::from_parts(seed, segment, payload, count).unwrap(); + assert_eq!(restored.is_empty(), None); + assert!(restored.contains_hash(zero_hash)); + } +} From 6e6c14b19c43ea9528e8758267ed563008584ebf Mon Sep 17 00:00:00 2001 From: tison Date: Tue, 27 Oct 2026 23:25:34 +0800 Subject: [PATCH 2/6] refactor(xor): align builder with consuming configuration pattern --- CHANGELOG.md | 2 +- benchmarks/xor/matched_build.rs | 22 ++- datasketches/src/xor/filter.rs | 161 ++++++++++----------- datasketches/src/xor/mod.rs | 11 +- tests-integration/tests/serde_tests/xor.rs | 22 +-- tests-integration/tests/xor_test/filter.rs | 59 ++++---- tests-integration/tests/xor_test/parts.rs | 15 +- 7 files changed, 153 insertions(+), 139 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 742aad27..95505a37 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ All significant changes to this project will be documented in this file. ### New features -* Add immutable `XorFilter` and `XorFilterBuilder` behind the `xor` feature, with 8-, 16-, and 32-bit fingerprints, bounded seeded construction with recoverable builder failures, optional cardinality metadata for imported filters, Java-compatible standard serialization for 8/16-bit filters, and raw parts for application-defined storage. +* Add immutable `XorFilter` and `XorFilterBuilder` behind the `xor` feature, with 8-, 16-, and 32-bit fingerprints, consuming builder configuration with complete-input construction and bounded seeded retries, optional cardinality metadata for imported filters, Java-compatible standard serialization for 8/16-bit filters, and raw parts for application-defined storage. ## v0.6.0 diff --git a/benchmarks/xor/matched_build.rs b/benchmarks/xor/matched_build.rs index 8fbb1add..44273809 100644 --- a/benchmarks/xor/matched_build.rs +++ b/benchmarks/xor/matched_build.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use datasketches::xor::XorFilter; +use datasketches::xor::XorFilterBuilder; use datasketches::xor::XorFilterType; use divan::Bencher; use divan::black_box; @@ -27,7 +27,10 @@ use crate::xor::hashes_with_offset; // equal winning seeds therefore imply equal attempt counts. The offset 917504 // case requires two attempts, while the other listed cases require one. fn verify_seed(keys: &[u64], kind: XorFilterType) { - let filter = XorFilter::from_distinct_hashes_with_seed(keys, kind, 1).unwrap(); + let filter = XorFilterBuilder::new(kind) + .seed(1) + .build_from_distinct_hashes(keys) + .unwrap(); assert_eq!(filter.seed(), xorf::Xor8::from(keys).seed); } @@ -40,7 +43,10 @@ fn datasketches_8(bencher: Bencher, case: (usize, u64)) { .counter(ItemsCount::new(cardinality)) .bench_local(|| { black_box( - XorFilter::from_distinct_hashes_with_seed(&keys, XorFilterType::Xor8, 1).unwrap(), + XorFilterBuilder::new(XorFilterType::Xor8) + .seed(1) + .build_from_distinct_hashes(&keys) + .unwrap(), ) }); } @@ -64,7 +70,10 @@ fn datasketches_16(bencher: Bencher, case: (usize, u64)) { .counter(ItemsCount::new(cardinality)) .bench_local(|| { black_box( - XorFilter::from_distinct_hashes_with_seed(&keys, XorFilterType::Xor16, 1).unwrap(), + XorFilterBuilder::new(XorFilterType::Xor16) + .seed(1) + .build_from_distinct_hashes(&keys) + .unwrap(), ) }); } @@ -88,7 +97,10 @@ fn datasketches_32(bencher: Bencher, case: (usize, u64)) { .counter(ItemsCount::new(cardinality)) .bench_local(|| { black_box( - XorFilter::from_distinct_hashes_with_seed(&keys, XorFilterType::Xor32, 1).unwrap(), + XorFilterBuilder::new(XorFilterType::Xor32) + .seed(1) + .build_from_distinct_hashes(&keys) + .unwrap(), ) }); } diff --git a/datasketches/src/xor/filter.rs b/datasketches/src/xor/filter.rs index 5fe95540..f19c04c5 100644 --- a/datasketches/src/xor/filter.rs +++ b/datasketches/src/xor/filter.rs @@ -107,7 +107,7 @@ impl XorFingerprints { /// /// Ordinary values passed to [`contains`](Self::contains) are reduced to 64-bit hashes with /// xxHash64 and seed `0`. Use [`contains_hash`](Self::contains_hash) only with the same hashes that -/// were supplied to [`XorFilter::from_hashes`] or [`XorFilterBuilder::update_hash`]. +/// were supplied to [`XorFilter::from_hashes`] or [`XorFilterBuilder::build_from_hashes`]. #[derive(Debug, Clone, PartialEq, Eq)] pub struct XorFilter { pub(crate) segment_length: usize, @@ -142,9 +142,7 @@ impl XorFilter { hashes: impl IntoIterator, filter_type: XorFilterType, ) -> Result { - let mut builder = XorFilterBuilder::new(filter_type); - builder.extend_hashes(hashes); - builder.build() + XorFilterBuilder::new(filter_type).build_from_hashes(hashes) } /// Builds a filter from distinct precomputed hashes without copying or sorting them. @@ -161,24 +159,7 @@ impl XorFilter { /// Changing the seed does not guarantee success. Use [`from_hashes`](Self::from_hashes) /// to remove duplicates. pub fn from_distinct_hashes(hashes: &[u64], filter_type: XorFilterType) -> Result { - Self::from_distinct_hashes_with_seed(hashes, filter_type, DEFAULT_CONSTRUCTION_SEED) - } - - /// Builds a filter from distinct hashes with an explicit base construction seed. - /// - /// This has the same input contract as [`from_distinct_hashes`](Self::from_distinct_hashes). - /// A different seed can be used to retry a failed construction without copying or sorting. - /// - /// # Errors - /// - /// Returns an error under the same conditions as - /// [`from_distinct_hashes`](Self::from_distinct_hashes). - pub fn from_distinct_hashes_with_seed( - hashes: &[u64], - filter_type: XorFilterType, - seed: u64, - ) -> Result { - Self::build(hashes, filter_type, seed, MAX_CONSTRUCTION_ATTEMPTS) + XorFilterBuilder::new(filter_type).build_from_distinct_hashes(hashes) } /// Imports raw parts without requiring a DataSketches serialization header. @@ -431,16 +412,15 @@ impl XorFilter { } } -/// Builder for accumulating values and creating an immutable [`XorFilter`]. +/// Builder for configuring and creating an immutable [`XorFilter`]. /// -/// Values are reduced to 64-bit hashes as they are added, so the builder retains one `u64` per -/// update regardless of the original value size. Duplicate hashes are removed by -/// [`build`](Self::build). +/// Stores only the fingerprint type and construction seed. Each build consumes the configuration +/// and takes the complete input set. Ordinary values and precomputed hashes are deduplicated in +/// temporary storage; already-distinct hashes can be borrowed without copying or sorting. #[derive(Debug, Clone)] pub struct XorFilterBuilder { filter_type: XorFilterType, seed: u64, - hashes: Vec, } impl XorFilterBuilder { @@ -451,7 +431,6 @@ impl XorFilterBuilder { Self { filter_type, seed: DEFAULT_CONSTRUCTION_SEED, - hashes: Vec::new(), } } @@ -464,47 +443,38 @@ impl XorFilterBuilder { self } - /// Updates the builder with a value hashed by xxHash64 with seed `0`. - pub fn update(&mut self, item: T) { - self.hashes.push(hash_item(&item)); - } - - /// Updates the builder with a precomputed 64-bit hash. + /// Builds a filter from values hashed by xxHash64 with seed `0`. /// - /// This method bypasses xxHash64. Query the resulting filter with - /// [`XorFilter::contains_hash`] using hashes from the same source. - pub fn update_hash(&mut self, hash: u64) { - self.hashes.push(hash); - } - - /// Extends the builder with values hashed by xxHash64 with seed `0`. - pub fn extend(&mut self, items: impl IntoIterator) { - self.hashes - .extend(items.into_iter().map(|item| hash_item(&item))); - } - - /// Extends the builder with precomputed 64-bit hashes. - pub fn extend_hashes(&mut self, hashes: impl IntoIterator) { - self.hashes.extend(hashes); - } - - /// Returns the number of retained hashes, including duplicates added since the last build. + /// Duplicate hashes are removed before construction. The input is consumed as an iterator; + /// pass borrowed values to retain them for a later build, including after an error. /// - /// Each build removes duplicates in place, even when construction fails. - pub fn num_items(&self) -> usize { - self.hashes.len() - } - - /// Returns `true` if no updates have been accumulated. - pub fn is_empty(&self) -> bool { - self.hashes.is_empty() + /// # Errors + /// + /// Returns an error under the same conditions as + /// [`build_from_hashes`](Self::build_from_hashes). + /// + /// # Examples + /// + /// ``` + /// use datasketches::xor::XorFilterBuilder; + /// use datasketches::xor::XorFilterType; + /// + /// let filter = XorFilterBuilder::new(XorFilterType::Xor8) + /// .seed(42) + /// .build(["apple", "banana"])?; + /// assert!(filter.contains("apple")); + /// # Ok::<(), datasketches::error::Error>(()) + /// ``` + pub fn build(self, items: impl IntoIterator) -> Result { + self.build_from_hashes(items.into_iter().map(|item| hash_item(&item))) } - /// Builds an immutable filter after removing duplicate hashes. + /// Builds a filter from precomputed 64-bit hashes, removing duplicates. /// - /// Retains the deduplicated input on success and failure. The builder can accept more values - /// or be moved through [`seed`](Self::seed) to retry with a different base seed; success is not - /// guaranteed. Drop the builder when its retained hashes are no longer needed. + /// This method bypasses xxHash64. Query the resulting filter with + /// [`XorFilter::contains_hash`] using hashes from the same source. Temporary input storage is + /// released on success or failure. Pass `hashes.iter().copied()` to retain the original input + /// for a later build, including after an error. /// /// # Errors /// @@ -512,15 +482,36 @@ impl XorFilterBuilder { /// or a scratch allocation would exceed `isize::MAX` bytes. Returns /// [`ErrorKind::ConstructionFailed`] if no peelable construction is found within 100 attempts. /// A successful build may still exceed the limits of [`XorFilter::serialize`]. - pub fn build(&mut self) -> Result { - self.build_with_attempts(MAX_CONSTRUCTION_ATTEMPTS) + pub fn build_from_hashes( + self, + hashes: impl IntoIterator, + ) -> Result { + let mut hashes: Vec<_> = hashes.into_iter().collect(); + hashes.sort_unstable(); + hashes.dedup(); + self.build_from_distinct_hashes(&hashes) + } + + /// Builds a filter from distinct precomputed hashes without copying or sorting them. + /// + /// Borrows the input without modifying it, so the same slice can be retried with another + /// construction seed after an error. Changing the seed does not guarantee success. + /// Input order does not affect the result. + /// + /// # Errors + /// + /// Returns an error under the size limits described by + /// [`build_from_hashes`](Self::build_from_hashes), or [`ErrorKind::ConstructionFailed`] if + /// no peelable construction is found within 100 attempts. Duplicate hashes prevent peeling + /// but are not checked separately; use [`build_from_hashes`](Self::build_from_hashes) to + /// remove duplicates. + pub fn build_from_distinct_hashes(self, hashes: &[u64]) -> Result { + self.build_with_attempts(hashes, MAX_CONSTRUCTION_ATTEMPTS) } // A private attempt bound makes exhaustion and recovery deterministic in unit tests. - fn build_with_attempts(&mut self, max_attempts: usize) -> Result { - self.hashes.sort_unstable(); - self.hashes.dedup(); - XorFilter::build(&self.hashes, self.filter_type, self.seed, max_attempts) + fn build_with_attempts(self, hashes: &[u64], max_attempts: usize) -> Result { + XorFilter::build(hashes, self.filter_type, self.seed, max_attempts) } } @@ -728,7 +719,7 @@ mod tests { } #[test] - fn builder_retains_input_after_exhaustion_and_can_retry() { + fn borrowed_input_survives_exhaustion_and_can_retry() { // This synthetic set fails its first peeling attempt with base seed 1. // Limiting the private engine to one attempt exercises the public build failure path // without relying on a rare failure across all 100 production attempts. @@ -740,26 +731,26 @@ mod tests { XorFilterType::Xor16, XorFilterType::Xor32, ] { - let mut builder = XorFilterBuilder::new(kind).seed(1); - builder.extend_hashes(keys.iter().copied().chain(keys.iter().copied())); - let error = builder.build_with_attempts(1).unwrap_err(); + let original = keys.clone(); + let error = XorFilterBuilder::new(kind) + .seed(1) + .build_with_attempts(&keys, 1) + .unwrap_err(); assert_eq!(error.kind(), ErrorKind::ConstructionFailed); - assert_eq!(builder.num_items(), keys.len()); - assert!(!builder.is_empty()); + assert_eq!(keys, original); - builder = builder.seed(1_u64.wrapping_add(SPLITMIX_GAMMA)); - let filter = builder.build().unwrap(); + let builder = XorFilterBuilder::new(kind).seed(1_u64.wrapping_add(SPLITMIX_GAMMA)); + let filter = builder.clone().build_from_distinct_hashes(&keys).unwrap(); assert_eq!(filter.num_items(), Some(keys.len())); for &key in &keys { assert!(filter.contains_hash(key)); } - assert_eq!(builder.build().unwrap(), filter); - builder.update_hash(42); - let extended = builder.build().unwrap(); - assert!(extended.contains_hash(42)); - for &key in &keys { - assert!(extended.contains_hash(key)); - } + assert_eq!( + builder + .build_from_hashes(keys.iter().copied().chain(keys.iter().copied())) + .unwrap(), + filter, + ); } } } diff --git a/datasketches/src/xor/mod.rs b/datasketches/src/xor/mod.rs index aa6175b5..652b1ff9 100644 --- a/datasketches/src/xor/mod.rs +++ b/datasketches/src/xor/mod.rs @@ -23,8 +23,8 @@ //! in the number of distinct hashes. //! //! The [`XorFilterBuilder`] hashes ordinary Rust values with xxHash64. Call -//! [`XorFilterBuilder::update_hash`] or [`XorFilter::from_hashes`] when hashes have already been -//! computed and must not be hashed again. +//! [`XorFilterBuilder::build_from_hashes`] or [`XorFilter::from_hashes`] when hashes have already +//! been computed and must not be hashed again. //! //! # Cross-language hashing //! @@ -57,10 +57,9 @@ //! use datasketches::xor::XorFilterBuilder; //! use datasketches::xor::XorFilterType; //! -//! let mut builder = XorFilterBuilder::new(XorFilterType::Xor8); -//! builder.update("apple"); -//! builder.update("banana"); -//! let filter = builder.build().unwrap(); +//! let filter = XorFilterBuilder::new(XorFilterType::Xor8) +//! .build(["apple", "banana"]) +//! .unwrap(); //! //! assert!(filter.contains(&"apple")); //! ``` diff --git a/tests-integration/tests/serde_tests/xor.rs b/tests-integration/tests/serde_tests/xor.rs index f900a4c7..f020f132 100644 --- a/tests-integration/tests/serde_tests/xor.rs +++ b/tests-integration/tests/serde_tests/xor.rs @@ -50,9 +50,10 @@ fn java_images_round_trip_byte_for_byte() { (XorFilterType::Xor16, JAVA_XOR16_IMAGE), ] { let expected = decode_hex(image); - let mut builder = XorFilterBuilder::new(filter_type).seed(42); - builder.extend(values); - let built = builder.build().unwrap(); + let built = XorFilterBuilder::new(filter_type) + .seed(42) + .build(values) + .unwrap(); let restored = XorFilter::deserialize(&expected).unwrap(); assert_eq!(restored.serialize().unwrap(), expected); @@ -184,13 +185,16 @@ fn java_utf8_and_raw_byte_inputs_match() { ), ]; for (kind, image) in images { - let mut builder = XorFilterBuilder::new(kind).seed(42); - builder.update(raw_bytes::from_str("hello")); - builder.update(raw_bytes::from_str("数据🔎")); - builder.update(raw_bytes::from_slice(&[])); - builder.update(raw_bytes::from_slice(&[0, 1, 255])); + let built = XorFilterBuilder::new(kind) + .seed(42) + .build([ + raw_bytes::from_slice(b"hello"), + raw_bytes::from_slice("数据🔎".as_bytes()), + raw_bytes::from_slice(&[]), + raw_bytes::from_slice(&[0, 1, 255]), + ]) + .unwrap(); let expected = decode_hex(image); - let built = builder.build().unwrap(); let restored = XorFilter::deserialize(&expected).unwrap(); assert_eq!(restored.serialize().unwrap(), expected); for bytes in [b"hello".as_slice(), "数据🔎".as_bytes(), &[], &[0, 1, 255]] { diff --git a/tests-integration/tests/xor_test/filter.rs b/tests-integration/tests/xor_test/filter.rs index 1610ac74..216313fb 100644 --- a/tests-integration/tests/xor_test/filter.rs +++ b/tests-integration/tests/xor_test/filter.rs @@ -21,11 +21,10 @@ use datasketches::xor::XorFilterType; #[test] fn values_have_no_false_negatives() { - let mut builder = XorFilterBuilder::new(XorFilterType::Xor8).seed(8123); - for value in 0..10_000_u64 { - builder.update(value); - } - let filter = builder.build().unwrap(); + let filter = XorFilterBuilder::new(XorFilterType::Xor8) + .seed(8123) + .build(0..10_000_u64) + .unwrap(); for value in 0..10_000_u64 { assert!(filter.contains(&value), "false negative for {value}"); @@ -51,25 +50,22 @@ fn precomputed_hashes_have_no_false_negatives() { } #[test] -fn builder_accepts_mixed_value_types() { - let mut builder = XorFilterBuilder::new(XorFilterType::Xor16); - builder.update("datasketches"); - builder.update(42_u64); - builder.update([1_u8, 2, 3, 4]); - let filter = builder.build().unwrap(); - - assert!(filter.contains("datasketches")); - assert!(filter.contains(&42_u64)); - assert!(filter.contains(&[1_u8, 2, 3, 4])); +fn borrowed_values_can_be_reused() { + let values = vec!["datasketches".to_string(), "xor filter".to_string()]; + let builder = XorFilterBuilder::new(XorFilterType::Xor16).seed(42); + let first = builder.clone().build(&values).unwrap(); + let second = builder.build(&values).unwrap(); + assert_eq!(first, second); + for value in &values { + assert!(first.contains(value)); + } } #[test] fn duplicates_are_removed_before_construction() { - let mut builder = XorFilterBuilder::new(XorFilterType::Xor8); - builder.extend_hashes([7, 7, 11, 11, 11]); - assert_eq!(builder.num_items(), 5); - - let filter = builder.build().unwrap(); + let filter = XorFilterBuilder::new(XorFilterType::Xor8) + .build_from_hashes([7, 7, 11, 11, 11]) + .unwrap(); assert_eq!(filter.num_items(), Some(2)); assert!(filter.contains_hash(7)); assert!(filter.contains_hash(11)); @@ -80,14 +76,17 @@ fn construction_is_independent_of_input_order() { let ascending = XorFilterBuilder::new(XorFilterType::Xor8).seed(424_242); let descending = ascending.clone(); - let mut ascending = ascending; - ascending.extend_hashes(0..50_000_u64); - let mut descending = descending; - descending.extend_hashes((0..50_000_u64).rev()); - assert_eq!( - ascending.build().unwrap().serialize().unwrap(), - descending.build().unwrap().serialize().unwrap() + ascending + .build_from_hashes(0..50_000_u64) + .unwrap() + .serialize() + .unwrap(), + descending + .build_from_hashes((0..50_000_u64).rev()) + .unwrap() + .serialize() + .unwrap() ); } @@ -108,9 +107,9 @@ fn metadata_describes_the_compact_payload() { #[test] fn empty_filter_is_well_formed() { - let mut builder = XorFilterBuilder::new(XorFilterType::Xor16); - assert!(builder.is_empty()); - let filter = builder.build().unwrap(); + let filter = XorFilterBuilder::new(XorFilterType::Xor16) + .build_from_hashes([]) + .unwrap(); assert_eq!(filter.is_empty(), Some(true)); assert_eq!(filter.num_items(), Some(0)); diff --git a/tests-integration/tests/xor_test/parts.rs b/tests-integration/tests/xor_test/parts.rs index 2869fdc7..1ac97d89 100644 --- a/tests-integration/tests/xor_test/parts.rs +++ b/tests-integration/tests/xor_test/parts.rs @@ -17,6 +17,7 @@ use datasketches::error::ErrorKind; use datasketches::xor::XorFilter; +use datasketches::xor::XorFilterBuilder; use datasketches::xor::XorFilterType; use datasketches::xor::XorFingerprints; use xorf::Filter; @@ -164,7 +165,9 @@ fn construction_recovers_after_an_unpeelable_first_attempt() { z ^ (z >> 31) }) .collect(); - let first_seed = XorFilter::from_distinct_hashes_with_seed(&[], XorFilterType::Xor8, 1) + let first_seed = XorFilterBuilder::new(XorFilterType::Xor8) + .seed(1) + .build_from_distinct_hashes(&[]) .unwrap() .seed(); let reference = xorf::Xor8::from(&keys); @@ -174,7 +177,10 @@ fn construction_recovers_after_an_unpeelable_first_attempt() { XorFilterType::Xor16, XorFilterType::Xor32, ] { - let filter = XorFilter::from_distinct_hashes_with_seed(&keys, kind, 1).unwrap(); + let filter = XorFilterBuilder::new(kind) + .seed(1) + .build_from_distinct_hashes(&keys) + .unwrap(); assert_eq!(filter.seed(), reference.seed); for &key in &keys { assert!(filter.contains_hash(key)); @@ -183,7 +189,10 @@ fn construction_recovers_after_an_unpeelable_first_attempt() { reversed.reverse(); assert_eq!( filter, - XorFilter::from_distinct_hashes_with_seed(&reversed, kind, 1).unwrap() + XorFilterBuilder::new(kind) + .seed(1) + .build_from_distinct_hashes(&reversed) + .unwrap() ); } } From 7226d2671a70745ec6de71a00a7a5c4d5a29d4af Mon Sep 17 00:00:00 2001 From: tison Date: Wed, 28 Oct 2026 00:52:13 +0800 Subject: [PATCH 3/6] perf(xor): map construction hashes one segment at a time --- benchmarks/xor/matched_build.rs | 14 +++++++------- datasketches/src/xor/filter.rs | 26 +++++++++++++++++--------- 2 files changed, 24 insertions(+), 16 deletions(-) diff --git a/benchmarks/xor/matched_build.rs b/benchmarks/xor/matched_build.rs index 44273809..fe1ad4d7 100644 --- a/benchmarks/xor/matched_build.rs +++ b/benchmarks/xor/matched_build.rs @@ -25,7 +25,7 @@ use crate::xor::hashes_with_offset; // xorf 0.13 starts SplitMix at 1. Both builders use the same seed sequence; // equal winning seeds therefore imply equal attempt counts. The offset 917504 -// case requires two attempts, while the other listed cases require one. +// case exercises a second attempt; every case verifies equal winning seeds. fn verify_seed(keys: &[u64], kind: XorFilterType) { let filter = XorFilterBuilder::new(kind) .seed(1) @@ -34,7 +34,7 @@ fn verify_seed(keys: &[u64], kind: XorFilterType) { assert_eq!(filter.seed(), xorf::Xor8::from(keys).seed); } -#[divan::bench(args = [(1_000, 0), (16_384, 0), (16_384, 917_504), (49_999, 0), (50_000, 0), (50_001, 0), (100_000, 0)])] +#[divan::bench(args = [(1_000, 0), (16_384, 0), (16_384, 917_504), (49_999, 0), (50_000, 0), (50_001, 0), (100_000, 0), (8_000_000, 0)])] fn datasketches_8(bencher: Bencher, case: (usize, u64)) { let (cardinality, offset) = case; let keys = hashes_with_offset(cardinality, offset); @@ -51,7 +51,7 @@ fn datasketches_8(bencher: Bencher, case: (usize, u64)) { }); } -#[divan::bench(args = [(1_000, 0), (16_384, 0), (16_384, 917_504), (49_999, 0), (50_000, 0), (50_001, 0), (100_000, 0)])] +#[divan::bench(args = [(1_000, 0), (16_384, 0), (16_384, 917_504), (49_999, 0), (50_000, 0), (50_001, 0), (100_000, 0), (8_000_000, 0)])] fn xorf_8(bencher: Bencher, case: (usize, u64)) { let (cardinality, offset) = case; let keys = hashes_with_offset(cardinality, offset); @@ -61,7 +61,7 @@ fn xorf_8(bencher: Bencher, case: (usize, u64)) { .bench_local(|| black_box(xorf::Xor8::from_iterator(keys.iter().copied()))); } -#[divan::bench(args = [(1_000, 0), (16_384, 0), (16_384, 917_504), (49_999, 0), (50_000, 0), (50_001, 0), (100_000, 0)])] +#[divan::bench(args = [(1_000, 0), (16_384, 0), (16_384, 917_504), (49_999, 0), (50_000, 0), (50_001, 0), (100_000, 0), (8_000_000, 0)])] fn datasketches_16(bencher: Bencher, case: (usize, u64)) { let (cardinality, offset) = case; let keys = hashes_with_offset(cardinality, offset); @@ -78,7 +78,7 @@ fn datasketches_16(bencher: Bencher, case: (usize, u64)) { }); } -#[divan::bench(args = [(1_000, 0), (16_384, 0), (16_384, 917_504), (49_999, 0), (50_000, 0), (50_001, 0), (100_000, 0)])] +#[divan::bench(args = [(1_000, 0), (16_384, 0), (16_384, 917_504), (49_999, 0), (50_000, 0), (50_001, 0), (100_000, 0), (8_000_000, 0)])] fn xorf_16(bencher: Bencher, case: (usize, u64)) { let (cardinality, offset) = case; let keys = hashes_with_offset(cardinality, offset); @@ -88,7 +88,7 @@ fn xorf_16(bencher: Bencher, case: (usize, u64)) { .bench_local(|| black_box(xorf::Xor16::from_iterator(keys.iter().copied()))); } -#[divan::bench(args = [(1_000, 0), (16_384, 0), (16_384, 917_504), (49_999, 0), (50_000, 0), (50_001, 0), (100_000, 0)])] +#[divan::bench(args = [(1_000, 0), (16_384, 0), (16_384, 917_504), (49_999, 0), (50_000, 0), (50_001, 0), (100_000, 0), (8_000_000, 0)])] fn datasketches_32(bencher: Bencher, case: (usize, u64)) { let (cardinality, offset) = case; let keys = hashes_with_offset(cardinality, offset); @@ -105,7 +105,7 @@ fn datasketches_32(bencher: Bencher, case: (usize, u64)) { }); } -#[divan::bench(args = [(1_000, 0), (16_384, 0), (16_384, 917_504), (49_999, 0), (50_000, 0), (50_001, 0), (100_000, 0)])] +#[divan::bench(args = [(1_000, 0), (16_384, 0), (16_384, 917_504), (49_999, 0), (50_000, 0), (50_001, 0), (100_000, 0), (8_000_000, 0)])] fn xorf_32(bencher: Bencher, case: (usize, u64)) { let (cardinality, offset) = case; let keys = hashes_with_offset(cardinality, offset); diff --git a/datasketches/src/xor/filter.rs b/datasketches/src/xor/filter.rs index f19c04c5..38cd1989 100644 --- a/datasketches/src/xor/filter.rs +++ b/datasketches/src/xor/filter.rs @@ -554,15 +554,14 @@ fn map( let mut masks = split_segments(xor_mask, segment_length); let mut counts = split_segments(count, segment_length); let mut queues = split_segments(queue, segment_length); - for &key in keys { - let hash = mix(key, seed); - let indexes = [ - reduce(hash as u32, segment_length), - reduce(hash.rotate_left(21) as u32, segment_length), - reduce(hash.rotate_left(42) as u32, segment_length), - ]; - for block in 0..3 { - let index = indexes[block]; + // Reuse the peel stack for mixed hashes while mapping one segment at a time. + // This bounds the random-access working set without allocating another buffer. + for (&key, hash) in keys.iter().zip(stack_hash.iter_mut()) { + *hash = mix(key, seed); + } + for block in 0..3 { + for &hash in stack_hash.iter() { + let index = reduce(hash.rotate_left(block as u32 * 21) as u32, segment_length); masks[block][index] ^= hash; counts[block][index] += 1; } @@ -745,6 +744,15 @@ mod tests { for &key in &keys { assert!(filter.contains_hash(key)); } + // A failed peel overwrites the mixed hashes in the scratch stack. An automatic + // retry must remap the original input before attempting the next seed. + assert_eq!( + XorFilterBuilder::new(kind) + .seed(1) + .build_from_distinct_hashes(&keys) + .unwrap(), + filter, + ); assert_eq!( builder .build_from_hashes(keys.iter().copied().chain(keys.iter().copied())) From 5d29ee52fbd05568e65674b560174d0ee4af69d3 Mon Sep 17 00:00:00 2001 From: tison Date: Wed, 28 Oct 2026 10:43:32 +0800 Subject: [PATCH 4/6] refactor(xor): use invalid argument for construction failures --- datasketches/src/error.rs | 3 -- datasketches/src/xor/filter.rs | 47 ++++++++++++----------- tests-integration/tests/xor_test/parts.rs | 2 +- 3 files changed, 25 insertions(+), 27 deletions(-) diff --git a/datasketches/src/error.rs b/datasketches/src/error.rs index 6d2bd4ec..110b8c31 100644 --- a/datasketches/src/error.rs +++ b/datasketches/src/error.rs @@ -27,8 +27,6 @@ pub enum ErrorKind { InvalidArgument, /// The serialized sketch data is malformed. InvalidData, - /// No valid construction was found within the allowed attempts. - ConstructionFailed, } impl ErrorKind { @@ -37,7 +35,6 @@ impl ErrorKind { match self { ErrorKind::InvalidArgument => "InvalidArgument", ErrorKind::InvalidData => "InvalidData", - ErrorKind::ConstructionFailed => "ConstructionFailed", } } } diff --git a/datasketches/src/xor/filter.rs b/datasketches/src/xor/filter.rs index 38cd1989..8820e16a 100644 --- a/datasketches/src/xor/filter.rs +++ b/datasketches/src/xor/filter.rs @@ -20,7 +20,6 @@ use std::hash::Hasher; use std::ops::BitXor; use crate::error::Error; -use crate::error::ErrorKind; use crate::hash::XxHash64; const NUM_HASHES: u8 = 3; @@ -124,10 +123,11 @@ impl XorFilter { /// /// # Errors /// - /// Returns [`ErrorKind::InvalidArgument`] if the fingerprint slot count exceeds `u32::MAX` - /// or a scratch allocation would exceed `isize::MAX` bytes. Returns - /// [`ErrorKind::ConstructionFailed`] if no peelable construction is found within 100 attempts. - /// A successful build may still exceed the limits of [`XorFilter::serialize`]. + /// Returns [`InvalidArgument`](crate::error::ErrorKind::InvalidArgument) if the fingerprint + /// slot count exceeds `u32::MAX` or a scratch allocation would exceed `isize::MAX` bytes, + /// or if no peelable construction is found within 100 attempts. Valid input can also + /// exhaust the construction attempts. A successful build may still exceed the limits of + /// [`XorFilter::serialize`]. /// /// # Examples /// @@ -152,10 +152,10 @@ impl XorFilter { /// /// # Errors /// - /// Returns [`ErrorKind::InvalidArgument`] for the size limits described by - /// [`from_hashes`](Self::from_hashes), or [`ErrorKind::ConstructionFailed`] if no peelable - /// construction is found within 100 attempts. Duplicate hashes prevent peeling but are not - /// checked separately; they produce the same error as an exhausted search over valid input. + /// Returns [`InvalidArgument`](crate::error::ErrorKind::InvalidArgument) for the size limits + /// described by [`from_hashes`](Self::from_hashes), or if no peelable construction is found + /// within 100 attempts. Duplicate hashes prevent peeling but are not checked separately; + /// they produce the same error as an exhausted search over valid input. /// Changing the seed does not guarantee success. Use [`from_hashes`](Self::from_hashes) /// to remove duplicates. pub fn from_distinct_hashes(hashes: &[u64], filter_type: XorFilterType) -> Result { @@ -360,10 +360,9 @@ impl XorFilter { } if stack_size != num_items { - return Err(Error::new( - ErrorKind::ConstructionFailed, - format!("xor filter construction failed after {max_attempts} attempts"), - ) + return Err(Error::invalid_argument(format!( + "xor filter construction failed after {max_attempts} attempts" + )) .with_context("num_items", num_items)); } @@ -478,10 +477,11 @@ impl XorFilterBuilder { /// /// # Errors /// - /// Returns [`ErrorKind::InvalidArgument`] if the fingerprint slot count exceeds `u32::MAX` - /// or a scratch allocation would exceed `isize::MAX` bytes. Returns - /// [`ErrorKind::ConstructionFailed`] if no peelable construction is found within 100 attempts. - /// A successful build may still exceed the limits of [`XorFilter::serialize`]. + /// Returns [`InvalidArgument`](crate::error::ErrorKind::InvalidArgument) if the fingerprint + /// slot count exceeds `u32::MAX` or a scratch allocation would exceed `isize::MAX` bytes, + /// or if no peelable construction is found within 100 attempts. Valid input can also + /// exhaust the construction attempts. A successful build may still exceed the limits of + /// [`XorFilter::serialize`]. pub fn build_from_hashes( self, hashes: impl IntoIterator, @@ -500,11 +500,11 @@ impl XorFilterBuilder { /// /// # Errors /// - /// Returns an error under the size limits described by - /// [`build_from_hashes`](Self::build_from_hashes), or [`ErrorKind::ConstructionFailed`] if - /// no peelable construction is found within 100 attempts. Duplicate hashes prevent peeling - /// but are not checked separately; use [`build_from_hashes`](Self::build_from_hashes) to - /// remove duplicates. + /// Returns [`InvalidArgument`](crate::error::ErrorKind::InvalidArgument) under the size limits + /// described by [`build_from_hashes`](Self::build_from_hashes), or if no peelable + /// construction is found within 100 attempts, even for valid input. Duplicate hashes + /// prevent peeling but are not checked separately; use + /// [`build_from_hashes`](Self::build_from_hashes) to remove duplicates. pub fn build_from_distinct_hashes(self, hashes: &[u64]) -> Result { self.build_with_attempts(hashes, MAX_CONSTRUCTION_ATTEMPTS) } @@ -706,6 +706,7 @@ fn splitmix64(state: u64) -> u64 { #[cfg(test)] mod tests { use super::*; + use crate::error::ErrorKind; #[test] fn oversized_input_is_rejected_before_allocating() { @@ -735,7 +736,7 @@ mod tests { .seed(1) .build_with_attempts(&keys, 1) .unwrap_err(); - assert_eq!(error.kind(), ErrorKind::ConstructionFailed); + assert_eq!(error.kind(), ErrorKind::InvalidArgument); assert_eq!(keys, original); let builder = XorFilterBuilder::new(kind).seed(1_u64.wrapping_add(SPLITMIX_GAMMA)); diff --git a/tests-integration/tests/xor_test/parts.rs b/tests-integration/tests/xor_test/parts.rs index 1ac97d89..320e166c 100644 --- a/tests-integration/tests/xor_test/parts.rs +++ b/tests-integration/tests/xor_test/parts.rs @@ -128,7 +128,7 @@ fn distinct_construction_is_bounded_and_does_not_deduplicate() { XorFilterType::Xor32, ] { let error = XorFilter::from_distinct_hashes(&[1, 2, 1], kind).unwrap_err(); - assert_eq!(error.kind(), ErrorKind::ConstructionFailed); + assert_eq!(error.kind(), ErrorKind::InvalidArgument); assert!(error.message().contains("100 attempts")); for keys in [vec![], vec![0], vec![u64::MAX], vec![0, u64::MAX]] { let f = XorFilter::from_distinct_hashes(&keys, kind).unwrap(); From e239c3989bfb80c6c6fcfe22f94db084e4b49db4 Mon Sep 17 00:00:00 2001 From: tison Date: Wed, 28 Oct 2026 11:00:13 +0800 Subject: [PATCH 5/6] feat(xor): support 32-bit fingerprint serialization --- CHANGELOG.md | 2 +- datasketches/src/xor/filter.rs | 5 +- datasketches/src/xor/mod.rs | 6 +- datasketches/src/xor/serialization.rs | 41 +++++++----- tests-integration/tests/serde_tests/xor.rs | 78 +++++++++++++++------- 5 files changed, 85 insertions(+), 47 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7063d7ce..9e3a83a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ All significant changes to this project will be documented in this file. ### New features -* Add immutable `XorFilter` and `XorFilterBuilder` behind the `xor` feature, with 8-, 16-, and 32-bit fingerprints, consuming builder configuration with complete-input construction and bounded seeded retries, optional cardinality metadata for imported filters, Java-compatible standard serialization for 8/16-bit filters, and raw parts for application-defined storage. +* Add immutable `XorFilter` and `XorFilterBuilder` behind the `xor` feature, with 8-, 16-, and 32-bit fingerprints, consuming builder configuration with complete-input construction and bounded seeded retries, optional cardinality metadata for imported filters, standard serialization for all three widths (Java-compatible for 8/16-bit filters), and raw parts for application-defined storage. ### Improvements diff --git a/datasketches/src/xor/filter.rs b/datasketches/src/xor/filter.rs index 8820e16a..d26c86a1 100644 --- a/datasketches/src/xor/filter.rs +++ b/datasketches/src/xor/filter.rs @@ -45,9 +45,6 @@ pub enum XorFilterType { /// `1 / 65_536`. Xor16, /// Uses 32-bit fingerprints and has an expected false-positive probability of about `2^-32`. - /// - /// This variant supports application-defined storage through [`XorFilter::from_parts`] and - /// [`XorFilter::into_parts`]. Standard serialization only supports 8- and 16-bit fingerprints. Xor32, } @@ -75,7 +72,7 @@ pub enum XorFingerprints { Xor8(Box<[u8]>), /// 16-bit fingerprint slots. Xor16(Box<[u16]>), - /// 32-bit fingerprint slots; not part of the Java serialization format. + /// 32-bit fingerprint slots. Xor32(Box<[u32]>), } diff --git a/datasketches/src/xor/mod.rs b/datasketches/src/xor/mod.rs index 652b1ff9..3584adfd 100644 --- a/datasketches/src/xor/mod.rs +++ b/datasketches/src/xor/mod.rs @@ -28,8 +28,10 @@ //! //! # Cross-language hashing //! -//! Standard serialization supports 8- and 16-bit fingerprints using family `22`, version `1`. -//! The 32-bit variant uses application-defined storage through raw parts only. +//! Standard serialization supports 8-, 16-, and 32-bit fingerprints using family `22`, version `1`. +//! Interoperability requires the reader to support the encoded fingerprint width. The 8- and +//! 16-bit images match Java's format; 32-bit images extend the same layout with little-endian +//! `u32` slots and cannot be read by implementations limited to 8/16 bits. //! Construction may choose a different valid fingerprint array from another implementation; //! byte-for-byte equality of independently constructed filters is not guaranteed. Rust's //! [`Hash`](std::hash::Hash) implementations do not always encode values like other languages. Use diff --git a/datasketches/src/xor/serialization.rs b/datasketches/src/xor/serialization.rs index 0d437d81..6cb3e1c4 100644 --- a/datasketches/src/xor/serialization.rs +++ b/datasketches/src/xor/serialization.rs @@ -33,13 +33,16 @@ const PREAMBLE_BYTES: usize = 3 * size_of::(); impl XorFilter { /// Serializes the filter using DataSketches family `22`, version `1`. /// - /// The format is compatible with Java for 8- and 16-bit fingerprints when input hashing - /// agrees. Reserved fields are written as zero; integers and fingerprints are little-endian. + /// Supports 8-, 16-, and 32-bit fingerprints, recorded in the header's fingerprint width + /// field. Reserved fields are written as zero; integers and fingerprints are little-endian. + /// The 8- and 16-bit images are compatible with Java when input hashing agrees. Readers + /// must support the encoded fingerprint width; implementations limited to 8/16 bits cannot + /// read 32-bit images. /// /// # Errors /// - /// Returns [`InvalidArgument`](crate::error::ErrorKind::InvalidArgument) for 32-bit - /// fingerprints, an unknown item count, or an item count or segment length above `i32::MAX`. + /// Returns [`InvalidArgument`](crate::error::ErrorKind::InvalidArgument) for an unknown + /// item count, or an item count or segment length above `i32::MAX`. /// The total number of slots must also fit `i32::MAX` so that Java can index all three /// segments; this is an additional interoperability limit, not a header field. Construction /// and raw parts allow larger capacities, so successful construction does not guarantee @@ -69,8 +72,10 @@ impl XorFilter { bytes.write_u16_le(fingerprint); } } - XorFingerprints::Xor32(_) => { - unreachable!("serialized_size rejected 32-bit fingerprints") + XorFingerprints::Xor32(fingerprints) => { + for &fingerprint in fingerprints.iter() { + bytes.write_u32_le(fingerprint); + } } } @@ -79,8 +84,8 @@ impl XorFilter { /// Deserializes an owned filter from bytes. /// - /// Accepts 8- and 16-bit images within the limits of [`serialize`](Self::serialize). Reserved - /// fields and trailing bytes are ignored. + /// Accepts 8-, 16-, and 32-bit images within the limits of [`serialize`](Self::serialize). + /// Reserved fields and trailing bytes are ignored. /// /// # Errors /// @@ -112,9 +117,10 @@ impl XorFilter { let filter_type = match bits_per_fingerprint { 8 => XorFilterType::Xor8, 16 => XorFilterType::Xor16, + 32 => XorFilterType::Xor32, _ => { return Err(Error::deserial( - "standard xor format requires 8- or 16-bit fingerprints", + "standard xor format requires 8-, 16-, or 32-bit fingerprints", )); } }; @@ -192,7 +198,17 @@ impl XorFilter { } XorFingerprints::Xor16(values.into_boxed_slice()) } - XorFilterType::Xor32 => unreachable!("header validation rejected 32-bit fingerprints"), + XorFilterType::Xor32 => { + let mut values = Vec::with_capacity(capacity); + for _ in 0..capacity { + values.push( + cursor + .read_u32_le() + .map_err(insufficient_data("fingerprints"))?, + ); + } + XorFingerprints::Xor32(values.into_boxed_slice()) + } }; Ok(Self { @@ -209,11 +225,6 @@ impl XorFilter { /// /// Returns an error under the same conditions as [`serialize`](Self::serialize). pub fn serialized_size(&self) -> Result { - if self.filter_type() == XorFilterType::Xor32 { - return Err(Error::invalid_argument( - "standard xor format does not support 32-bit fingerprints", - )); - } let count = self .num_items .ok_or_else(|| Error::invalid_argument("standard xor format requires an item count"))?; diff --git a/tests-integration/tests/serde_tests/xor.rs b/tests-integration/tests/serde_tests/xor.rs index f020f132..f23e2f3a 100644 --- a/tests-integration/tests/serde_tests/xor.rs +++ b/tests-integration/tests/serde_tests/xor.rs @@ -19,6 +19,7 @@ use datasketches::error::ErrorKind; use datasketches::xor::XorFilter; use datasketches::xor::XorFilterBuilder; use datasketches::xor::XorFilterType; +use datasketches::xor::XorFingerprints; // Generated from datasketches-java commit 1034fb33b6e013334ff870f130b9c388cae5dfa9. // XorFilterBuilder.create([1, 2, 3, Long.MIN_VALUE], bits, 42).toByteArray(). @@ -71,22 +72,35 @@ fn java_images_round_trip_byte_for_byte() { #[test] fn serialization_round_trips_standard_fingerprint_types() { - for filter_type in [XorFilterType::Xor8, XorFilterType::Xor16] { - let original = XorFilter::from_hashes(0..10_000_u64, filter_type).unwrap(); - let bytes = original.serialize().unwrap(); - assert_eq!(bytes.len(), original.serialized_size().unwrap()); + for filter_type in [ + XorFilterType::Xor8, + XorFilterType::Xor16, + XorFilterType::Xor32, + ] { + for count in [0, 1, 10_000_u64] { + let original = XorFilter::from_hashes(0..count, filter_type).unwrap(); + let bytes = original.serialize().unwrap(); + assert_eq!(bytes.len(), original.serialized_size().unwrap()); - let restored = XorFilter::deserialize(&bytes).unwrap(); - assert_eq!(restored, original); - for hash in 0..10_000_u64 { - assert!(restored.contains_hash(hash)); + let restored = XorFilter::deserialize(&bytes).unwrap(); + assert_eq!(restored, original); + for hash in 0..count { + assert!(restored.contains_hash(hash)); + } + if count == 0 { + assert!(!restored.contains_hash(restored.seed().wrapping_neg())); + } } } } #[test] fn every_truncated_image_is_rejected() { - for filter_type in [XorFilterType::Xor8, XorFilterType::Xor16] { + for filter_type in [ + XorFilterType::Xor8, + XorFilterType::Xor16, + XorFilterType::Xor32, + ] { let bytes = XorFilter::from_hashes(0..100_u64, filter_type) .unwrap() .serialize() @@ -112,7 +126,7 @@ fn invalid_preamble_fields_are_rejected() { (1, 2), (2, 21), (4, 7), - (4, 32), + (4, 24), (4, 64), (5, 4), ] { @@ -151,11 +165,17 @@ fn unsafe_lengths_are_rejected_before_indexing() { #[test] fn trailing_storage_is_ignored() { - let original = XorFilter::from_hashes(0..100_u64, XorFilterType::Xor16).unwrap(); - let mut bytes = original.serialize().unwrap(); - bytes.extend_from_slice(&[0xaa; 16]); + for kind in [ + XorFilterType::Xor8, + XorFilterType::Xor16, + XorFilterType::Xor32, + ] { + let original = XorFilter::from_hashes(0..100_u64, kind).unwrap(); + let mut bytes = original.serialize().unwrap(); + bytes.extend_from_slice(&[0xaa; 16]); - assert_eq!(XorFilter::deserialize(&bytes).unwrap(), original); + assert_eq!(XorFilter::deserialize(&bytes).unwrap(), original); + } } fn decode_hex(input: &str) -> Vec { @@ -205,22 +225,30 @@ fn java_utf8_and_raw_byte_inputs_match() { } #[test] -fn nonstandard_filters_require_application_defined_storage() { - let filter = XorFilter::from_hashes(0..10, XorFilterType::Xor32).unwrap(); - assert_eq!( - filter.serialize().unwrap_err().kind(), - ErrorKind::InvalidArgument - ); - assert_eq!( - filter.serialized_size().unwrap_err().kind(), - ErrorKind::InvalidArgument - ); +fn xor32_image_preserves_full_width_little_endian_slots() { + // Hand-authored layout fixture, independent of the construction algorithm: + // family 22/version 1, width 32, three one-slot segments, one item. + let bytes = decode_hex(concat!( + "0301160020030000", + "efcdab8967452301", + "0100000001000000", + "67452301efcdab89ffffffff", + )); + let filter = XorFilter::from_parts( + 0x0123_4567_89ab_cdef, + 1, + XorFingerprints::Xor32(vec![0x0123_4567, 0x89ab_cdef, u32::MAX].into_boxed_slice()), + Some(1), + ) + .unwrap(); + assert_eq!(filter.serialized_size().unwrap(), bytes.len()); + assert_eq!(filter.serialize().unwrap(), bytes); + assert_eq!(XorFilter::deserialize(&bytes).unwrap(), filter); } #[test] fn raw_parts_preserve_an_application_storage_layout() { use datasketches::codec::SketchSlice; - use datasketches::xor::XorFingerprints; // Synthetic storage fixtures with public xorf 0.13.0 payloads for [0, 1, 2, u64::MAX]. // This test-only envelope omits cardinality: magic, width:u8, segment:u32, seed:u64, // and little-endian slots. It is not a production storage format. From 95c990229309ed1386c959dc133de97b3c55530e Mon Sep 17 00:00:00 2001 From: tison Date: Fri, 30 Oct 2026 01:49:03 +0800 Subject: [PATCH 6/6] refactor(xor): name raw parts and stop early on duplicate hashes `XorFilter::into_parts` and `from_parts` exchanged a four-element tuple, leaving the seed, segment length, and optional item count unnamed at call sites and making any additional field a breaking change. They now exchange `XorFilterParts`, created with `XorFilterParts::new(seed, segment_length, fingerprints)` and an optional `with_num_items`, while `from_parts` keeps validating the layout. `XorFilter::from_hashes` and `XorFilter::from_distinct_hashes` duplicated the builder methods. Construction now goes through `XorFilterBuilder` only, as Bloom filters do through `BloomFilterBuilder`. Duplicate input to `build_from_distinct_hashes` previously consumed all 100 construction attempts before failing, several dozen times the cost of a successful build. After a failed attempt, a slot that still holds two hashes whose XOR is zero now reports duplicate input immediately. Mixing is a bijection for a fixed seed, so only equal input hashes can cancel out. The Java fixture tests now also check that filters built from the same inputs and base seed reproduce Java's header: construction seed, segment length, and item count. Fingerprint slots may still differ because the peeling order differs. The README now lists 32-bit fingerprints in the standard format, and the module documentation no longer promises layout parity with a third-party crate. --- README.md | 2 +- benchmarks/xor/build.rs | 38 ++++- benchmarks/xor/query.rs | 32 ++-- datasketches/src/xor/filter.rs | 173 +++++++++++++-------- datasketches/src/xor/mod.rs | 16 +- tests-integration/tests/serde_tests/xor.rs | 50 ++++-- tests-integration/tests/xor_test/filter.rs | 21 ++- tests-integration/tests/xor_test/parts.rs | 56 ++++--- 8 files changed, 254 insertions(+), 134 deletions(-) diff --git a/README.md b/README.md index 4bcc3be3..37a86a31 100644 --- a/README.md +++ b/README.md @@ -83,7 +83,7 @@ See the [API documentation](https://docs.rs/datasketches) for configuration, acc The minimum supported Rust version is 1.86.0. The crate currently supports little-endian targets only. -Supported serialization formats are tested with fixtures produced by Apache DataSketches Java, C++, and Go through the [DataSketches TCK](https://github.com/apache/datasketches-tck). XOR additionally uses fixtures generated from the merged Java implementation until TCK snapshots are available. Its standard format supports 8/16-bit fingerprints; 32-bit fingerprints and formats without an item count use the raw-parts API. +Supported serialization formats are tested with fixtures produced by Apache DataSketches Java, C++, and Go through the [DataSketches TCK](https://github.com/apache/datasketches-tck). XOR additionally uses fixtures generated from the merged Java implementation until TCK snapshots are available. Its standard format supports 8-, 16-, and 32-bit fingerprints; 32-bit images require a reader that supports that width. Formats without an item count use the raw-parts API. Serialization compatibility does not imply that an ordinary Rust `Hash` implementation produces the same update bytes across languages. When sketches must represent the same inputs across implementations, use `hash::value::{raw_bytes, canonical_float, sign_extend, natural_extend}` (and the constructors within those modules) to select matching input encodings. When interoperating with an implementation that skips empty strings, skip them before updating here as well. diff --git a/benchmarks/xor/build.rs b/benchmarks/xor/build.rs index 8d212007..153c0636 100644 --- a/benchmarks/xor/build.rs +++ b/benchmarks/xor/build.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use datasketches::xor::XorFilter; +use datasketches::xor::XorFilterBuilder; use datasketches::xor::XorFilterType; use divan::Bencher; use divan::black_box; @@ -29,7 +29,11 @@ fn datasketches_distinct_8(bencher: Bencher, cardinality: usize) { bencher .counter(ItemsCount::new(cardinality)) .bench_local(|| { - black_box(XorFilter::from_distinct_hashes(&keys, XorFilterType::Xor8).unwrap()) + black_box( + XorFilterBuilder::new(XorFilterType::Xor8) + .build_from_distinct_hashes(&keys) + .unwrap(), + ) }); } @@ -39,7 +43,11 @@ fn datasketches_deduplicate_8(bencher: Bencher, cardinality: usize) { bencher .counter(ItemsCount::new(cardinality)) .bench_local(|| { - black_box(XorFilter::from_hashes(keys.iter().copied(), XorFilterType::Xor8).unwrap()) + black_box( + XorFilterBuilder::new(XorFilterType::Xor8) + .build_from_hashes(keys.iter().copied()) + .unwrap(), + ) }); } @@ -57,7 +65,11 @@ fn datasketches_distinct_16(bencher: Bencher, cardinality: usize) { bencher .counter(ItemsCount::new(cardinality)) .bench_local(|| { - black_box(XorFilter::from_distinct_hashes(&keys, XorFilterType::Xor16).unwrap()) + black_box( + XorFilterBuilder::new(XorFilterType::Xor16) + .build_from_distinct_hashes(&keys) + .unwrap(), + ) }); } @@ -67,7 +79,11 @@ fn datasketches_deduplicate_16(bencher: Bencher, cardinality: usize) { bencher .counter(ItemsCount::new(cardinality)) .bench_local(|| { - black_box(XorFilter::from_hashes(keys.iter().copied(), XorFilterType::Xor16).unwrap()) + black_box( + XorFilterBuilder::new(XorFilterType::Xor16) + .build_from_hashes(keys.iter().copied()) + .unwrap(), + ) }); } @@ -85,7 +101,11 @@ fn datasketches_distinct_32(bencher: Bencher, cardinality: usize) { bencher .counter(ItemsCount::new(cardinality)) .bench_local(|| { - black_box(XorFilter::from_distinct_hashes(&keys, XorFilterType::Xor32).unwrap()) + black_box( + XorFilterBuilder::new(XorFilterType::Xor32) + .build_from_distinct_hashes(&keys) + .unwrap(), + ) }); } @@ -95,7 +115,11 @@ fn datasketches_deduplicate_32(bencher: Bencher, cardinality: usize) { bencher .counter(ItemsCount::new(cardinality)) .bench_local(|| { - black_box(XorFilter::from_hashes(keys.iter().copied(), XorFilterType::Xor32).unwrap()) + black_box( + XorFilterBuilder::new(XorFilterType::Xor32) + .build_from_hashes(keys.iter().copied()) + .unwrap(), + ) }); } diff --git a/benchmarks/xor/query.rs b/benchmarks/xor/query.rs index 9e30e8a7..af31987b 100644 --- a/benchmarks/xor/query.rs +++ b/benchmarks/xor/query.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use datasketches::xor::XorFilter; +use datasketches::xor::XorFilterBuilder; use datasketches::xor::XorFilterType; use divan::Bencher; use divan::black_box; @@ -27,8 +27,9 @@ use crate::xor::hashes; #[divan::bench(args = [16_384, 50_000])] fn datasketches_8_present(bencher: Bencher, cardinality: usize) { let keys = hashes(cardinality * 2); - let filter = - XorFilter::from_distinct_hashes(&keys[..cardinality], XorFilterType::Xor8).unwrap(); + let filter = XorFilterBuilder::new(XorFilterType::Xor8) + .build_from_distinct_hashes(&keys[..cardinality]) + .unwrap(); bencher .counter(ItemsCount::new(cardinality)) .bench_local(|| { @@ -41,8 +42,9 @@ fn datasketches_8_present(bencher: Bencher, cardinality: usize) { #[divan::bench(args = [16_384, 50_000])] fn datasketches_8_absent(bencher: Bencher, cardinality: usize) { let keys = hashes(cardinality * 2); - let filter = - XorFilter::from_distinct_hashes(&keys[..cardinality], XorFilterType::Xor8).unwrap(); + let filter = XorFilterBuilder::new(XorFilterType::Xor8) + .build_from_distinct_hashes(&keys[..cardinality]) + .unwrap(); bencher .counter(ItemsCount::new(cardinality)) .bench_local(|| { @@ -81,8 +83,9 @@ fn xorf_8_absent(bencher: Bencher, cardinality: usize) { #[divan::bench(args = [16_384, 50_000])] fn datasketches_16_present(bencher: Bencher, cardinality: usize) { let keys = hashes(cardinality * 2); - let filter = - XorFilter::from_distinct_hashes(&keys[..cardinality], XorFilterType::Xor16).unwrap(); + let filter = XorFilterBuilder::new(XorFilterType::Xor16) + .build_from_distinct_hashes(&keys[..cardinality]) + .unwrap(); bencher .counter(ItemsCount::new(cardinality)) .bench_local(|| { @@ -95,8 +98,9 @@ fn datasketches_16_present(bencher: Bencher, cardinality: usize) { #[divan::bench(args = [16_384, 50_000])] fn datasketches_16_absent(bencher: Bencher, cardinality: usize) { let keys = hashes(cardinality * 2); - let filter = - XorFilter::from_distinct_hashes(&keys[..cardinality], XorFilterType::Xor16).unwrap(); + let filter = XorFilterBuilder::new(XorFilterType::Xor16) + .build_from_distinct_hashes(&keys[..cardinality]) + .unwrap(); bencher .counter(ItemsCount::new(cardinality)) .bench_local(|| { @@ -135,8 +139,9 @@ fn xorf_16_absent(bencher: Bencher, cardinality: usize) { #[divan::bench(args = [16_384, 50_000])] fn datasketches_32_present(bencher: Bencher, cardinality: usize) { let keys = hashes(cardinality * 2); - let filter = - XorFilter::from_distinct_hashes(&keys[..cardinality], XorFilterType::Xor32).unwrap(); + let filter = XorFilterBuilder::new(XorFilterType::Xor32) + .build_from_distinct_hashes(&keys[..cardinality]) + .unwrap(); bencher .counter(ItemsCount::new(cardinality)) .bench_local(|| { @@ -149,8 +154,9 @@ fn datasketches_32_present(bencher: Bencher, cardinality: usize) { #[divan::bench(args = [16_384, 50_000])] fn datasketches_32_absent(bencher: Bencher, cardinality: usize) { let keys = hashes(cardinality * 2); - let filter = - XorFilter::from_distinct_hashes(&keys[..cardinality], XorFilterType::Xor32).unwrap(); + let filter = XorFilterBuilder::new(XorFilterType::Xor32) + .build_from_distinct_hashes(&keys[..cardinality]) + .unwrap(); bencher .counter(ItemsCount::new(cardinality)) .bench_local(|| { diff --git a/datasketches/src/xor/filter.rs b/datasketches/src/xor/filter.rs index d26c86a1..0762b408 100644 --- a/datasketches/src/xor/filter.rs +++ b/datasketches/src/xor/filter.rs @@ -94,6 +94,65 @@ impl XorFingerprints { } } +/// Raw state of an [`XorFilter`] for application-defined storage. +/// +/// Created by [`XorFilter::into_parts`] or [`XorFilterParts::new`], and consumed by +/// [`XorFilter::from_parts`], which validates the layout. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct XorFilterParts { + seed: u64, + segment_length: usize, + fingerprints: XorFingerprints, + num_items: Option, +} + +impl XorFilterParts { + /// Creates parts from a construction seed, segment length, and fingerprint slots. + /// + /// The item count is unknown unless set with [`with_num_items`](Self::with_num_items). + pub fn new(seed: u64, segment_length: usize, fingerprints: XorFingerprints) -> Self { + Self { + seed, + segment_length, + fingerprints, + num_items: None, + } + } + + /// Records the number of distinct hashes in the original input set. + /// + /// Standard serialization and emptiness checks require a known item count. + pub fn with_num_items(mut self, num_items: usize) -> Self { + self.num_items = Some(num_items); + self + } + + /// Returns the construction seed. + pub fn seed(&self) -> u64 { + self.seed + } + + /// Returns the number of fingerprint slots in each of the three segments. + pub fn segment_length(&self) -> usize { + self.segment_length + } + + /// Returns the fingerprint slots. + pub fn fingerprints(&self) -> &XorFingerprints { + &self.fingerprints + } + + /// Returns the number of distinct input hashes, or `None` if unknown. + pub fn num_items(&self) -> Option { + self.num_items + } + + /// Returns the fingerprint slots without copying. + pub fn into_fingerprints(self) -> XorFingerprints { + self.fingerprints + } +} + /// An immutable xor filter for probabilistic set membership. /// /// A query that returns `false` proves that the item was not in the input set. A query that returns @@ -103,7 +162,8 @@ impl XorFingerprints { /// /// Ordinary values passed to [`contains`](Self::contains) are reduced to 64-bit hashes with /// xxHash64 and seed `0`. Use [`contains_hash`](Self::contains_hash) only with the same hashes that -/// were supplied to [`XorFilter::from_hashes`] or [`XorFilterBuilder::build_from_hashes`]. +/// were supplied to [`XorFilterBuilder::build_from_hashes`] or +/// [`XorFilterBuilder::build_from_distinct_hashes`]. #[derive(Debug, Clone, PartialEq, Eq)] pub struct XorFilter { pub(crate) segment_length: usize, @@ -113,71 +173,46 @@ pub struct XorFilter { } impl XorFilter { - /// Builds a filter from precomputed 64-bit hashes. + /// Imports raw parts without requiring a DataSketches serialization header. /// - /// Duplicate hashes are removed before construction. The same hash values must be passed to - /// [`contains_hash`](Self::contains_hash) when querying the result. + /// The caller must preserve the original construction seed and fingerprint order, and query + /// with the original input hashes. A supplied item count must describe the original set. + /// A valid layout alone cannot establish these invariants. /// /// # Errors /// - /// Returns [`InvalidArgument`](crate::error::ErrorKind::InvalidArgument) if the fingerprint - /// slot count exceeds `u32::MAX` or a scratch allocation would exceed `isize::MAX` bytes, - /// or if no peelable construction is found within 100 attempts. Valid input can also - /// exhaust the construction attempts. A successful build may still exceed the limits of - /// [`XorFilter::serialize`]. + /// Returns an error unless the segment length is positive, the payload has exactly three + /// segments, and a supplied item count fits the capacity. Segment lengths above `u32::MAX` + /// are rejected because the index reduction uses 32-bit hash words. /// /// # Examples /// /// ``` /// use datasketches::xor::XorFilter; + /// use datasketches::xor::XorFilterBuilder; + /// use datasketches::xor::XorFilterParts; /// use datasketches::xor::XorFilterType; /// - /// let filter = XorFilter::from_hashes([10, 20, 30], XorFilterType::Xor8).unwrap(); - /// assert!(filter.contains_hash(20)); + /// let filter = XorFilterBuilder::new(XorFilterType::Xor8).build_from_hashes([10, 20, 30])?; + /// let parts = filter.into_parts(); + /// // A storage format that does not record the item count keeps only these fields. + /// let parts = XorFilterParts::new( + /// parts.seed(), + /// parts.segment_length(), + /// parts.into_fingerprints(), + /// ); + /// let imported = XorFilter::from_parts(parts)?; + /// assert!(imported.contains_hash(20)); + /// assert_eq!(imported.num_items(), None); + /// # Ok::<(), datasketches::error::Error>(()) /// ``` - pub fn from_hashes( - hashes: impl IntoIterator, - filter_type: XorFilterType, - ) -> Result { - XorFilterBuilder::new(filter_type).build_from_hashes(hashes) - } - - /// Builds a filter from distinct precomputed hashes without copying or sorting them. - /// - /// This is useful when the caller already owns a deduplicated set of hashes. The default - /// construction seed is `0`; input order does not affect the result. - /// - /// # Errors - /// - /// Returns [`InvalidArgument`](crate::error::ErrorKind::InvalidArgument) for the size limits - /// described by [`from_hashes`](Self::from_hashes), or if no peelable construction is found - /// within 100 attempts. Duplicate hashes prevent peeling but are not checked separately; - /// they produce the same error as an exhausted search over valid input. - /// Changing the seed does not guarantee success. Use [`from_hashes`](Self::from_hashes) - /// to remove duplicates. - pub fn from_distinct_hashes(hashes: &[u64], filter_type: XorFilterType) -> Result { - XorFilterBuilder::new(filter_type).build_from_distinct_hashes(hashes) - } - - /// Imports raw parts without requiring a DataSketches serialization header. - /// - /// The caller must preserve the original construction seed and fingerprint order, and query - /// with the original input hashes. A supplied item count must describe the original set. - /// A valid layout alone cannot establish these invariants. - /// Pass `None` when the storage format does not record the distinct item count. No count is - /// inferred from the payload, and standard serialization requires a known count. - /// - /// # Errors - /// - /// Returns an error unless the segment length is positive, the payload has exactly three - /// segments, and a supplied item count fits the capacity. Segment lengths above `u32::MAX` - /// are rejected because the index reduction uses 32-bit hash words. - pub fn from_parts( - seed: u64, - segment_length: usize, - fingerprints: XorFingerprints, - num_items: Option, - ) -> Result { + pub fn from_parts(parts: XorFilterParts) -> Result { + let XorFilterParts { + seed, + segment_length, + fingerprints, + num_items, + } = parts; if segment_length == 0 || segment_length > u32::MAX as usize || segment_length.checked_mul(3) != Some(fingerprints.len()) @@ -209,15 +244,14 @@ impl XorFilter { &self.fingerprints } - /// Returns the seed, segment length, fingerprint slots, and optional item count without - /// copying. - pub fn into_parts(self) -> (u64, usize, XorFingerprints, Option) { - ( - self.seed, - self.segment_length, - self.fingerprints, - self.num_items, - ) + /// Converts the filter into raw parts without copying the fingerprint slots. + pub fn into_parts(self) -> XorFilterParts { + XorFilterParts { + seed: self.seed, + segment_length: self.segment_length, + fingerprints: self.fingerprints, + num_items: self.num_items, + } } /// Returns `true` if an item is possibly in the input set. @@ -354,6 +388,14 @@ impl XorFilter { if stack_size == num_items { break; } + // For a fixed seed, mixing is a bijection, so two remaining hashes that cancel + // out are the same input hash. Reseeding cannot separate duplicates. + if count.iter().zip(&xor_mask).any(|(&c, &m)| c == 2 && m == 0) { + return Err( + Error::invalid_argument("xor filter input contains duplicate hashes") + .with_context("num_items", num_items), + ); + } } if stack_size != num_items { @@ -500,8 +542,9 @@ impl XorFilterBuilder { /// Returns [`InvalidArgument`](crate::error::ErrorKind::InvalidArgument) under the size limits /// described by [`build_from_hashes`](Self::build_from_hashes), or if no peelable /// construction is found within 100 attempts, even for valid input. Duplicate hashes - /// prevent peeling but are not checked separately; use - /// [`build_from_hashes`](Self::build_from_hashes) to remove duplicates. + /// prevent peeling; a failed attempt usually exposes them and stops construction early, + /// but detection is not guaranteed. Use [`build_from_hashes`](Self::build_from_hashes) to + /// remove duplicates. pub fn build_from_distinct_hashes(self, hashes: &[u64]) -> Result { self.build_with_attempts(hashes, MAX_CONSTRUCTION_ATTEMPTS) } diff --git a/datasketches/src/xor/mod.rs b/datasketches/src/xor/mod.rs index 3584adfd..796db5fc 100644 --- a/datasketches/src/xor/mod.rs +++ b/datasketches/src/xor/mod.rs @@ -23,8 +23,8 @@ //! in the number of distinct hashes. //! //! The [`XorFilterBuilder`] hashes ordinary Rust values with xxHash64. Call -//! [`XorFilterBuilder::build_from_hashes`] or [`XorFilter::from_hashes`] when hashes have already -//! been computed and must not be hashed again. +//! [`XorFilterBuilder::build_from_hashes`] or [`XorFilterBuilder::build_from_distinct_hashes`] +//! when hashes have already been computed and must not be hashed again. //! //! # Cross-language hashing //! @@ -46,12 +46,11 @@ //! //! # Application-defined storage //! -//! [`XorFilter::from_parts`] and [`XorFilter::into_parts`] preserve the construction seed, segment -//! length, and typed fingerprint slots without adding metadata to an existing disk format. -//! Imported parts may omit the item count. Raw parts use the same query algorithm as `xorf` -//! 0.13 for all three widths, but builders need not produce identical fingerprint arrays. -//! Filters with a known zero item count always reject queries; imported parts with an unknown -//! count retain the ordinary fingerprint comparison, including possible false positives. +//! [`XorFilter::into_parts`] and [`XorFilter::from_parts`] exchange [`XorFilterParts`], which +//! preserve the construction seed, segment length, and typed fingerprint slots without adding +//! metadata to an existing disk format. Imported parts may omit the item count. Filters with a +//! known zero item count always reject queries; imported parts with an unknown count retain the +//! ordinary fingerprint comparison, including possible false positives. //! //! # Examples //! @@ -76,5 +75,6 @@ mod serialization; pub use self::filter::XorFilter; pub use self::filter::XorFilterBuilder; +pub use self::filter::XorFilterParts; pub use self::filter::XorFilterType; pub use self::filter::XorFingerprints; diff --git a/tests-integration/tests/serde_tests/xor.rs b/tests-integration/tests/serde_tests/xor.rs index f23e2f3a..e91fdde9 100644 --- a/tests-integration/tests/serde_tests/xor.rs +++ b/tests-integration/tests/serde_tests/xor.rs @@ -18,6 +18,7 @@ use datasketches::error::ErrorKind; use datasketches::xor::XorFilter; use datasketches::xor::XorFilterBuilder; +use datasketches::xor::XorFilterParts; use datasketches::xor::XorFilterType; use datasketches::xor::XorFingerprints; @@ -58,6 +59,13 @@ fn java_images_round_trip_byte_for_byte() { let restored = XorFilter::deserialize(&expected).unwrap(); assert_eq!(restored.serialize().unwrap(), expected); + // Peelability does not depend on peeling order, so both builders select the same + // construction seed and layout. Fingerprint slots may still differ by peeling order. + assert_eq!( + built.serialize().unwrap()[..24], + expected[..24], + "{filter_type:?} header differs from Java" + ); assert_eq!(restored.filter_type(), filter_type); assert_eq!(restored.num_items(), Some(values.len())); for value in values { @@ -78,7 +86,9 @@ fn serialization_round_trips_standard_fingerprint_types() { XorFilterType::Xor32, ] { for count in [0, 1, 10_000_u64] { - let original = XorFilter::from_hashes(0..count, filter_type).unwrap(); + let original = XorFilterBuilder::new(filter_type) + .build_from_hashes(0..count) + .unwrap(); let bytes = original.serialize().unwrap(); assert_eq!(bytes.len(), original.serialized_size().unwrap()); @@ -101,7 +111,8 @@ fn every_truncated_image_is_rejected() { XorFilterType::Xor16, XorFilterType::Xor32, ] { - let bytes = XorFilter::from_hashes(0..100_u64, filter_type) + let bytes = XorFilterBuilder::new(filter_type) + .build_from_hashes(0..100_u64) .unwrap() .serialize() .unwrap(); @@ -115,7 +126,8 @@ fn every_truncated_image_is_rejected() { #[test] fn invalid_preamble_fields_are_rejected() { - let valid = XorFilter::from_hashes(0..100_u64, XorFilterType::Xor8) + let valid = XorFilterBuilder::new(XorFilterType::Xor8) + .build_from_hashes(0..100_u64) .unwrap() .serialize() .unwrap(); @@ -139,7 +151,8 @@ fn invalid_preamble_fields_are_rejected() { #[test] fn unsafe_lengths_are_rejected_before_indexing() { - let valid = XorFilter::from_hashes(0..100_u64, XorFilterType::Xor8) + let valid = XorFilterBuilder::new(XorFilterType::Xor8) + .build_from_hashes(0..100_u64) .unwrap() .serialize() .unwrap(); @@ -170,7 +183,9 @@ fn trailing_storage_is_ignored() { XorFilterType::Xor16, XorFilterType::Xor32, ] { - let original = XorFilter::from_hashes(0..100_u64, kind).unwrap(); + let original = XorFilterBuilder::new(kind) + .build_from_hashes(0..100_u64) + .unwrap(); let mut bytes = original.serialize().unwrap(); bytes.extend_from_slice(&[0xaa; 16]); @@ -217,6 +232,13 @@ fn java_utf8_and_raw_byte_inputs_match() { let expected = decode_hex(image); let restored = XorFilter::deserialize(&expected).unwrap(); assert_eq!(restored.serialize().unwrap(), expected); + // Peelability does not depend on peeling order, so both builders select the same + // construction seed and layout. Fingerprint slots may still differ by peeling order. + assert_eq!( + built.serialize().unwrap()[..24], + expected[..24], + "{kind:?} header differs from Java" + ); for bytes in [b"hello".as_slice(), "数据🔎".as_bytes(), &[], &[0, 1, 255]] { assert!(built.contains(&raw_bytes::from_slice(bytes))); assert!(restored.contains(&raw_bytes::from_slice(bytes))); @@ -235,10 +257,12 @@ fn xor32_image_preserves_full_width_little_endian_slots() { "67452301efcdab89ffffffff", )); let filter = XorFilter::from_parts( - 0x0123_4567_89ab_cdef, - 1, - XorFingerprints::Xor32(vec![0x0123_4567, 0x89ab_cdef, u32::MAX].into_boxed_slice()), - Some(1), + XorFilterParts::new( + 0x0123_4567_89ab_cdef, + 1, + XorFingerprints::Xor32(vec![0x0123_4567, 0x89ab_cdef, u32::MAX].into_boxed_slice()), + ) + .with_num_items(1), ) .unwrap(); assert_eq!(filter.serialized_size().unwrap(), bytes.len()); @@ -288,15 +312,17 @@ fn raw_parts_preserve_an_application_storage_layout() { } _ => unreachable!(), }; - let filter = XorFilter::from_parts(seed, segment, payload, None).unwrap(); + let filter = XorFilter::from_parts(XorFilterParts::new(seed, segment, payload)).unwrap(); for hash in [0, 1, 2, u64::MAX] { assert!(filter.contains_hash(hash)); } assert_eq!(filter.capacity(), slots); assert_eq!(filter.num_items(), None); - let (seed, segment, payload, count) = filter.into_parts(); - assert_eq!(count, None); + let parts = filter.into_parts(); + assert_eq!(parts.num_items(), None); + let (seed, segment) = (parts.seed(), parts.segment_length()); + let payload = parts.into_fingerprints(); let mut encoded = b"TEST".to_vec(); encoded.push(width); encoded.extend((segment as u32).to_le_bytes()); diff --git a/tests-integration/tests/xor_test/filter.rs b/tests-integration/tests/xor_test/filter.rs index 216313fb..bda1197d 100644 --- a/tests-integration/tests/xor_test/filter.rs +++ b/tests-integration/tests/xor_test/filter.rs @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. -use datasketches::xor::XorFilter; use datasketches::xor::XorFilterBuilder; use datasketches::xor::XorFilterType; @@ -42,7 +41,9 @@ fn precomputed_hashes_have_no_false_negatives() { XorFilterType::Xor16, XorFilterType::Xor32, ] { - let filter = XorFilter::from_hashes(hashes.iter().copied(), filter_type).unwrap(); + let filter = XorFilterBuilder::new(filter_type) + .build_from_hashes(hashes.iter().copied()) + .unwrap(); for &hash in &hashes { assert!(filter.contains_hash(hash), "false negative for {hash}"); } @@ -92,7 +93,9 @@ fn construction_is_independent_of_input_order() { #[test] fn metadata_describes_the_compact_payload() { - let filter = XorFilter::from_hashes(0..10_000_u64, XorFilterType::Xor8).unwrap(); + let filter = XorFilterBuilder::new(XorFilterType::Xor8) + .build_from_hashes(0..10_000_u64) + .unwrap(); assert_eq!(filter.is_empty(), Some(false)); assert_eq!(filter.num_items(), Some(10_000)); @@ -123,9 +126,15 @@ fn false_positive_rates_follow_fingerprint_width() { const NUM_ITEMS: u64 = 50_000; const NUM_QUERIES: u64 = 100_000; - let xor8 = XorFilter::from_hashes(0..NUM_ITEMS, XorFilterType::Xor8).unwrap(); - let xor16 = XorFilter::from_hashes(0..NUM_ITEMS, XorFilterType::Xor16).unwrap(); - let xor32 = XorFilter::from_hashes(0..NUM_ITEMS, XorFilterType::Xor32).unwrap(); + let xor8 = XorFilterBuilder::new(XorFilterType::Xor8) + .build_from_hashes(0..NUM_ITEMS) + .unwrap(); + let xor16 = XorFilterBuilder::new(XorFilterType::Xor16) + .build_from_hashes(0..NUM_ITEMS) + .unwrap(); + let xor32 = XorFilterBuilder::new(XorFilterType::Xor32) + .build_from_hashes(0..NUM_ITEMS) + .unwrap(); let false_positives8 = (NUM_ITEMS..NUM_ITEMS + NUM_QUERIES) .filter(|&hash| xor8.contains_hash(hash)) diff --git a/tests-integration/tests/xor_test/parts.rs b/tests-integration/tests/xor_test/parts.rs index 320e166c..61bf1866 100644 --- a/tests-integration/tests/xor_test/parts.rs +++ b/tests-integration/tests/xor_test/parts.rs @@ -18,6 +18,7 @@ use datasketches::error::ErrorKind; use datasketches::xor::XorFilter; use datasketches::xor::XorFilterBuilder; +use datasketches::xor::XorFilterParts; use datasketches::xor::XorFilterType; use datasketches::xor::XorFingerprints; use xorf::Filter; @@ -58,7 +59,7 @@ fn raw_parts_interoperate_with_xorf_in_both_directions() { ) } }; - let imported = XorFilter::from_parts(seed, segment, payload, None).unwrap(); + let imported = XorFilter::from_parts(XorFilterParts::new(seed, segment, payload)).unwrap(); assert_eq!(imported.num_items(), None); assert_eq!(imported.bits_per_item(), None); assert_eq!(imported.is_empty(), None); @@ -70,9 +71,13 @@ fn raw_parts_interoperate_with_xorf_in_both_directions() { assert!(imported.contains_hash(key)); } - let built = XorFilter::from_distinct_hashes(&keys, kind).unwrap(); - let (seed, block_length, payload, count) = built.clone().into_parts(); - assert_eq!(count, Some(keys.len())); + let built = XorFilterBuilder::new(kind) + .build_from_distinct_hashes(&keys) + .unwrap(); + let parts = built.clone().into_parts(); + assert_eq!(parts.num_items(), Some(keys.len())); + let (seed, block_length) = (parts.seed(), parts.segment_length()); + let payload = parts.into_fingerprints(); // Compare all answers, including false positives, after transferring ownership. let reference: Box> = match payload { XorFingerprints::Xor8(fingerprints) => Box::new(xorf::Xor8 { @@ -106,15 +111,13 @@ fn raw_parts_reject_invalid_layouts_and_counts() { (usize::MAX, 3, None), (1, 3, Some(4)), ] { + let mut parts = + XorFilterParts::new(0, segment, XorFingerprints::Xor8(vec![0; length].into())); + if let Some(count) = count { + parts = parts.with_num_items(count); + } assert_eq!( - XorFilter::from_parts( - 0, - segment, - XorFingerprints::Xor8(vec![0; length].into()), - count - ) - .unwrap_err() - .kind(), + XorFilter::from_parts(parts).unwrap_err().kind(), ErrorKind::InvalidArgument ); } @@ -127,11 +130,15 @@ fn distinct_construction_is_bounded_and_does_not_deduplicate() { XorFilterType::Xor16, XorFilterType::Xor32, ] { - let error = XorFilter::from_distinct_hashes(&[1, 2, 1], kind).unwrap_err(); + let error = XorFilterBuilder::new(kind) + .build_from_distinct_hashes(&[1, 2, 1]) + .unwrap_err(); assert_eq!(error.kind(), ErrorKind::InvalidArgument); - assert!(error.message().contains("100 attempts")); + assert!(error.message().contains("duplicate hashes")); for keys in [vec![], vec![0], vec![u64::MAX], vec![0, u64::MAX]] { - let f = XorFilter::from_distinct_hashes(&keys, kind).unwrap(); + let f = XorFilterBuilder::new(kind) + .build_from_distinct_hashes(&keys) + .unwrap(); assert_eq!(f.num_items(), Some(keys.len())); for key in keys { assert!(f.contains_hash(key)); @@ -147,7 +154,7 @@ fn empty_filters_never_match_even_when_the_fingerprint_is_zero() { XorFilterType::Xor16, XorFilterType::Xor32, ] { - let f = XorFilter::from_hashes([], kind).unwrap(); + let f = XorFilterBuilder::new(kind).build_from_hashes([]).unwrap(); // This key mixes to zero under the filter's construction seed. assert!(!f.contains_hash(f.seed().wrapping_neg())); } @@ -204,12 +211,17 @@ fn unknown_count_does_not_infer_emptiness_from_zero_fingerprints() { XorFilterType::Xor16, XorFilterType::Xor32, ] { - let empty = XorFilter::from_hashes([], kind).unwrap(); + let empty = XorFilterBuilder::new(kind).build_from_hashes([]).unwrap(); let zero_hash = empty.seed().wrapping_neg(); assert_eq!(empty.is_empty(), Some(true)); assert!(!empty.contains_hash(zero_hash)); - let (seed, segment, payload, _) = empty.into_parts(); - let unknown = XorFilter::from_parts(seed, segment, payload, None).unwrap(); + let parts = empty.into_parts(); + let unknown = XorFilter::from_parts(XorFilterParts::new( + parts.seed(), + parts.segment_length(), + parts.into_fingerprints(), + )) + .unwrap(); assert_eq!(unknown.is_empty(), None); assert_eq!(unknown.num_items(), None); assert_eq!(unknown.bits_per_item(), None); @@ -223,9 +235,9 @@ fn unknown_count_does_not_infer_emptiness_from_zero_fingerprints() { unknown.serialize().unwrap_err().kind(), ErrorKind::InvalidArgument ); - let (seed, segment, payload, count) = unknown.into_parts(); - assert_eq!(count, None); - let restored = XorFilter::from_parts(seed, segment, payload, count).unwrap(); + let parts = unknown.into_parts(); + assert_eq!(parts.num_items(), None); + let restored = XorFilter::from_parts(parts).unwrap(); assert_eq!(restored.is_empty(), None); assert!(restored.contains_hash(zero_hash)); }