feat(xor): add immutable filters with portable serialization

CHANGELOG.md

@@ -4,6 +4,10 @@ All significant changes to this project will be documented in this file.
 
 ## Unreleased
 
+### New features
+
+* Add immutable `XorFilter` and `XorFilterBuilder` behind the `xor` feature, with 8-, 16-, and 32-bit fingerprints, consuming builder configuration with complete-input construction and bounded seeded retries, optional cardinality metadata for imported filters, standard serialization for all three widths (Java-compatible for 8/16-bit filters), and raw parts for application-defined storage.
+
 ### Improvements
 
 * `KllSketch::rank` is faster for repeated queries on compacted sketches.

CONTRIBUTING.md

@@ -168,6 +168,8 @@ cargo x prepare-testdata java
 
 If no language is specified, all languages are prepared. These directories are not stored in Git. Run the command before the first test run and again whenever the pinned TCK revision changes. It requires network access and replaces the selected generated directories.
 
+The pinned TCK revision does not yet contain XOR filter snapshots. Until it does, `serde_tests/xor.rs` retains small hexadecimal fixtures generated from the merged Java implementation, with the source revision and inputs documented alongside them. Keep these separate from the synchronized `*_generated_files` directories; replace them with TCK fixtures when available.
+
 ## Code of Conduct
 
 We expect all community members to follow our [Code of Conduct](https://www.apache.org/foundation/policies/conduct.html).

README.md

@@ -75,14 +75,15 @@ Enable multiple algorithms by listing their features together, such as `features
 | `tdigest`     | `TDigestMut`, `TDigest`               | Quantile and rank estimation, with high accuracy near distribution tails.                         |
 | `theta`       | `ThetaSketch` and set operations      | Distinct counts, set expressions, and Jaccard similarity.                                         |
 | `tuple`       | `TupleSketch` and set operations      | Theta-style keys with user-defined summaries attached to retained entries.                        |
+| `xor`         | `XorFilter`, `XorFilterBuilder`        | Immutable probabilistic membership with 8-, 16-, or 32-bit fingerprints.                           |
 
 See the [API documentation](https://docs.rs/datasketches) for configuration, accuracy guarantees, serialization, and examples for each algorithm. Runnable end-to-end scenarios live in [`datasketches/examples`](datasketches/examples).
 
 ## Compatibility
 
 The minimum supported Rust version is 1.86.0. The crate currently supports little-endian targets only.
 
-Supported serialization formats are tested with fixtures produced by Apache DataSketches Java, C++, and Go through the [DataSketches TCK](https://github.com/apache/datasketches-tck).
+Supported serialization formats are tested with fixtures produced by Apache DataSketches Java, C++, and Go through the [DataSketches TCK](https://github.com/apache/datasketches-tck). XOR additionally uses fixtures generated from the merged Java implementation until TCK snapshots are available. Its standard format supports 8-, 16-, and 32-bit fingerprints; 32-bit images require a reader that supports that width. Formats without an item count use the raw-parts API.
 
 Serialization compatibility does not imply that an ordinary Rust `Hash` implementation produces the same update bytes across languages. When sketches must represent the same inputs across implementations, use `hash::value::{raw_bytes, canonical_float, sign_extend, natural_extend}` (and the constructors within those modules) to select matching input encodings. When interoperating with an implementation that skips empty strings, skip them before updating here as well.
 

benchmarks/Cargo.toml

@@ -23,6 +23,7 @@ edition.workspace = true
 rust-version.workspace = true
 
 [dev-dependencies]
+xorf = { version = "=0.13.0", default-features = false }
 datasketches = { workspace = true, features = [
   "bloom",
   "countmin",
@@ -34,6 +35,7 @@ datasketches = { workspace = true, features = [
   "tdigest",
   "theta",
   "tuple",
+  "xor",
 ] }
 divan = { workspace = true }
 rand = { workspace = true }

benchmarks/main.rs

@@ -31,6 +31,7 @@ mod req;
 mod tdigest;
 mod theta;
 mod tuple;
+mod xor;
 
 fn main() {
     divan::main();

benchmarks/xor/build.rs

@@ -0,0 +1,132 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use datasketches::xor::XorFilterBuilder;
+use datasketches::xor::XorFilterType;
+use divan::Bencher;
+use divan::black_box;
+use divan::counter::ItemsCount;
+
+use crate::xor::hashes;
+
+#[divan::bench(args = [1_000, 16_384, 49_999, 50_000, 50_001, 100_000])]
+fn datasketches_distinct_8(bencher: Bencher, cardinality: usize) {
+    let keys = hashes(cardinality);
+    bencher
+        .counter(ItemsCount::new(cardinality))
+        .bench_local(|| {
+            black_box(
+                XorFilterBuilder::new(XorFilterType::Xor8)
+                    .build_from_distinct_hashes(&keys)
+                    .unwrap(),
+            )
+        });
+}
+
+#[divan::bench(args = [1_000, 16_384, 49_999, 50_000, 50_001, 100_000])]
+fn datasketches_deduplicate_8(bencher: Bencher, cardinality: usize) {
+    let keys = hashes(cardinality);
+    bencher
+        .counter(ItemsCount::new(cardinality))
+        .bench_local(|| {
+            black_box(
+                XorFilterBuilder::new(XorFilterType::Xor8)
+                    .build_from_hashes(keys.iter().copied())
+                    .unwrap(),
+            )
+        });
+}
+
+#[divan::bench(args = [1_000, 16_384, 49_999, 50_000, 50_001, 100_000])]
+fn xorf_8(bencher: Bencher, cardinality: usize) {
+    let keys = hashes(cardinality);
+    bencher
+        .counter(ItemsCount::new(cardinality))
+        .bench_local(|| black_box(xorf::Xor8::from_iterator(keys.iter().copied())));
+}
+
+#[divan::bench(args = [1_000, 16_384, 49_999, 50_000, 50_001, 100_000])]
+fn datasketches_distinct_16(bencher: Bencher, cardinality: usize) {
+    let keys = hashes(cardinality);
+    bencher
+        .counter(ItemsCount::new(cardinality))
+        .bench_local(|| {
+            black_box(
+                XorFilterBuilder::new(XorFilterType::Xor16)
+                    .build_from_distinct_hashes(&keys)
+                    .unwrap(),
+            )
+        });
+}
+
+#[divan::bench(args = [1_000, 16_384, 49_999, 50_000, 50_001, 100_000])]
+fn datasketches_deduplicate_16(bencher: Bencher, cardinality: usize) {
+    let keys = hashes(cardinality);
+    bencher
+        .counter(ItemsCount::new(cardinality))
+        .bench_local(|| {
+            black_box(
+                XorFilterBuilder::new(XorFilterType::Xor16)
+                    .build_from_hashes(keys.iter().copied())
+                    .unwrap(),
+            )
+        });
+}
+
+#[divan::bench(args = [1_000, 16_384, 49_999, 50_000, 50_001, 100_000])]
+fn xorf_16(bencher: Bencher, cardinality: usize) {
+    let keys = hashes(cardinality);
+    bencher
+        .counter(ItemsCount::new(cardinality))
+        .bench_local(|| black_box(xorf::Xor16::from_iterator(keys.iter().copied())));
+}
+
+#[divan::bench(args = [1_000, 16_384, 49_999, 50_000, 50_001, 100_000])]
+fn datasketches_distinct_32(bencher: Bencher, cardinality: usize) {
+    let keys = hashes(cardinality);
+    bencher
+        .counter(ItemsCount::new(cardinality))
+        .bench_local(|| {
+            black_box(
+                XorFilterBuilder::new(XorFilterType::Xor32)
+                    .build_from_distinct_hashes(&keys)
+                    .unwrap(),
+            )
+        });
+}
+
+#[divan::bench(args = [1_000, 16_384, 49_999, 50_000, 50_001, 100_000])]
+fn datasketches_deduplicate_32(bencher: Bencher, cardinality: usize) {
+    let keys = hashes(cardinality);
+    bencher
+        .counter(ItemsCount::new(cardinality))
+        .bench_local(|| {
+            black_box(
+                XorFilterBuilder::new(XorFilterType::Xor32)
+                    .build_from_hashes(keys.iter().copied())
+                    .unwrap(),
+            )
+        });
+}
+
+#[divan::bench(args = [1_000, 16_384, 49_999, 50_000, 50_001, 100_000])]
+fn xorf_32(bencher: Bencher, cardinality: usize) {
+    let keys = hashes(cardinality);
+    bencher
+        .counter(ItemsCount::new(cardinality))
+        .bench_local(|| black_box(xorf::Xor32::from_iterator(keys.iter().copied())));
+}

benchmarks/xor/matched_build.rs

@@ -0,0 +1,116 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use datasketches::xor::XorFilterBuilder;
+use datasketches::xor::XorFilterType;
+use divan::Bencher;
+use divan::black_box;
+use divan::counter::ItemsCount;
+
+use crate::xor::hashes_with_offset;
+
+// xorf 0.13 starts SplitMix at 1. Both builders use the same seed sequence;
+// equal winning seeds therefore imply equal attempt counts. The offset 917504
+// case exercises a second attempt; every case verifies equal winning seeds.
+fn verify_seed(keys: &[u64], kind: XorFilterType) {
+    let filter = XorFilterBuilder::new(kind)
+        .seed(1)
+        .build_from_distinct_hashes(keys)
+        .unwrap();
+    assert_eq!(filter.seed(), xorf::Xor8::from(keys).seed);
+}
+
+#[divan::bench(args = [(1_000, 0), (16_384, 0), (16_384, 917_504), (49_999, 0), (50_000, 0), (50_001, 0), (100_000, 0), (8_000_000, 0)])]
+fn datasketches_8(bencher: Bencher, case: (usize, u64)) {
+    let (cardinality, offset) = case;
+    let keys = hashes_with_offset(cardinality, offset);
+    verify_seed(&keys, XorFilterType::Xor8);
+    bencher
+        .counter(ItemsCount::new(cardinality))
+        .bench_local(|| {
+            black_box(
+                XorFilterBuilder::new(XorFilterType::Xor8)
+                    .seed(1)
+                    .build_from_distinct_hashes(&keys)
+                    .unwrap(),
+            )
+        });
+}
+
+#[divan::bench(args = [(1_000, 0), (16_384, 0), (16_384, 917_504), (49_999, 0), (50_000, 0), (50_001, 0), (100_000, 0), (8_000_000, 0)])]
+fn xorf_8(bencher: Bencher, case: (usize, u64)) {
+    let (cardinality, offset) = case;
+    let keys = hashes_with_offset(cardinality, offset);
+    verify_seed(&keys, XorFilterType::Xor8);
+    bencher
+        .counter(ItemsCount::new(cardinality))
+        .bench_local(|| black_box(xorf::Xor8::from_iterator(keys.iter().copied())));
+}
+
+#[divan::bench(args = [(1_000, 0), (16_384, 0), (16_384, 917_504), (49_999, 0), (50_000, 0), (50_001, 0), (100_000, 0), (8_000_000, 0)])]
+fn datasketches_16(bencher: Bencher, case: (usize, u64)) {
+    let (cardinality, offset) = case;
+    let keys = hashes_with_offset(cardinality, offset);
+    verify_seed(&keys, XorFilterType::Xor16);
+    bencher
+        .counter(ItemsCount::new(cardinality))
+        .bench_local(|| {
+            black_box(
+                XorFilterBuilder::new(XorFilterType::Xor16)
+                    .seed(1)
+                    .build_from_distinct_hashes(&keys)
+                    .unwrap(),
+            )
+        });
+}
+
+#[divan::bench(args = [(1_000, 0), (16_384, 0), (16_384, 917_504), (49_999, 0), (50_000, 0), (50_001, 0), (100_000, 0), (8_000_000, 0)])]
+fn xorf_16(bencher: Bencher, case: (usize, u64)) {
+    let (cardinality, offset) = case;
+    let keys = hashes_with_offset(cardinality, offset);
+    verify_seed(&keys, XorFilterType::Xor16);
+    bencher
+        .counter(ItemsCount::new(cardinality))
+        .bench_local(|| black_box(xorf::Xor16::from_iterator(keys.iter().copied())));
+}
+
+#[divan::bench(args = [(1_000, 0), (16_384, 0), (16_384, 917_504), (49_999, 0), (50_000, 0), (50_001, 0), (100_000, 0), (8_000_000, 0)])]
+fn datasketches_32(bencher: Bencher, case: (usize, u64)) {
+    let (cardinality, offset) = case;
+    let keys = hashes_with_offset(cardinality, offset);
+    verify_seed(&keys, XorFilterType::Xor32);
+    bencher
+        .counter(ItemsCount::new(cardinality))
+        .bench_local(|| {
+            black_box(
+                XorFilterBuilder::new(XorFilterType::Xor32)
+                    .seed(1)
+                    .build_from_distinct_hashes(&keys)
+                    .unwrap(),
+            )
+        });
+}
+
+#[divan::bench(args = [(1_000, 0), (16_384, 0), (16_384, 917_504), (49_999, 0), (50_000, 0), (50_001, 0), (100_000, 0), (8_000_000, 0)])]
+fn xorf_32(bencher: Bencher, case: (usize, u64)) {
+    let (cardinality, offset) = case;
+    let keys = hashes_with_offset(cardinality, offset);
+    verify_seed(&keys, XorFilterType::Xor32);
+    bencher
+        .counter(ItemsCount::new(cardinality))
+        .bench_local(|| black_box(xorf::Xor32::from_iterator(keys.iter().copied())));
+}

benchmarks/xor/mod.rs

@@ -0,0 +1,36 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+mod build;
+mod matched_build;
+mod query;
+
+// SplitMix is a permutation, so these pseudorandom precomputed hashes are distinct.
+fn hashes(count: usize) -> Vec<u64> {
+    hashes_with_offset(count, 0)
+}
+
+fn hashes_with_offset(count: usize, offset: u64) -> Vec<u64> {
+    (offset..offset + count as u64)
+        .map(|i| {
+            let mut z = i.wrapping_add(0x9e3779b97f4a7c15);
+            z = (z ^ (z >> 30)).wrapping_mul(0xbf58476d1ce4e5b9);
+            z = (z ^ (z >> 27)).wrapping_mul(0x94d049bb133111eb);
+            z ^ (z >> 31)
+        })
+        .collect()
+}