From 968c04881900bb0f7f2b05ca906349791fe87022 Mon Sep 17 00:00:00 2001 From: Piotr Kubaj Date: Wed, 4 Nov 2026 08:11:11 +0100 Subject: [PATCH] feat: support big-endian targets The crate refuses to build for big-endian targets (#28, #31) because the Java library requires a little-endian native byte order. The Rust code does not need that for serialization: SketchBytes and SketchSlice read and write with an explicit byte order, and the crate has no native-endian loads or stores. The one place where the host byte order leaked in was hashing. Both MurmurHash3X64128 and XxHash64 implement only Hasher::write, so the default write_u16/u32/u64/u128/usize methods (and the signed variants, which forward to them) hash the native-endian bytes of the integer. On a big-endian host every integer item therefore hashes to a different value than on a little-endian host or in the Java, C++ and Go implementations. With only the compile_error removed, 17 integration tests fail on FreeBSD powerpc64 (big-endian): the bloom, CPC, HLL and XOR serialization compatibility tests against the Java, C++ and Go reference images, several theta set-operation tests and three HLL tests. Override the fixed-width Hasher methods in both hashers to hash the little-endian representation. This changes nothing on little-endian targets. With it, `cargo x test` passes on powerpc64 with the same results as on powerpc64le, including all serialization tests against the TCK snapshots, and stable clippy with -D warnings is clean. Drop the compile_error and the README note, and add unit tests that pin the contract: hashing an integer equals hashing its little-endian bytes. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 1 + README.md | 2 +- datasketches/src/hash/murmurhash.rs | 67 +++++++++++++++++++++++++++++ datasketches/src/hash/xxhash.rs | 63 +++++++++++++++++++++++++++ datasketches/src/lib.rs | 4 -- 5 files changed, 132 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8fc9c254..0446c4e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ All significant changes to this project will be documented in this file. * Add `HllUnion::serialize` to serialize union results directly. * Add immutable `XorFilter` behind the `xor` feature, with 8-, 16-, and 32-bit fingerprints, construction from values with automatic deduplication or borrowed distinct precomputed hashes, bounded seeded retries and compact temporary storage, optional cardinality metadata for imported filters, standard serialization for all three widths (Java-compatible for 8/16-bit filters), and validated imports for application-defined storage. +* Big-endian targets are supported. Sketches updated with the same items produce the same estimates and serialized images on big-endian and little-endian hosts. ### Improvements diff --git a/README.md b/README.md index 37a86a31..af11fbc6 100644 --- a/README.md +++ b/README.md @@ -81,7 +81,7 @@ See the [API documentation](https://docs.rs/datasketches) for configuration, acc ## Compatibility -The minimum supported Rust version is 1.86.0. The crate currently supports little-endian targets only. +The minimum supported Rust version is 1.86.0. Supported serialization formats are tested with fixtures produced by Apache DataSketches Java, C++, and Go through the [DataSketches TCK](https://github.com/apache/datasketches-tck). XOR additionally uses fixtures generated from the merged Java implementation until TCK snapshots are available. Its standard format supports 8-, 16-, and 32-bit fingerprints; 32-bit images require a reader that supports that width. Formats without an item count use the raw-parts API. diff --git a/datasketches/src/hash/murmurhash.rs b/datasketches/src/hash/murmurhash.rs index 5f62a402..a5ad3c69 100644 --- a/datasketches/src/hash/murmurhash.rs +++ b/datasketches/src/hash/murmurhash.rs @@ -154,6 +154,37 @@ impl Hasher for MurmurHash3X64128 { self.buf[..bytes.len()].copy_from_slice(bytes); self.buf_len = bytes.len(); } + + // The default implementations of the fixed-width methods hash the + // native-endian bytes of the integer, which would make the hash of an + // integer depend on the byte order of the host. Always hash the + // little-endian representation so that sketches built on big-endian + // hosts are identical to those built on little-endian hosts (and to + // those built by the Java and C++ implementations). + #[inline(always)] + fn write_u16(&mut self, i: u16) { + self.write(&i.to_le_bytes()); + } + + #[inline(always)] + fn write_u32(&mut self, i: u32) { + self.write(&i.to_le_bytes()); + } + + #[inline(always)] + fn write_u64(&mut self, i: u64) { + self.write(&i.to_le_bytes()); + } + + #[inline(always)] + fn write_u128(&mut self, i: u128) { + self.write(&i.to_le_bytes()); + } + + #[inline(always)] + fn write_usize(&mut self, i: usize) { + self.write(&i.to_le_bytes()); + } } #[inline(always)] @@ -179,6 +210,42 @@ fn fmix64(mut k: u64) -> u64 { mod tests { use super::*; + #[test] + fn integers_hash_as_little_endian_bytes() { + fn hash_with(f: impl FnOnce(&mut MurmurHash3X64128)) -> (u64, u64) { + let mut hasher = MurmurHash3X64128::with_seed(DEFAULT_UPDATE_SEED); + f(&mut hasher); + hasher.finish128() + } + + let v = 0x0123_4567_89ab_cdef_u64; + let w = ((v as u128) << 64) | v as u128; + assert_eq!( + hash_with(|h| h.write_u16(v as u16)), + hash_with(|h| h.write(&(v as u16).to_le_bytes())) + ); + assert_eq!( + hash_with(|h| h.write_u32(v as u32)), + hash_with(|h| h.write(&(v as u32).to_le_bytes())) + ); + assert_eq!( + hash_with(|h| h.write_u64(v)), + hash_with(|h| h.write(&v.to_le_bytes())) + ); + assert_eq!( + hash_with(|h| h.write_u128(w)), + hash_with(|h| h.write(&w.to_le_bytes())) + ); + assert_eq!( + hash_with(|h| h.write_usize(v as usize)), + hash_with(|h| h.write(&(v as usize).to_le_bytes())) + ); + assert_eq!( + hash_with(|h| h.write_i64(v as i64)), + hash_with(|h| h.write(&v.to_le_bytes())) + ); + } + fn murmurhash3_x64_128(key: &[u8], seed: u64) -> (u64, u64) { let mut hasher = MurmurHash3X64128::with_seed(seed); hasher.write(key); diff --git a/datasketches/src/hash/xxhash.rs b/datasketches/src/hash/xxhash.rs index 1ea55035..8914e60b 100644 --- a/datasketches/src/hash/xxhash.rs +++ b/datasketches/src/hash/xxhash.rs @@ -173,6 +173,33 @@ impl Hasher for XxHash64 { self.buffer_len = remainder.len(); } } + + // See MurmurHash3X64128: hash integers as little-endian bytes so that + // the result does not depend on the byte order of the host. + #[inline(always)] + fn write_u16(&mut self, i: u16) { + self.write(&i.to_le_bytes()); + } + + #[inline(always)] + fn write_u32(&mut self, i: u32) { + self.write(&i.to_le_bytes()); + } + + #[inline(always)] + fn write_u64(&mut self, i: u64) { + self.write(&i.to_le_bytes()); + } + + #[inline(always)] + fn write_u128(&mut self, i: u128) { + self.write(&i.to_le_bytes()); + } + + #[inline(always)] + fn write_usize(&mut self, i: usize) { + self.write(&i.to_le_bytes()); + } } #[inline(always)] @@ -205,6 +232,42 @@ fn finalize(mut hash: u64) -> u64 { mod tests { use super::*; + #[test] + fn integers_hash_as_little_endian_bytes() { + fn hash_with(f: impl FnOnce(&mut XxHash64)) -> u64 { + let mut hasher = XxHash64::with_seed(DEFAULT_SEED); + f(&mut hasher); + hasher.finish64() + } + + let v = 0x0123_4567_89ab_cdef_u64; + let w = ((v as u128) << 64) | v as u128; + assert_eq!( + hash_with(|h| h.write_u16(v as u16)), + hash_with(|h| h.write(&(v as u16).to_le_bytes())) + ); + assert_eq!( + hash_with(|h| h.write_u32(v as u32)), + hash_with(|h| h.write(&(v as u32).to_le_bytes())) + ); + assert_eq!( + hash_with(|h| h.write_u64(v)), + hash_with(|h| h.write(&v.to_le_bytes())) + ); + assert_eq!( + hash_with(|h| h.write_u128(w)), + hash_with(|h| h.write(&w.to_le_bytes())) + ); + assert_eq!( + hash_with(|h| h.write_usize(v as usize)), + hash_with(|h| h.write(&(v as usize).to_le_bytes())) + ); + assert_eq!( + hash_with(|h| h.write_i64(v as i64)), + hash_with(|h| h.write(&v.to_le_bytes())) + ); + } + const PRIME32: u64 = 0x9E3779B1; const PRIME64: u64 = 0x9E3779B185EBCA8D; diff --git a/datasketches/src/lib.rs b/datasketches/src/lib.rs index 18cd2bc4..50ef9f93 100644 --- a/datasketches/src/lib.rs +++ b/datasketches/src/lib.rs @@ -62,10 +62,6 @@ #![cfg_attr(docsrs, feature(doc_cfg))] #![deny(missing_docs)] -// See https://github.com/apache/datasketches-rust/issues/28 for more information. -#[cfg(target_endian = "big")] -compile_error!("datasketches does not support big-endian targets"); - // sketches modules #[cfg(feature = "bloom")] pub mod bloom;