diff --git a/CHANGELOG.md b/CHANGELOG.md index 8fc9c254..0446c4e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ All significant changes to this project will be documented in this file. * Add `HllUnion::serialize` to serialize union results directly. * Add immutable `XorFilter` behind the `xor` feature, with 8-, 16-, and 32-bit fingerprints, construction from values with automatic deduplication or borrowed distinct precomputed hashes, bounded seeded retries and compact temporary storage, optional cardinality metadata for imported filters, standard serialization for all three widths (Java-compatible for 8/16-bit filters), and validated imports for application-defined storage. +* Big-endian targets are supported. Sketches updated with the same items produce the same estimates and serialized images on big-endian and little-endian hosts. ### Improvements diff --git a/README.md b/README.md index 37a86a31..af11fbc6 100644 --- a/README.md +++ b/README.md @@ -81,7 +81,7 @@ See the [API documentation](https://docs.rs/datasketches) for configuration, acc ## Compatibility -The minimum supported Rust version is 1.86.0. The crate currently supports little-endian targets only. +The minimum supported Rust version is 1.86.0. Supported serialization formats are tested with fixtures produced by Apache DataSketches Java, C++, and Go through the [DataSketches TCK](https://github.com/apache/datasketches-tck). XOR additionally uses fixtures generated from the merged Java implementation until TCK snapshots are available. Its standard format supports 8-, 16-, and 32-bit fingerprints; 32-bit images require a reader that supports that width. Formats without an item count use the raw-parts API. diff --git a/datasketches/src/hash/murmurhash.rs b/datasketches/src/hash/murmurhash.rs index 5f62a402..a5ad3c69 100644 --- a/datasketches/src/hash/murmurhash.rs +++ b/datasketches/src/hash/murmurhash.rs @@ -154,6 +154,37 @@ impl Hasher for MurmurHash3X64128 { self.buf[..bytes.len()].copy_from_slice(bytes); self.buf_len = bytes.len(); } + + // The default implementations of the fixed-width methods hash the + // native-endian bytes of the integer, which would make the hash of an + // integer depend on the byte order of the host. Always hash the + // little-endian representation so that sketches built on big-endian + // hosts are identical to those built on little-endian hosts (and to + // those built by the Java and C++ implementations). + #[inline(always)] + fn write_u16(&mut self, i: u16) { + self.write(&i.to_le_bytes()); + } + + #[inline(always)] + fn write_u32(&mut self, i: u32) { + self.write(&i.to_le_bytes()); + } + + #[inline(always)] + fn write_u64(&mut self, i: u64) { + self.write(&i.to_le_bytes()); + } + + #[inline(always)] + fn write_u128(&mut self, i: u128) { + self.write(&i.to_le_bytes()); + } + + #[inline(always)] + fn write_usize(&mut self, i: usize) { + self.write(&i.to_le_bytes()); + } } #[inline(always)] @@ -179,6 +210,42 @@ fn fmix64(mut k: u64) -> u64 { mod tests { use super::*; + #[test] + fn integers_hash_as_little_endian_bytes() { + fn hash_with(f: impl FnOnce(&mut MurmurHash3X64128)) -> (u64, u64) { + let mut hasher = MurmurHash3X64128::with_seed(DEFAULT_UPDATE_SEED); + f(&mut hasher); + hasher.finish128() + } + + let v = 0x0123_4567_89ab_cdef_u64; + let w = ((v as u128) << 64) | v as u128; + assert_eq!( + hash_with(|h| h.write_u16(v as u16)), + hash_with(|h| h.write(&(v as u16).to_le_bytes())) + ); + assert_eq!( + hash_with(|h| h.write_u32(v as u32)), + hash_with(|h| h.write(&(v as u32).to_le_bytes())) + ); + assert_eq!( + hash_with(|h| h.write_u64(v)), + hash_with(|h| h.write(&v.to_le_bytes())) + ); + assert_eq!( + hash_with(|h| h.write_u128(w)), + hash_with(|h| h.write(&w.to_le_bytes())) + ); + assert_eq!( + hash_with(|h| h.write_usize(v as usize)), + hash_with(|h| h.write(&(v as usize).to_le_bytes())) + ); + assert_eq!( + hash_with(|h| h.write_i64(v as i64)), + hash_with(|h| h.write(&v.to_le_bytes())) + ); + } + fn murmurhash3_x64_128(key: &[u8], seed: u64) -> (u64, u64) { let mut hasher = MurmurHash3X64128::with_seed(seed); hasher.write(key); diff --git a/datasketches/src/hash/xxhash.rs b/datasketches/src/hash/xxhash.rs index 1ea55035..8914e60b 100644 --- a/datasketches/src/hash/xxhash.rs +++ b/datasketches/src/hash/xxhash.rs @@ -173,6 +173,33 @@ impl Hasher for XxHash64 { self.buffer_len = remainder.len(); } } + + // See MurmurHash3X64128: hash integers as little-endian bytes so that + // the result does not depend on the byte order of the host. + #[inline(always)] + fn write_u16(&mut self, i: u16) { + self.write(&i.to_le_bytes()); + } + + #[inline(always)] + fn write_u32(&mut self, i: u32) { + self.write(&i.to_le_bytes()); + } + + #[inline(always)] + fn write_u64(&mut self, i: u64) { + self.write(&i.to_le_bytes()); + } + + #[inline(always)] + fn write_u128(&mut self, i: u128) { + self.write(&i.to_le_bytes()); + } + + #[inline(always)] + fn write_usize(&mut self, i: usize) { + self.write(&i.to_le_bytes()); + } } #[inline(always)] @@ -205,6 +232,42 @@ fn finalize(mut hash: u64) -> u64 { mod tests { use super::*; + #[test] + fn integers_hash_as_little_endian_bytes() { + fn hash_with(f: impl FnOnce(&mut XxHash64)) -> u64 { + let mut hasher = XxHash64::with_seed(DEFAULT_SEED); + f(&mut hasher); + hasher.finish64() + } + + let v = 0x0123_4567_89ab_cdef_u64; + let w = ((v as u128) << 64) | v as u128; + assert_eq!( + hash_with(|h| h.write_u16(v as u16)), + hash_with(|h| h.write(&(v as u16).to_le_bytes())) + ); + assert_eq!( + hash_with(|h| h.write_u32(v as u32)), + hash_with(|h| h.write(&(v as u32).to_le_bytes())) + ); + assert_eq!( + hash_with(|h| h.write_u64(v)), + hash_with(|h| h.write(&v.to_le_bytes())) + ); + assert_eq!( + hash_with(|h| h.write_u128(w)), + hash_with(|h| h.write(&w.to_le_bytes())) + ); + assert_eq!( + hash_with(|h| h.write_usize(v as usize)), + hash_with(|h| h.write(&(v as usize).to_le_bytes())) + ); + assert_eq!( + hash_with(|h| h.write_i64(v as i64)), + hash_with(|h| h.write(&v.to_le_bytes())) + ); + } + const PRIME32: u64 = 0x9E3779B1; const PRIME64: u64 = 0x9E3779B185EBCA8D; diff --git a/datasketches/src/lib.rs b/datasketches/src/lib.rs index 18cd2bc4..50ef9f93 100644 --- a/datasketches/src/lib.rs +++ b/datasketches/src/lib.rs @@ -62,10 +62,6 @@ #![cfg_attr(docsrs, feature(doc_cfg))] #![deny(missing_docs)] -// See https://github.com/apache/datasketches-rust/issues/28 for more information. -#[cfg(target_endian = "big")] -compile_error!("datasketches does not support big-endian targets"); - // sketches modules #[cfg(feature = "bloom")] pub mod bloom;