Add XXHash64, XXHash3, and XXHash128 to smithy-checksums

packages/smithy-checksums/.changes/next-release/smithy-checksums-feature-4efedfc0ab3648858d67c42495fb580b.json

@@ -0,0 +1,4 @@
+{
+  "type": "feature",
+  "description": "Added XXHash64, XXHash3, and XXHash128 checksums."
+}

packages/smithy-checksums/Cargo.toml

@@ -14,6 +14,7 @@ path = "rust/lib.rs"
 [dependencies]
 crc-fast = { version = "1.10", default-features = false, features = ["std"] }
 pyo3 = { version = "0.30", features = ["abi3-py312", "abi3t-py315"] }
+twox-hash = { version = "2.1", default-features = false, features = ["std", "xxhash64", "xxhash3_64", "xxhash3_128"] }
 
 [profile.release]
 codegen-units = 1

packages/smithy-checksums/LICENSE-THIRD-PARTY

@@ -958,6 +958,35 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
 
+================================================================================
+MIT License (MIT)
+
+Used by:
+  - twox-hash 2.1.5 (https://github.com/shepmaster/twox-hash)
+
+The MIT License (MIT)
+
+Copyright (c) 2015 Jake Goulding
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+
 ================================================================================
 MIT License (MIT)
 

packages/smithy-checksums/README.md

@@ -1,7 +1,8 @@
 # smithy-checksums
 
 This package provides the checksum algorithms used by Smithy clients: CRC32,
-CRC32C, CRC64NVME, and, through `new()`, SHA-1, SHA-256, SHA-512, and MD5.
+CRC32C, CRC64NVME, XXHash64, XXHash3, XXHash128, and, through `new()`, SHA-1,
+SHA-256, SHA-512, and MD5.
 
 ```python
 import smithy_checksums
@@ -13,7 +14,7 @@ checksum.digest()
 
 ## Native acceleration
 
-The CRC algorithms are implemented in Rust and published as wheels for
+The CRC and XXHash algorithms are implemented in Rust and published as wheels for
 common platforms. Where no wheel is available, the package installs as pure Python
 and produces identical results, much more slowly. `HAS_NATIVE_EXTENSION` reports
 which implementation is in use. CRC32 always uses `zlib` in pure Python, and the

packages/smithy-checksums/build_backend/smithy_checksums_build.py

@@ -36,7 +36,8 @@
 _CARGO_TOML = Path(__file__).parent.parent / "Cargo.toml"
 _PURE_PYTHON_HINT = (
     f"To install the pure-Python implementation instead, set {_ENV_VAR}=never. It "
-    "produces identical checksums, but CRC32C and CRC64NVME are much slower."
+    "produces identical checksums, but CRC32C, CRC64NVME, and the XXHash algorithms "
+    "are much slower."
 )
 
 

packages/smithy-checksums/pyproject.toml

@@ -9,7 +9,7 @@ authors = [
 description = "Checksum algorithms for Smithy tooling, with native acceleration."
 readme = "README.md"
 license = {text = "Apache License 2.0"}
-keywords = ["smithy", "sdk", "checksum", "crc32c", "crc64nvme"]
+keywords = ["smithy", "sdk", "checksum", "crc32c", "crc64nvme", "xxhash"]
 classifiers = [
   "Development Status :: 2 - Pre-Alpha",
   "Intended Audience :: Developers",

packages/smithy-checksums/rust/lib.rs

@@ -4,8 +4,10 @@
 //! Native checksum implementations for `smithy_checksums`.
 //!
 //! Each class mirrors the pure-Python implementation of the same name in
-//! `smithy_checksums._private.crc`, so the two are interchangeable.
+//! `smithy_checksums._private.crc` or `smithy_checksums._private.xxhash`, so
+//! the two are interchangeable.
 
+use std::hash::Hasher as _;
 use std::sync::{Mutex, MutexGuard};
 
 use pyo3::buffer::PyBuffer;
@@ -50,6 +52,57 @@ crc_state!(Crc32State, Crc32IsoHdlc, 4);
 crc_state!(Crc32cState, Crc32Iscsi, 4);
 crc_state!(Crc64NvmeState, Crc64Nvme, 8);
 
+#[derive(Clone)]
+struct XxHash64State(twox_hash::XxHash64);
+
+impl State for XxHash64State {
+    fn new() -> Self {
+        Self(twox_hash::XxHash64::with_seed(0))
+    }
+
+    fn update(&mut self, data: &[u8]) {
+        self.0.write(data);
+    }
+
+    fn digest(&self) -> Vec<u8> {
+        self.0.finish().to_be_bytes().to_vec()
+    }
+}
+
+#[derive(Clone)]
+struct XxHash3State(twox_hash::XxHash3_64);
+
+impl State for XxHash3State {
+    fn new() -> Self {
+        Self(twox_hash::XxHash3_64::new())
+    }
+
+    fn update(&mut self, data: &[u8]) {
+        self.0.write(data);
+    }
+
+    fn digest(&self) -> Vec<u8> {
+        self.0.finish().to_be_bytes().to_vec()
+    }
+}
+
+#[derive(Clone)]
+struct XxHash128State(twox_hash::XxHash3_128);
+
+impl State for XxHash128State {
+    fn new() -> Self {
+        Self(twox_hash::XxHash3_128::new())
+    }
+
+    fn update(&mut self, data: &[u8]) {
+        self.0.write(data);
+    }
+
+    fn digest(&self) -> Vec<u8> {
+        self.0.finish_128().to_be_bytes().to_vec()
+    }
+}
+
 fn lock<S>(state: &Mutex<S>) -> MutexGuard<'_, S> {
     // A panic while holding the lock can't leave a checksum half-updated in a
     // way that matters more than the panic itself, so ignore poisoning.
@@ -129,9 +182,12 @@ macro_rules! checksum_class {
 checksum_class!(Crc32, Crc32State);
 checksum_class!(Crc32C, Crc32cState);
 checksum_class!(Crc64Nvme, Crc64NvmeState);
+checksum_class!(XxHash64, XxHash64State);
+checksum_class!(XxHash3, XxHash3State);
+checksum_class!(XxHash128, XxHash128State);
 
 #[pymodule]
 mod _native {
     #[pymodule_export]
-    use super::{Crc32, Crc32C, Crc64Nvme};
+    use super::{Crc32, Crc32C, Crc64Nvme, XxHash3, XxHash64, XxHash128};
 }

packages/smithy-checksums/src/smithy_checksums/__init__.py

@@ -2,11 +2,11 @@
 #  SPDX-License-Identifier: Apache-2.0
 """Checksum algorithms for Smithy clients.
 
-CRC32, CRC32C, and CRC64NVME use a native extension when one is available for the
-platform and fall back to pure Python otherwise. The fallback produces identical
-results but is orders of magnitude slower, except for CRC32, which uses :mod:`zlib`.
-SHA-1, SHA-256, SHA-512, and MD5 are available through :func:`new` and always use
-:mod:`hashlib`.
+CRC32, CRC32C, CRC64NVME, XXHash64, XXHash3, and XXHash128 use a native extension when
+one is available for the platform and fall back to pure Python otherwise. The
+fallback produces identical results but is orders of magnitude slower, except for
+CRC32, which uses :mod:`zlib`. SHA-1, SHA-256, SHA-512, and MD5 are available
+through :func:`new` and always use :mod:`hashlib`.
 """
 
 import hashlib
@@ -26,13 +26,17 @@
     "CRC64NVME",
     "HAS_NATIVE_EXTENSION",
     "Checksum",
+    "XXHash3",
+    "XXHash64",
+    "XXHash128",
     "new",
 )
 
 HAS_NATIVE_EXTENSION: Final[bool] = _NATIVE
 """Whether the native extension is in use.
 
-When it isn't, CRC32C and CRC64NVME are computed in pure Python.
+When it isn't, CRC32C, CRC64NVME, XXHash64, XXHash3, and XXHash128 are computed in
+pure Python.
 """
 
 
@@ -132,7 +136,36 @@ class CRC64NVME(_BackendChecksum):
     digest_size = 8
 
 
-_CHECKSUMS: Final = {cls.name: cls for cls in (CRC32, CRC32C, CRC64NVME)}
+class XXHash64(_BackendChecksum):
+    """XXH64 with a seed of zero."""
+
+    __slots__ = ()
+
+    name = "xxhash64"
+    digest_size = 8
+
+
+class XXHash3(_BackendChecksum):
+    """XXH3 (64-bit) with a seed of zero and the default secret."""
+
+    __slots__ = ()
+
+    name = "xxhash3"
+    digest_size = 8
+
+
+class XXHash128(_BackendChecksum):
+    """XXH3 (128-bit) with a seed of zero and the default secret."""
+
+    __slots__ = ()
+
+    name = "xxhash128"
+    digest_size = 16
+
+
+_CHECKSUMS: Final = {
+    cls.name: cls for cls in (CRC32, CRC32C, CRC64NVME, XXHash64, XXHash3, XXHash128)
+}
 _HASHLIB_ALGORITHMS: Final = frozenset(("sha1", "sha256", "sha512"))
 
 
@@ -142,8 +175,8 @@ def new(name: str) -> Checksum:
     Names are case-insensitive, so the values of a checksum algorithm enum such as
     ``CRC64NVME`` or ``SHA256`` can be passed directly.
 
-    :param name: One of ``crc32``, ``crc32c``, ``crc64nvme``, ``sha1``, ``sha256``,
-        ``sha512``, or ``md5``.
+    :param name: One of ``crc32``, ``crc32c``, ``crc64nvme``, ``xxhash64``,
+        ``xxhash3``, ``xxhash128``, ``sha1``, ``sha256``, ``sha512``, or ``md5``.
     :raises ValueError: If the algorithm isn't supported.
     """
     key = name.lower()

packages/smithy-checksums/src/smithy_checksums/_native.pyi

@@ -20,3 +20,18 @@ class Crc64Nvme:
     def update(self, data: _Bytes, /) -> None: ...
     def digest(self) -> bytes: ...
     def copy(self) -> Self: ...
+
+class XxHash64:
+    def update(self, data: _Bytes, /) -> None: ...
+    def digest(self) -> bytes: ...
+    def copy(self) -> Self: ...
+
+class XxHash3:
+    def update(self, data: _Bytes, /) -> None: ...
+    def digest(self) -> bytes: ...
+    def copy(self) -> Self: ...
+
+class XxHash128:
+    def update(self, data: _Bytes, /) -> None: ...
+    def digest(self) -> bytes: ...
+    def copy(self) -> Self: ...

packages/smithy-checksums/src/smithy_checksums/_private/backend.py

@@ -26,18 +26,24 @@ def _load() -> tuple[bool, Mapping[str, ImplFactory]]:
     try:
         from smithy_checksums import _native
     except ImportError:
-        from smithy_checksums._private import crc
+        from smithy_checksums._private import crc, xxhash
 
         return False, {
             "crc32": crc.Crc32,
             "crc32c": crc.Crc32C,
             "crc64nvme": crc.Crc64Nvme,
+            "xxhash64": xxhash.XxHash64,
+            "xxhash3": xxhash.XxHash3,
+            "xxhash128": xxhash.XxHash128,
         }
 
     return True, {
         "crc32": _native.Crc32,
         "crc32c": _native.Crc32C,
         "crc64nvme": _native.Crc64Nvme,
+        "xxhash64": _native.XxHash64,
+        "xxhash3": _native.XxHash3,
+        "xxhash128": _native.XxHash128,
     }