From 8d321bd8f9df891b1b1557b2de306fcc2a2480f0 Mon Sep 17 00:00:00 2001 From: jonathan343 Date: Sat, 24 Oct 2026 22:05:05 -0400 Subject: [PATCH 1/9] Add smithy-checksums package with CRC32 and CRC32C Adds the smithy-checksums package, which provides the checksum algorithms Smithy clients need for flexible checksums. CRC32 and CRC32C are backed by a native Rust extension (PyO3, crc-fast) with a pure-Python fallback, and SHA-1, SHA-256, SHA-512, and MD5 come from hashlib through new(). Wheels are built by an in-tree PEP 517 backend: maturin when a Rust toolchain is available, hatchling (pure Python) otherwise. Source distributions are always built with hatchling. SMITHY_PYTHON_NATIVE_BUILD (auto, always, never) overrides the choice. If Rust is present but too old, or the native build fails, the build stops with instructions instead of silently falling back. Native wheels target abi3 for 3.12+, cp314t, and abi3t for 3.15t+. Native and pure-Python checksum objects are safe to share between threads. Repository setup: - CI builds the native extension in the main matrix, checks Rust formatting, lints, dependencies (cargo-deny), and third-party license attribution (cargo-about), builds and tests native wheels on Linux, macOS, and Windows, and tests the pure-Python wheel and an sdist install without Rust. - Adds make targets check-rust, lint-rust, and third-party-licenses. - License header checks cover .rs and .pyi files. - Dependabot updates Cargo dependencies. --- .github/dependabot.yml | 13 + .github/workflows/ci.yml | 112 ++ .licenserc.yaml | 3 + Makefile | 38 +- ...ture-2c585fb742b2409690bb374cf08cd6cf.json | 4 + packages/smithy-checksums/.gitignore | 3 + packages/smithy-checksums/CHANGELOG.md | 1 + packages/smithy-checksums/Cargo.lock | 190 +++ packages/smithy-checksums/Cargo.toml | 20 + packages/smithy-checksums/LICENSE-THIRD-PARTY | 1034 +++++++++++++++++ packages/smithy-checksums/NOTICE | 1 + packages/smithy-checksums/README.md | 31 + packages/smithy-checksums/about.hbs | 16 + packages/smithy-checksums/about.toml | 11 + .../build_backend/smithy_checksums_build.py | 151 +++ packages/smithy-checksums/deny.toml | 24 + packages/smithy-checksums/pyproject.toml | 81 ++ packages/smithy-checksums/rust/lib.rs | 135 +++ .../src/smithy_checksums/__init__.py | 144 +++ .../src/smithy_checksums/_native.pyi | 17 + .../src/smithy_checksums/_private/__init__.py | 2 + .../src/smithy_checksums/_private/backend.py | 42 + .../src/smithy_checksums/_private/buffers.py | 23 + .../src/smithy_checksums/_private/crc.py | 78 ++ .../src/smithy_checksums/_private/pure.py | 42 + .../src/smithy_checksums/py.typed | 1 + packages/smithy-checksums/tests/__init__.py | 2 + .../smithy-checksums/tests/unit/__init__.py | 2 + .../tests/unit/test_build_backend.py | 162 +++ .../tests/unit/test_checksums.py | 114 ++ .../tests/unit/test_implementations.py | 182 +++ .../smithy-checksums/tests/unit/vectors.py | 67 ++ pyproject.toml | 1 + scripts/api-snapshot/api.json | 7 + uv.lock | 6 + 35 files changed, 2759 insertions(+), 1 deletion(-) create mode 100644 packages/smithy-checksums/.changes/next-release/smithy-checksums-feature-2c585fb742b2409690bb374cf08cd6cf.json create mode 100644 packages/smithy-checksums/.gitignore create mode 100644 packages/smithy-checksums/CHANGELOG.md create mode 100644 packages/smithy-checksums/Cargo.lock create mode 100644 packages/smithy-checksums/Cargo.toml create mode 100644 packages/smithy-checksums/LICENSE-THIRD-PARTY create mode 100644 packages/smithy-checksums/NOTICE create mode 100644 packages/smithy-checksums/README.md create mode 100644 packages/smithy-checksums/about.hbs create mode 100644 packages/smithy-checksums/about.toml create mode 100644 packages/smithy-checksums/build_backend/smithy_checksums_build.py create mode 100644 packages/smithy-checksums/deny.toml create mode 100644 packages/smithy-checksums/pyproject.toml create mode 100644 packages/smithy-checksums/rust/lib.rs create mode 100644 packages/smithy-checksums/src/smithy_checksums/__init__.py create mode 100644 packages/smithy-checksums/src/smithy_checksums/_native.pyi create mode 100644 packages/smithy-checksums/src/smithy_checksums/_private/__init__.py create mode 100644 packages/smithy-checksums/src/smithy_checksums/_private/backend.py create mode 100644 packages/smithy-checksums/src/smithy_checksums/_private/buffers.py create mode 100644 packages/smithy-checksums/src/smithy_checksums/_private/crc.py create mode 100644 packages/smithy-checksums/src/smithy_checksums/_private/pure.py create mode 100644 packages/smithy-checksums/src/smithy_checksums/py.typed create mode 100644 packages/smithy-checksums/tests/__init__.py create mode 100644 packages/smithy-checksums/tests/unit/__init__.py create mode 100644 packages/smithy-checksums/tests/unit/test_build_backend.py create mode 100644 packages/smithy-checksums/tests/unit/test_checksums.py create mode 100644 packages/smithy-checksums/tests/unit/test_implementations.py create mode 100644 packages/smithy-checksums/tests/unit/vectors.py diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 6b54affa2..7c197c07a 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -12,6 +12,19 @@ updates: - "patch" cooldown: default-days: 7 + - package-ecosystem: "cargo" + directories: + - "/packages/smithy-checksums" + schedule: + interval: "weekly" + day: "wednesday" + groups: + cargo: + update-types: + - "minor" + - "patch" + cooldown: + default-days: 7 - package-ecosystem: "gradle" directory: "/codegen" schedule: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 80cb8a5d4..505add17c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,6 +21,9 @@ jobs: fail-fast: false matrix: python-version: ["3.12", "3.13", "3.14", "3.14t", "3.15"] + env: + # Fail rather than fall back to pure Python if a native extension can't be built. + SMITHY_PYTHON_NATIVE_BUILD: always steps: - name: Checkout Repository @@ -49,6 +52,11 @@ jobs: with: version: 3.8.2 + - name: Set up Rust + run: | + rustup toolchain install stable --profile minimal --component clippy,rustfmt + rustup default stable + - name: Setup workspace run: | make install @@ -58,6 +66,17 @@ jobs: run: | make check-py + - name: Install Rust dependency tools + if: ${{ matrix.python-version == '3.12' }} + uses: taiki-e/install-action@1e29709c0ec63babbe8f266397a465f29a825f91 + with: + tool: cargo-about@0.9.2,cargo-deny@0.20.2 + + - name: Check rust extensions + if: ${{ matrix.python-version == '3.12' }} + run: | + make check-rust + - name: Check API snapshot if: ${{ matrix.python-version == '3.12' }} run: | @@ -88,3 +107,96 @@ jobs: if: ${{ matrix.python-version == '3.12' }} run: | make typecheck-protocols + + # Builds each native extension as a wheel on every platform and tests the + # installed wheel. The release pipeline builds the published wheels; this + # catches platform-specific build or test failures before then. + native-wheels: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, ubuntu-24.04-arm, macos-latest, windows-latest] + # 3.12 builds the abi3 wheel, 3.14t a version-specific free-threaded + # wheel, and 3.15t the abi3t wheel. + python-version: ["3.12", "3.14t", "3.15t"] + package: [smithy-checksums] + env: + SMITHY_PYTHON_NATIVE_BUILD: always + PACKAGE: ${{ matrix.package }} + defaults: + run: + shell: bash + + steps: + - name: Checkout Repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + + - name: Install uv and set the Python version + uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 + with: + python-version: ${{ matrix.python-version }} + activate-environment: true + enable-cache: true + + - name: Set up Rust + run: | + rustup toolchain install stable --profile minimal + rustup default stable + + - name: Build wheel + run: | + uv build "packages/$PACKAGE" --wheel --out-dir dist + + - name: Test wheel + run: | + uv pip install pytest dist/*.whl + python -c "import smithy_checksums; assert smithy_checksums.HAS_NATIVE_EXTENSION" + python -m pytest "packages/$PACKAGE/tests" --import-mode=importlib -o addopts="" + + # Tests the pure-Python wheel, and installing from the sdist without a Rust + # toolchain, which should fall back to the same pure-Python build. + pure-python-fallback: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + package: [smithy-checksums] + env: + PACKAGE: ${{ matrix.package }} + + steps: + - name: Checkout Repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + + - name: Install uv and set the Python version + uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 + with: + python-version: "3.12" + activate-environment: true + enable-cache: true + + - name: Build sdist and pure-Python wheel + env: + SMITHY_PYTHON_NATIVE_BUILD: never + run: | + uv build "packages/$PACKAGE" --out-dir dist + + - name: Test pure-Python wheel + run: | + uv pip install pytest awscrt dist/*-py3-none-any.whl + python -c "import smithy_checksums; assert not smithy_checksums.HAS_NATIVE_EXTENSION" + python -m pytest "packages/$PACKAGE/tests" --import-mode=importlib -o addopts="" + + - name: Test sdist install without Rust + run: | + PATH="$(echo "$PATH" | tr ':' '\n' | grep -v '/\.cargo/bin' | paste -sd: -)" + if command -v cargo; then echo "cargo is still on the PATH"; exit 1; fi + uv venv .venv-sdist + uv pip install --python .venv-sdist --no-cache pytest awscrt dist/*.tar.gz + .venv-sdist/bin/python -c "import smithy_checksums; assert not smithy_checksums.HAS_NATIVE_EXTENSION" + .venv-sdist/bin/python -m pytest "packages/$PACKAGE/tests" --import-mode=importlib -o addopts="" diff --git a/.licenserc.yaml b/.licenserc.yaml index 012b76f65..d2e11f094 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -8,6 +8,8 @@ header: paths: - '**/*.py' + - '**/*.pyi' + - '**/*.rs' paths-ignore: - '**/*.md' @@ -22,3 +24,4 @@ header: Python: extensions: - ".py" + - ".pyi" diff --git a/Makefile b/Makefile index d0e5baefc..bc94ded0a 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ .DEFAULT_GOAL:=help -.PHONY: help install format-java check-java build-java test-protocols typecheck-protocols generate-protocol-tests generate-codegen-output-py api-snapshot ci lint-py check-py test-py test-integration benchmark build-py clean +.PHONY: help install format-java check-java build-java test-protocols typecheck-protocols generate-protocol-tests generate-codegen-output-py api-snapshot ci lint-py check-py lint-rust check-rust third-party-licenses test-py test-integration benchmark build-py clean help: ## Show this help. @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n\nTargets:\n"} /^[a-zA-Z_-]+:.*?##/ { printf " \033[36m%-25s\033[0m %s\n", $$1, $$2 }' $(MAKEFILE_LIST) @@ -91,8 +91,44 @@ check-py: ## Runs checks (formatting, lints, type-checking) on the python packag uv run pyright packages +# Packages with a native extension, each built from a Cargo.toml in the package root. +RUST_MANIFESTS := $(wildcard packages/*/Cargo.toml) + +lint-rust: ## Formats and fixes lints in the Rust extensions. + @set -e; for manifest in $(RUST_MANIFESTS); do \ + cargo fmt --manifest-path "$$manifest"; \ + cargo clippy --manifest-path "$$manifest" --all-targets --fix --allow-dirty --allow-staged; \ + done + + +check-rust: ## Checks formatting, lints, dependencies, and license attribution of the Rust extensions. + @set -e; for manifest in $(RUST_MANIFESTS); do \ + dir=$$(dirname "$$manifest"); \ + cargo fmt --manifest-path "$$manifest" --check; \ + cargo clippy --manifest-path "$$manifest" --all-targets --locked -- -D warnings; \ + cargo deny --manifest-path "$$manifest" check; \ + cargo about generate --manifest-path "$$manifest" \ + --output-file "$$dir/LICENSE-THIRD-PARTY.expected" "$$dir/about.hbs"; \ + if ! diff -u "$$dir/LICENSE-THIRD-PARTY" "$$dir/LICENSE-THIRD-PARTY.expected"; then \ + rm "$$dir/LICENSE-THIRD-PARTY.expected"; \ + echo "$$dir/LICENSE-THIRD-PARTY is out of date. Run 'make third-party-licenses'."; \ + exit 1; \ + fi; \ + rm "$$dir/LICENSE-THIRD-PARTY.expected"; \ + done + + +third-party-licenses: ## Regenerates the third-party license file of each Rust extension from its Cargo.lock. + @set -e; for manifest in $(RUST_MANIFESTS); do \ + dir=$$(dirname "$$manifest"); \ + cargo about generate --manifest-path "$$manifest" \ + --output-file "$$dir/LICENSE-THIRD-PARTY" "$$dir/about.hbs"; \ + done + + ci: ## Runs the full CI sequence locally (mirrors the CI workflow steps). make check-py + make check-rust make api-snapshot make test-py make build-py diff --git a/packages/smithy-checksums/.changes/next-release/smithy-checksums-feature-2c585fb742b2409690bb374cf08cd6cf.json b/packages/smithy-checksums/.changes/next-release/smithy-checksums-feature-2c585fb742b2409690bb374cf08cd6cf.json new file mode 100644 index 000000000..6375c1caf --- /dev/null +++ b/packages/smithy-checksums/.changes/next-release/smithy-checksums-feature-2c585fb742b2409690bb374cf08cd6cf.json @@ -0,0 +1,4 @@ +{ + "type": "feature", + "description": "Added the `smithy-checksums` package, which provides CRC32 and CRC32C checksums backed by a native Rust extension with a pure-Python fallback, and SHA-1, SHA-256, SHA-512, and MD5 checksums through `new()`." +} \ No newline at end of file diff --git a/packages/smithy-checksums/.gitignore b/packages/smithy-checksums/.gitignore new file mode 100644 index 000000000..f122d7c95 --- /dev/null +++ b/packages/smithy-checksums/.gitignore @@ -0,0 +1,3 @@ +# Native extension built in place by editable installs. +*.so +*.pyd diff --git a/packages/smithy-checksums/CHANGELOG.md b/packages/smithy-checksums/CHANGELOG.md new file mode 100644 index 000000000..5ddad421e --- /dev/null +++ b/packages/smithy-checksums/CHANGELOG.md @@ -0,0 +1 @@ +# Changelog \ No newline at end of file diff --git a/packages/smithy-checksums/Cargo.lock b/packages/smithy-checksums/Cargo.lock new file mode 100644 index 000000000..792443944 --- /dev/null +++ b/packages/smithy-checksums/Cargo.lock @@ -0,0 +1,190 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "crc-fast" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e75b2483e97a5a7da73ac68a05b629f9c53cff58d8ed1c77866079e18b00dba5" +dependencies = [ + "digest", + "spin", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "crypto-common", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "libc" +version = "0.2.191" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "pyo3" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "229ca13b8a15bb0d1487b81ab975bfcd97513416499a8312011b6036e42e0de4" +dependencies = [ + "libc", + "once_cell", + "portable-atomic", + "pyo3-build-config", + "pyo3-ffi", + "pyo3-macros", +] + +[[package]] +name = "pyo3-build-config" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70130e2b6dd1ee589a8788026a429561ca5c40fa7a2fa7914c2282eddae24700" +dependencies = [ + "target-lexicon", +] + +[[package]] +name = "pyo3-ffi" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e7ff73c555c286d212d4ce7e5fbda845a973c4f61bd5f160f5f2c3c2a04d1c4" +dependencies = [ + "libc", + "pyo3-build-config", +] + +[[package]] +name = "pyo3-macros" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c46d2e5862c7bb8207e3b639f713dda3c499933444784154c11d013c5a691b60" +dependencies = [ + "proc-macro2", + "pyo3-macros-backend", + "quote", + "syn", +] + +[[package]] +name = "pyo3-macros-backend" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38b89f066f2ae72a2aef52d1537ca2fcd104440ebfadee06e97aa90738a25b9b" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "smithy-checksums" +version = "0.0.0" +dependencies = [ + "crc-fast", + "pyo3", +] + +[[package]] +name = "spin" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" + +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "target-lexicon" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" diff --git a/packages/smithy-checksums/Cargo.toml b/packages/smithy-checksums/Cargo.toml new file mode 100644 index 000000000..7fb945a43 --- /dev/null +++ b/packages/smithy-checksums/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "smithy-checksums" +version = "0.0.0" +edition = "2024" +rust-version = "1.89" +publish = false +license = "Apache-2.0" + +[lib] +name = "_native" +crate-type = ["cdylib"] +path = "rust/lib.rs" + +[dependencies] +crc-fast = { version = "1.10", default-features = false, features = ["std"] } +pyo3 = { version = "0.30", features = ["abi3-py312", "abi3t-py315"] } + +[profile.release] +codegen-units = 1 +lto = true diff --git a/packages/smithy-checksums/LICENSE-THIRD-PARTY b/packages/smithy-checksums/LICENSE-THIRD-PARTY new file mode 100644 index 000000000..bf00306c1 --- /dev/null +++ b/packages/smithy-checksums/LICENSE-THIRD-PARTY @@ -0,0 +1,1034 @@ +Third-party software compiled into the smithy-checksums native extension. + +This file is generated from Cargo.lock by `make third-party-licenses`. Don't edit +it by hand. + +================================================================================ +Apache License 2.0 (Apache-2.0) + +Used by: + - crc-fast 1.10.0 (https://github.com/awesomized/crc-fast-rust) + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +================================================================================ +Apache License 2.0 (Apache-2.0) + +Used by: + - typenum 1.20.1 (https://github.com/paholg/typenum) + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright 2014 Paho Lurie-Gregg + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +================================================================================ +Apache License 2.0 (Apache-2.0) + +Used by: + - heck 0.5.0 (https://github.com/withoutboats/heck) + - once_cell 1.21.4 (https://github.com/matklad/once_cell) + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + +================================================================================ +Apache License 2.0 (Apache-2.0) + +Used by: + - crypto-common 0.1.7 (https://github.com/RustCrypto/traits) + - digest 0.10.7 (https://github.com/RustCrypto/traits) + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + +================================================================================ +Apache License 2.0 (Apache-2.0) + +Used by: + - libc 0.2.191 (https://github.com/rust-lang/libc) + - portable-atomic 1.15.0 (https://github.com/taiki-e/portable-atomic) + - proc-macro2 1.0.107 (https://github.com/dtolnay/proc-macro2) + - pyo3-ffi 0.30.1 (https://github.com/pyo3/pyo3) + - pyo3-macros-backend 0.30.1 (https://github.com/pyo3/pyo3) + - pyo3-macros 0.30.1 (https://github.com/pyo3/pyo3) + - pyo3 0.30.1 (https://github.com/pyo3/pyo3) + - quote 1.0.47 (https://github.com/dtolnay/quote) + - syn 3.0.9 (https://github.com/dtolnay/syn) + - unicode-ident 1.0.27 (https://github.com/dtolnay/unicode-ident) + +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. + +"Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: + + (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. + + You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + +To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + +================================================================================ +MIT License (MIT) + +Used by: + - spin 0.10.1 (https://github.com/mvdnes/spin-rs.git) + +The MIT License (MIT) + +Copyright (c) 2014 Mathijs van de Nes + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +================================================================================ +MIT License (MIT) + +Used by: + - generic-array 0.14.7 (https://github.com/fizyk20/generic-array.git) + +The MIT License (MIT) + +Copyright (c) 2015 BartÅ‚omiej KamiÅ„ski + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +================================================================================ +Unicode License v3 (Unicode-3.0) + +Used by: + - unicode-ident 1.0.27 (https://github.com/dtolnay/unicode-ident) + +UNICODE LICENSE V3 + +COPYRIGHT AND PERMISSION NOTICE + +Copyright © 1991-2023 Unicode, Inc. + +NOTICE TO USER: Carefully read the following legal agreement. BY +DOWNLOADING, INSTALLING, COPYING OR OTHERWISE USING DATA FILES, AND/OR +SOFTWARE, YOU UNEQUIVOCALLY ACCEPT, AND AGREE TO BE BOUND BY, ALL OF THE +TERMS AND CONDITIONS OF THIS AGREEMENT. IF YOU DO NOT AGREE, DO NOT +DOWNLOAD, INSTALL, COPY, DISTRIBUTE OR USE THE DATA FILES OR SOFTWARE. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of data files and any associated documentation (the "Data Files") or +software and any associated documentation (the "Software") to deal in the +Data Files or Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, and/or sell +copies of the Data Files or Software, and to permit persons to whom the +Data Files or Software are furnished to do so, provided that either (a) +this copyright and permission notice appear with all copies of the Data +Files or Software, or (b) this copyright and permission notice appear in +associated Documentation. + +THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF +THIRD PARTY RIGHTS. + +IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS NOTICE +BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, +OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, +WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, +ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THE DATA +FILES OR SOFTWARE. + +Except as contained in this notice, the name of a copyright holder shall +not be used in advertising or otherwise to promote the sale, use or other +dealings in these Data Files or Software without prior written +authorization of the copyright holder. + diff --git a/packages/smithy-checksums/NOTICE b/packages/smithy-checksums/NOTICE new file mode 100644 index 000000000..616fc5889 --- /dev/null +++ b/packages/smithy-checksums/NOTICE @@ -0,0 +1 @@ +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. diff --git a/packages/smithy-checksums/README.md b/packages/smithy-checksums/README.md new file mode 100644 index 000000000..227cf26c2 --- /dev/null +++ b/packages/smithy-checksums/README.md @@ -0,0 +1,31 @@ +# smithy-checksums + +This package provides the checksum algorithms used by Smithy clients: CRC32 and +CRC32C, and, through `new()`, SHA-1, SHA-256, SHA-512, and MD5. + +```python +import smithy_checksums + +checksum = smithy_checksums.new("CRC32C") +checksum.update(b"hello world") +checksum.digest() +``` + +## Native acceleration + +The CRC algorithms are implemented in Rust and published as wheels for +common platforms. Where no wheel is available, the package installs as pure Python +and produces identical results, much more slowly. `HAS_NATIVE_EXTENSION` reports +which implementation is in use. CRC32 always uses `zlib` in pure Python, and the +SHA and MD5 algorithms always use `hashlib`. + +## Building + +Wheels are built with [maturin](https://www.maturin.rs/) when `cargo` is on the +`PATH`, and as pure Python when no Rust toolchain is installed. If Rust is installed +but can't build the extension, for example because it's older than the +`rust-version` in `Cargo.toml`, the build fails with instructions instead of +silently falling back. + +Set `SMITHY_PYTHON_NATIVE_BUILD` to `never` to build a pure-Python wheel, or to +`always` to require the native extension even when `cargo` isn't on the `PATH`. diff --git a/packages/smithy-checksums/about.hbs b/packages/smithy-checksums/about.hbs new file mode 100644 index 000000000..d23cd9c79 --- /dev/null +++ b/packages/smithy-checksums/about.hbs @@ -0,0 +1,16 @@ +Third-party software compiled into the smithy-checksums native extension. + +This file is generated from Cargo.lock by `make third-party-licenses`. Don't edit +it by hand. +{{#each licenses}} + +================================================================================ +{{name}} ({{id}}) + +Used by: +{{#each used_by}} + - {{crate.name}} {{crate.version}}{{#if crate.repository}} ({{crate.repository}}){{/if}} +{{/each}} + +{{text}} +{{/each}} diff --git a/packages/smithy-checksums/about.toml b/packages/smithy-checksums/about.toml new file mode 100644 index 000000000..9279ab1b3 --- /dev/null +++ b/packages/smithy-checksums/about.toml @@ -0,0 +1,11 @@ +# Configuration for cargo-about, which generates LICENSE-THIRD-PARTY. +accepted = [ + "Apache-2.0", + "Apache-2.0 WITH LLVM-exception", + "MIT", + "Unicode-3.0", +] +ignore-build-dependencies = true +ignore-dev-dependencies = true +# Our own crate is covered by the package license. +private = { ignore = true } diff --git a/packages/smithy-checksums/build_backend/smithy_checksums_build.py b/packages/smithy-checksums/build_backend/smithy_checksums_build.py new file mode 100644 index 000000000..c41cdc8ad --- /dev/null +++ b/packages/smithy-checksums/build_backend/smithy_checksums_build.py @@ -0,0 +1,151 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""PEP 517 build backend for smithy-checksums. + +Wheels are built with maturin, including the native extension, when a Rust +toolchain is available. Without one, they're built with hatchling and contain only +the pure-Python implementation, which the package also falls back to at runtime. +If a Rust toolchain is available but can't build the extension, for example because +it's too old, the build fails with instructions rather than silently falling back. + +Source distributions are always built with hatchling, since maturin needs cargo to +build them. The contents are the same either way. + +Set ``SMITHY_PYTHON_NATIVE_BUILD`` to control which backend builds wheels: + +* ``auto`` (the default): maturin if ``cargo`` is on the ``PATH``, else hatchling. +* ``always``: maturin, failing if the extension can't be built. +* ``never``: hatchling, producing a ``py3-none-any`` wheel. +""" + +import os +import re +import shutil +import subprocess +import sys +import tomllib +from importlib import import_module +from pathlib import Path + +type _ConfigSettings = dict[str, str | list[str]] | None +type _Version = tuple[int, ...] + +_ENV_VAR = "SMITHY_PYTHON_NATIVE_BUILD" +_MATURIN = "maturin>=1.10,<2" +_HATCHLING = "hatchling" +_CARGO_TOML = Path(__file__).parent.parent / "Cargo.toml" +_PURE_PYTHON_HINT = ( + f"To install the pure-Python implementation instead, set {_ENV_VAR}=never. It " + "produces identical checksums, but CRC32C is much slower." +) + + +def _build_native() -> bool: + mode = os.environ.get(_ENV_VAR, "auto").strip().lower() + match mode: + case "auto": + if shutil.which("cargo") is None: + return False + case "always": + pass + case "never": + return False + case _: + raise ValueError( + f"{_ENV_VAR} must be 'auto', 'always', or 'never', not {mode!r}" + ) + _check_rust_toolchain() + return True + + +def _check_rust_toolchain() -> None: + """Fails early, with instructions, if the Rust toolchain can't build the extension.""" + minimum = _minimum_rust_version() + required = ".".join(map(str, minimum)) + rustc = shutil.which("rustc") + if shutil.which("cargo") is None or rustc is None: + raise RuntimeError( + f"Building the smithy-checksums native extension requires Rust {required} " + f"or newer, but no Rust toolchain was found. Install one from " + f"https://rustup.rs. {_PURE_PYTHON_HINT}" + ) + found = _rustc_version(rustc) + # If the version can't be determined, let the build itself decide. + if found is not None and found < minimum: + raise RuntimeError( + f"Building the smithy-checksums native extension requires Rust {required} " + f"or newer, but found {'.'.join(map(str, found))}. Upgrade Rust, for " + f"example with `rustup update`. {_PURE_PYTHON_HINT}" + ) + + +def _minimum_rust_version() -> _Version: + with _CARGO_TOML.open("rb") as manifest: + rust_version: str = tomllib.load(manifest)["package"]["rust-version"] + return tuple(int(part) for part in rust_version.split(".")) + + +def _rustc_version(rustc: str) -> _Version | None: + try: + result = subprocess.run( # noqa: S603 + [rustc, "--version"], capture_output=True, text=True, timeout=60 + ) + except (OSError, subprocess.SubprocessError): + return None + return _parse_rustc_version(result.stdout) + + +def _parse_rustc_version(output: str) -> _Version | None: + match = re.match(r"rustc (\d+)\.(\d+)\.(\d+)", output) + return None if match is None else tuple(int(part) for part in match.groups()) + + +def _build(hook: str, wheel_directory: str, config_settings: _ConfigSettings) -> str: + if not _build_native(): + backend = import_module("hatchling.build") + return getattr(backend, hook)(wheel_directory, config_settings) + try: + return getattr(import_module("maturin"), hook)(wheel_directory, config_settings) + except (Exception, SystemExit): + # maturin prints the compiler output and exits the process on failure. + sys.stderr.write( + "\nBuilding the smithy-checksums native extension failed. " + f"{_PURE_PYTHON_HINT}\n" + ) + raise + + +def get_requires_for_build_wheel(config_settings: _ConfigSettings = None) -> list[str]: + return [_MATURIN] if _build_native() else [_HATCHLING] + + +def get_requires_for_build_editable( + config_settings: _ConfigSettings = None, +) -> list[str]: + return [_MATURIN] if _build_native() else [_HATCHLING, "editables"] + + +def get_requires_for_build_sdist(config_settings: _ConfigSettings = None) -> list[str]: + return [_HATCHLING] + + +def build_wheel( + wheel_directory: str, + config_settings: _ConfigSettings = None, + metadata_directory: str | None = None, +) -> str: + return _build("build_wheel", wheel_directory, config_settings) + + +def build_editable( + wheel_directory: str, + config_settings: _ConfigSettings = None, + metadata_directory: str | None = None, +) -> str: + return _build("build_editable", wheel_directory, config_settings) + + +def build_sdist(sdist_directory: str, config_settings: _ConfigSettings = None) -> str: + return import_module("hatchling.build").build_sdist( + sdist_directory, config_settings + ) diff --git a/packages/smithy-checksums/deny.toml b/packages/smithy-checksums/deny.toml new file mode 100644 index 000000000..b737758ca --- /dev/null +++ b/packages/smithy-checksums/deny.toml @@ -0,0 +1,24 @@ +# Configuration for cargo-deny, which checks the dependencies of the native extension. + +[advisories] +version = 2 +yanked = "deny" + +[licenses] +version = 2 +allow = [ + "Apache-2.0", + "Apache-2.0 WITH LLVM-exception", + "MIT", + "Unicode-3.0", +] +# Our own crate is covered by the package license. +private = { ignore = true } + +[bans] +multiple-versions = "warn" +wildcards = "deny" + +[sources] +unknown-registry = "deny" +unknown-git = "deny" diff --git a/packages/smithy-checksums/pyproject.toml b/packages/smithy-checksums/pyproject.toml new file mode 100644 index 000000000..91ed1fd03 --- /dev/null +++ b/packages/smithy-checksums/pyproject.toml @@ -0,0 +1,81 @@ +[project] +name = "smithy-checksums" +# The version must stay static: both build backends read it from here. +version = "0.0.0" +requires-python = ">=3.12" +authors = [ + {name = "Amazon Web Services"}, +] +description = "Checksum algorithms for Smithy tooling, with native acceleration." +readme = "README.md" +license = {text = "Apache License 2.0"} +keywords = ["smithy", "sdk", "checksum", "crc32c"] +classifiers = [ + "Development Status :: 2 - Pre-Alpha", + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + "Natural Language :: English", + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Programming Language :: Python :: 3.15", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Free Threading :: 2 - Beta", + "Programming Language :: Rust", + "Topic :: Software Development :: Libraries" +] +dependencies = [] + +[project.urls] +"Changelog" = "https://github.com/smithy-lang/smithy-python/blob/develop/packages/smithy-checksums/CHANGELOG.md" +"Code" = "https://github.com/smithy-lang/smithy-python/blob/develop/packages/smithy-checksums/" +"Issue tracker" = "https://github.com/smithy-lang/smithy-python/issues" + +# Wheels are built with maturin when a Rust toolchain is available and with +# hatchling (pure Python) otherwise. See build_backend/smithy_checksums_build.py. +[build-system] +requires = [] +build-backend = "smithy_checksums_build" +backend-path = ["build_backend"] + +[tool.maturin] +python-source = "src" +module-name = "smithy_checksums._native" +features = ["pyo3/extension-module"] + +[tool.hatch.build.targets.wheel] +packages = ["src/smithy_checksums"] +exclude = ["*.so", "*.pyd"] + +[tool.hatch.build.targets.sdist] +include = [ + "src", + "rust", + "build_backend", + "Cargo.toml", + "Cargo.lock", + "CHANGELOG.md", + "LICENSE-THIRD-PARTY", + "NOTICE", + "README.md", +] +exclude = ["*.so", "*.pyd"] + +[tool.uv] +# Rebuild the native extension in the workspace when the Rust sources change. +cache-keys = [ + { file = "pyproject.toml" }, + { file = "Cargo.toml" }, + { file = "Cargo.lock" }, + { file = "rust/**/*.rs" }, + { file = "build_backend/*.py" }, + { env = "SMITHY_PYTHON_NATIVE_BUILD" }, +] + +[tool.ruff] +src = ["src"] diff --git a/packages/smithy-checksums/rust/lib.rs b/packages/smithy-checksums/rust/lib.rs new file mode 100644 index 000000000..4ed6086c1 --- /dev/null +++ b/packages/smithy-checksums/rust/lib.rs @@ -0,0 +1,135 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Native checksum implementations for `smithy_checksums`. +//! +//! Each class mirrors the pure-Python implementation of the same name in +//! `smithy_checksums._private.crc`, so the two are interchangeable. + +use std::sync::{Mutex, MutexGuard}; + +use pyo3::buffer::PyBuffer; +use pyo3::prelude::*; +use pyo3::types::PyBytes; + +/// Inputs at least this large are hashed with the interpreter detached so +/// other threads can run. Below it, detaching costs more than it saves. This +/// matches the threshold `hashlib` uses. +const DETACH_THRESHOLD: usize = 2048; + +/// Streaming checksum state shared by every class in this module. +trait State: Clone + Send { + fn new() -> Self; + fn update(&mut self, data: &[u8]); + fn digest(&self) -> Vec; +} + +macro_rules! crc_state { + ($state:ident, $algorithm:ident, $size:literal) => { + #[derive(Clone)] + struct $state(crc_fast::Digest); + + impl State for $state { + fn new() -> Self { + Self(crc_fast::Digest::new(crc_fast::CrcAlgorithm::$algorithm)) + } + + fn update(&mut self, data: &[u8]) { + self.0.update(data); + } + + fn digest(&self) -> Vec { + // The digest is a u64 regardless of the CRC's width. + self.0.finalize().to_be_bytes()[8 - $size..].to_vec() + } + } + }; +} + +crc_state!(Crc32State, Crc32IsoHdlc, 4); +crc_state!(Crc32cState, Crc32Iscsi, 4); + +fn lock(state: &Mutex) -> MutexGuard<'_, S> { + // A panic while holding the lock can't leave a checksum half-updated in a + // way that matters more than the panic itself, so ignore poisoning. + state + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) +} + +/// Feeds the bytes of `data` to `state`. +/// +/// `bytes` and other read-only buffers are hashed in place. Writable buffers +/// (such as `bytearray`) are copied first, because another thread could +/// otherwise modify them while they're being read. +fn update(py: Python<'_>, state: &Mutex, data: &Bound<'_, PyAny>) -> PyResult<()> { + if let Ok(bytes) = data.cast::() { + hash_slice(py, state, bytes.as_bytes()); + return Ok(()); + } + + let buffer = PyBuffer::::get(data)?; + if buffer.readonly() && buffer.is_c_contiguous() { + // SAFETY: the buffer is read-only and C-contiguous, and `buffer` + // keeps the exporter alive and its memory pinned until it's dropped + // at the end of this function. + let slice = unsafe { + std::slice::from_raw_parts(buffer.buf_ptr().cast::(), buffer.len_bytes()) + }; + hash_slice(py, state, slice); + } else { + let copy = buffer.to_vec(py)?; + hash_slice(py, state, ©); + } + Ok(()) +} + +fn hash_slice(py: Python<'_>, state: &Mutex, data: &[u8]) { + if data.len() < DETACH_THRESHOLD { + lock(state).update(data); + } else { + py.detach(|| lock(state).update(data)); + } +} + +macro_rules! checksum_class { + ($name:ident, $state:ident) => { + #[pyclass(frozen, module = "smithy_checksums._native")] + struct $name { + state: Mutex<$state>, + } + + #[pymethods] + impl $name { + #[new] + fn new() -> Self { + Self { + state: Mutex::new($state::new()), + } + } + + fn update(&self, py: Python<'_>, data: &Bound<'_, PyAny>) -> PyResult<()> { + update(py, &self.state, data) + } + + fn digest<'py>(&self, py: Python<'py>) -> Bound<'py, PyBytes> { + PyBytes::new(py, &lock(&self.state).digest()) + } + + fn copy(&self) -> Self { + Self { + state: Mutex::new(lock(&self.state).clone()), + } + } + } + }; +} + +checksum_class!(Crc32, Crc32State); +checksum_class!(Crc32C, Crc32cState); + +#[pymodule] +mod _native { + #[pymodule_export] + use super::{Crc32, Crc32C}; +} diff --git a/packages/smithy-checksums/src/smithy_checksums/__init__.py b/packages/smithy-checksums/src/smithy_checksums/__init__.py new file mode 100644 index 000000000..d865b9e9d --- /dev/null +++ b/packages/smithy-checksums/src/smithy_checksums/__init__.py @@ -0,0 +1,144 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Checksum algorithms for Smithy clients. + +CRC32 and CRC32C use a native extension when one is available for the platform +and fall back to pure Python otherwise. The fallback produces identical results +but is orders of magnitude slower, except for CRC32, which uses :mod:`zlib`. SHA-1, SHA-256, SHA-512, and MD5 are available +through :func:`new` and always use :mod:`hashlib`. +""" + +import hashlib +from collections.abc import Buffer +from importlib.metadata import version +from typing import ClassVar, Final, Protocol, Self + +from ._private.backend import IMPLEMENTATIONS as _IMPLEMENTATIONS +from ._private.backend import NATIVE as _NATIVE +from ._private.backend import ChecksumImpl as _ChecksumImpl +from ._private.buffers import as_byte_view as _as_byte_view + +__version__ = version("smithy-checksums") +__all__ = ( + "CRC32", + "CRC32C", + "HAS_NATIVE_EXTENSION", + "Checksum", + "new", +) + +HAS_NATIVE_EXTENSION: Final[bool] = _NATIVE +"""Whether the native extension is in use. + +When it isn't, CRC32C is computed in pure Python. +""" + + +class Checksum(Protocol): + """A streaming checksum. + + Objects returned by :mod:`hashlib` also satisfy this protocol. + """ + + @property + def name(self) -> str: + """The lowercase name of the algorithm, such as ``crc32c``.""" + ... + + @property + def digest_size(self) -> int: + """The size of the digest in bytes.""" + ... + + def update(self, data: Buffer, /) -> None: + """Feeds bytes to the checksum. + + :param data: The bytes to add. Buffers that aren't C-contiguous are rejected. + """ + ... + + def digest(self) -> bytes: + """Returns the checksum of the data passed to :meth:`update` so far. + + The digest is big-endian, the byte order used by checksum headers such as + ``x-amz-checksum-crc32c``. Calling this doesn't reset the checksum. + """ + ... + + def copy(self) -> Self: + """Returns an independent copy of the checksum's current state.""" + ... + + +class _BackendChecksum: + __slots__ = ("_impl",) + + name: ClassVar[str] + digest_size: ClassVar[int] + + _impl: _ChecksumImpl + + def __init__(self) -> None: + self._impl = _IMPLEMENTATIONS[self.name]() + + def update(self, data: Buffer, /) -> None: + """Feeds bytes to the checksum. + + :param data: The bytes to add. Buffers that aren't C-contiguous are rejected. + """ + self._impl.update(_as_byte_view(data)) + + def digest(self) -> bytes: + """Returns the big-endian checksum of the data passed to :meth:`update` so far. + + Calling this doesn't reset the checksum. + """ + return self._impl.digest() + + def copy(self) -> Self: + """Returns an independent copy of the checksum's current state.""" + new = object.__new__(type(self)) + new._impl = self._impl.copy() + return new + + +class CRC32(_BackendChecksum): + """CRC-32 (ISO-HDLC), the variant used by zlib and gzip.""" + + __slots__ = () + + name = "crc32" + digest_size = 4 + + +class CRC32C(_BackendChecksum): + """CRC-32C (Castagnoli).""" + + __slots__ = () + + name = "crc32c" + digest_size = 4 + + +_CHECKSUMS: Final = {cls.name: cls for cls in (CRC32, CRC32C)} +_HASHLIB_ALGORITHMS: Final = frozenset(("sha1", "sha256", "sha512")) + + +def new(name: str) -> Checksum: + """Creates a checksum from the name of its algorithm. + + Names are case-insensitive, so the values of a checksum algorithm enum such as + ``CRC32C`` or ``SHA256`` can be passed directly. + + :param name: One of ``crc32``, ``crc32c``, ``sha1``, ``sha256``, ``sha512``, or + ``md5``. + :raises ValueError: If the algorithm isn't supported. + """ + key = name.lower() + if (checksum := _CHECKSUMS.get(key)) is not None: + return checksum() + if key in _HASHLIB_ALGORITHMS: + return hashlib.new(key) + if key == "md5": + return hashlib.md5(usedforsecurity=False) + raise ValueError(f"Unsupported checksum algorithm: {name!r}") diff --git a/packages/smithy-checksums/src/smithy_checksums/_native.pyi b/packages/smithy-checksums/src/smithy_checksums/_native.pyi new file mode 100644 index 000000000..1163b9bb9 --- /dev/null +++ b/packages/smithy-checksums/src/smithy_checksums/_native.pyi @@ -0,0 +1,17 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Type stubs for the native extension built from ``rust/lib.rs``.""" + +from typing import Self + +type _Bytes = bytes | bytearray | memoryview + +class Crc32: + def update(self, data: _Bytes, /) -> None: ... + def digest(self) -> bytes: ... + def copy(self) -> Self: ... + +class Crc32C: + def update(self, data: _Bytes, /) -> None: ... + def digest(self) -> bytes: ... + def copy(self) -> Self: ... diff --git a/packages/smithy-checksums/src/smithy_checksums/_private/__init__.py b/packages/smithy-checksums/src/smithy_checksums/_private/__init__.py new file mode 100644 index 000000000..33cbe867a --- /dev/null +++ b/packages/smithy-checksums/src/smithy_checksums/_private/__init__.py @@ -0,0 +1,2 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/packages/smithy-checksums/src/smithy_checksums/_private/backend.py b/packages/smithy-checksums/src/smithy_checksums/_private/backend.py new file mode 100644 index 000000000..c542afd87 --- /dev/null +++ b/packages/smithy-checksums/src/smithy_checksums/_private/backend.py @@ -0,0 +1,42 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Selects the native checksum implementations, or the pure-Python ones if unavailable.""" + +from collections.abc import Callable, Mapping +from typing import Protocol, Self + + +class ChecksumImpl(Protocol): + """The interface shared by the native and pure-Python implementations. + + Callers normalize input with :func:`.buffers.as_byte_view` first. + """ + + def update(self, data: bytes | bytearray | memoryview, /) -> None: ... + + def digest(self) -> bytes: ... + + def copy(self) -> Self: ... + + +type ImplFactory = Callable[[], ChecksumImpl] + + +def _load() -> tuple[bool, Mapping[str, ImplFactory]]: + try: + from smithy_checksums import _native + except ImportError: + from smithy_checksums._private import crc + + return False, { + "crc32": crc.Crc32, + "crc32c": crc.Crc32C, + } + + return True, { + "crc32": _native.Crc32, + "crc32c": _native.Crc32C, + } + + +NATIVE, IMPLEMENTATIONS = _load() diff --git a/packages/smithy-checksums/src/smithy_checksums/_private/buffers.py b/packages/smithy-checksums/src/smithy_checksums/_private/buffers.py new file mode 100644 index 000000000..ee21e54be --- /dev/null +++ b/packages/smithy-checksums/src/smithy_checksums/_private/buffers.py @@ -0,0 +1,23 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +from collections.abc import Buffer + + +def as_bytes(data: Buffer) -> bytes | bytearray: + """Returns the bytes of ``data``, copying only if it isn't already bytes-like.""" + if isinstance(data, bytes | bytearray): + return data + return memoryview(data).tobytes() + + +def as_byte_view(data: Buffer) -> bytes | bytearray | memoryview: + """Returns ``data`` as an object the native extension accepts, without copying. + + The native extension reads any buffer with a byte format. Other formats (such + as an ``array('I')``) are recast to bytes, which requires a C-contiguous buffer, + the same requirement :mod:`hashlib` has. + """ + if isinstance(data, bytes | bytearray): + return data + view = memoryview(data) + return view if view.format == "B" else view.cast("B") diff --git a/packages/smithy-checksums/src/smithy_checksums/_private/crc.py b/packages/smithy-checksums/src/smithy_checksums/_private/crc.py new file mode 100644 index 000000000..92e5f3682 --- /dev/null +++ b/packages/smithy-checksums/src/smithy_checksums/_private/crc.py @@ -0,0 +1,78 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Pure-Python CRC implementations, used when the native extension is unavailable.""" + +import zlib +from typing import ClassVar, Self + +from .pure import PureChecksum + + +def _reflected_table(poly: int) -> tuple[int, ...]: + table: list[int] = [] + for byte in range(256): + crc = byte + for _ in range(8): + crc = (crc >> 1) ^ poly if crc & 1 else crc >> 1 + table.append(crc) + return tuple(table) + + +class Crc32(PureChecksum): + """CRC-32 (ISO-HDLC), backed by :func:`zlib.crc32`.""" + + __slots__ = ("_crc",) + + def __init__(self) -> None: + super().__init__() + self._crc = 0 + + def _update(self, data: bytes | bytearray) -> None: + self._crc = zlib.crc32(data, self._crc) + + def _digest(self) -> bytes: + return self._crc.to_bytes(4, "big") + + def _copy(self) -> Self: + new = type(self)() + new._crc = self._crc + return new + + +class _ReflectedCrc(PureChecksum): + """A table-driven CRC whose initial value and final XOR are all ones.""" + + __slots__ = ("_crc",) + + _table: ClassVar[tuple[int, ...]] + _mask: ClassVar[int] + _size: ClassVar[int] + + def __init__(self) -> None: + super().__init__() + self._crc = self._mask + + def _update(self, data: bytes | bytearray) -> None: + table = self._table + crc = self._crc + for byte in data: + crc = table[(crc ^ byte) & 0xFF] ^ (crc >> 8) + self._crc = crc + + def _digest(self) -> bytes: + return (self._crc ^ self._mask).to_bytes(self._size, "big") + + def _copy(self) -> Self: + new = type(self)() + new._crc = self._crc + return new + + +class Crc32C(_ReflectedCrc): + """CRC-32C (Castagnoli, also called iSCSI).""" + + __slots__ = () + + _table = _reflected_table(0x82F63B78) + _mask = 0xFFFFFFFF + _size = 4 diff --git a/packages/smithy-checksums/src/smithy_checksums/_private/pure.py b/packages/smithy-checksums/src/smithy_checksums/_private/pure.py new file mode 100644 index 000000000..e42bac3dd --- /dev/null +++ b/packages/smithy-checksums/src/smithy_checksums/_private/pure.py @@ -0,0 +1,42 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +from collections.abc import Buffer +from threading import Lock +from typing import Self + +from .buffers import as_bytes + + +class PureChecksum: + """Base class for the pure-Python implementations. + + Each method holds a lock, so an instance can be shared between threads like the + native implementations and :mod:`hashlib` objects can. Subclasses implement the + underscored methods, which are only called with the lock held. + """ + + __slots__ = ("_lock",) + + def __init__(self) -> None: + self._lock = Lock() + + def update(self, data: Buffer, /) -> None: + with self._lock: + self._update(as_bytes(data)) + + def digest(self) -> bytes: + with self._lock: + return self._digest() + + def copy(self) -> Self: + with self._lock: + return self._copy() + + def _update(self, data: bytes | bytearray) -> None: + raise NotImplementedError + + def _digest(self) -> bytes: + raise NotImplementedError + + def _copy(self) -> Self: + raise NotImplementedError diff --git a/packages/smithy-checksums/src/smithy_checksums/py.typed b/packages/smithy-checksums/src/smithy_checksums/py.typed new file mode 100644 index 000000000..f5642f79f --- /dev/null +++ b/packages/smithy-checksums/src/smithy_checksums/py.typed @@ -0,0 +1 @@ +Marker diff --git a/packages/smithy-checksums/tests/__init__.py b/packages/smithy-checksums/tests/__init__.py new file mode 100644 index 000000000..04f8b7b76 --- /dev/null +++ b/packages/smithy-checksums/tests/__init__.py @@ -0,0 +1,2 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/packages/smithy-checksums/tests/unit/__init__.py b/packages/smithy-checksums/tests/unit/__init__.py new file mode 100644 index 000000000..04f8b7b76 --- /dev/null +++ b/packages/smithy-checksums/tests/unit/__init__.py @@ -0,0 +1,2 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/packages/smithy-checksums/tests/unit/test_build_backend.py b/packages/smithy-checksums/tests/unit/test_build_backend.py new file mode 100644 index 000000000..85c6b9c7d --- /dev/null +++ b/packages/smithy-checksums/tests/unit/test_build_backend.py @@ -0,0 +1,162 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +import importlib.util +import shutil +from pathlib import Path +from types import ModuleType, SimpleNamespace + +import pytest + +_BACKEND_PATH = ( + Path(__file__).parents[2] / "build_backend" / "smithy_checksums_build.py" +) +_ENV_VAR = "SMITHY_PYTHON_NATIVE_BUILD" + + +@pytest.fixture +def backend() -> ModuleType: + spec = importlib.util.spec_from_file_location( + "smithy_checksums_build", _BACKEND_PATH + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _set_toolchain( + monkeypatch: pytest.MonkeyPatch, + backend: ModuleType, + *, + installed: bool = True, + version: tuple[int, ...] | None = (1, 93, 0), +) -> None: + tools = {"cargo": "/bin/cargo", "rustc": "/bin/rustc"} if installed else {} + monkeypatch.setattr(shutil, "which", tools.get) + monkeypatch.setattr(backend, "_rustc_version", lambda _: version) # type: ignore + + +def _set_mode(monkeypatch: pytest.MonkeyPatch, mode: str | None) -> None: + if mode is None: + monkeypatch.delenv(_ENV_VAR, raising=False) + else: + monkeypatch.setenv(_ENV_VAR, mode) + + +def _requires_native(backend: ModuleType) -> bool: + requires = backend.get_requires_for_build_wheel() + native = any(req.startswith("maturin") for req in requires) + assert ("hatchling" in requires) is not native + + editable_requires = backend.get_requires_for_build_editable() + assert ("editables" in editable_requires) is not native + return native + + +@pytest.mark.parametrize( + ("mode", "installed", "native"), + [ + (None, True, True), + (None, False, False), + ("auto", True, True), + ("AUTO", False, False), + ("always", True, True), + ("never", True, False), + ("never", False, False), + ], +) +def test_selects_backend( + backend: ModuleType, + monkeypatch: pytest.MonkeyPatch, + mode: str | None, + installed: bool, + native: bool, +) -> None: + _set_mode(monkeypatch, mode) + _set_toolchain(monkeypatch, backend, installed=installed) + assert _requires_native(backend) is native + + +@pytest.mark.parametrize("mode", [None, "never", "always"]) +def test_sdist_never_needs_rust( + backend: ModuleType, monkeypatch: pytest.MonkeyPatch, mode: str | None +) -> None: + _set_mode(monkeypatch, mode) + _set_toolchain(monkeypatch, backend, installed=False) + assert backend.get_requires_for_build_sdist() == ["hatchling"] + + +@pytest.mark.parametrize("mode", [None, "always"]) +def test_rejects_old_rust( + backend: ModuleType, monkeypatch: pytest.MonkeyPatch, mode: str | None +) -> None: + _set_mode(monkeypatch, mode) + _set_toolchain(monkeypatch, backend, version=(1, 70, 0)) + with pytest.raises(RuntimeError, match=r"requires Rust 1\.89.*found 1\.70\.0") as e: + backend.get_requires_for_build_wheel() + assert f"{_ENV_VAR}=never" in str(e.value) + + +def test_always_requires_rust( + backend: ModuleType, monkeypatch: pytest.MonkeyPatch +) -> None: + _set_mode(monkeypatch, "always") + _set_toolchain(monkeypatch, backend, installed=False) + with pytest.raises(RuntimeError, match="no Rust toolchain was found") as e: + backend.get_requires_for_build_wheel() + assert f"{_ENV_VAR}=never" in str(e.value) + + +def test_unknown_rust_version_builds_natively( + backend: ModuleType, monkeypatch: pytest.MonkeyPatch +) -> None: + _set_mode(monkeypatch, None) + _set_toolchain(monkeypatch, backend, version=None) + assert _requires_native(backend) + + +def test_failed_native_build_suggests_pure_python( + backend: ModuleType, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + def fail(wheel_directory: str, config_settings: object) -> str: + raise SystemExit(1) + + _set_mode(monkeypatch, None) + _set_toolchain(monkeypatch, backend) + monkeypatch.setattr( + backend, + "import_module", + lambda _: SimpleNamespace(build_wheel=fail), # type: ignore + ) + with pytest.raises(SystemExit): + backend.build_wheel("dist") + assert f"{_ENV_VAR}=never" in capsys.readouterr().err + + +def test_rejects_unknown_mode( + backend: ModuleType, monkeypatch: pytest.MonkeyPatch +) -> None: + _set_mode(monkeypatch, "sometimes") + with pytest.raises(ValueError, match="sometimes"): + backend.get_requires_for_build_wheel() + + +def test_minimum_rust_version_matches_cargo_toml(backend: ModuleType) -> None: + assert backend._minimum_rust_version() == (1, 89) + + +@pytest.mark.parametrize( + ("output", "version"), + [ + ("rustc 1.93.0 (254b59607 2026-01-19)\n", (1, 93, 0)), + ("rustc 1.95.0-nightly (0123abcde 2026-02-01)\n", (1, 95, 0)), + ("error: no default toolchain configured\n", None), + ("", None), + ], +) +def test_parses_rustc_version( + backend: ModuleType, output: str, version: tuple[int, ...] | None +) -> None: + assert backend._parse_rustc_version(output) == version diff --git a/packages/smithy-checksums/tests/unit/test_checksums.py b/packages/smithy-checksums/tests/unit/test_checksums.py new file mode 100644 index 000000000..567ab4672 --- /dev/null +++ b/packages/smithy-checksums/tests/unit/test_checksums.py @@ -0,0 +1,114 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +import array +import hashlib +import importlib.util +from concurrent.futures import ThreadPoolExecutor + +import pytest +import smithy_checksums +from smithy_checksums import ( + CRC32, + CRC32C, + Checksum, + new, +) + +from .vectors import ALGORITHMS, KNOWN_ANSWERS, PATTERN_DIGESTS, pattern + +CLASSES = { + "crc32": CRC32, + "crc32c": CRC32C, +} + + +@pytest.mark.parametrize("algorithm", ALGORITHMS) +def test_class_attributes(algorithm: str) -> None: + checksum = CLASSES[algorithm]() + assert checksum.name == algorithm + assert checksum.digest_size == len(checksum.digest()) + + +@pytest.mark.parametrize("algorithm", ALGORITHMS) +def test_known_answers(algorithm: str) -> None: + for data, expected in KNOWN_ANSWERS[algorithm].items(): + checksum = CLASSES[algorithm]() + checksum.update(data) + assert checksum.digest().hex() == expected + + +@pytest.mark.parametrize("algorithm", ALGORITHMS) +def test_copy_returns_same_type(algorithm: str) -> None: + checksum = CLASSES[algorithm]() + checksum.update(b"Hello ") + copied = checksum.copy() + copied.update(b"world") + assert type(copied) is type(checksum) + assert copied.digest().hex() == KNOWN_ANSWERS[algorithm][b"Hello world"] + assert checksum.digest().hex() != copied.digest().hex() + + +@pytest.mark.parametrize("algorithm", ALGORITHMS) +def test_accepts_non_byte_formats(algorithm: str) -> None: + data = array.array("I", range(1024)) + checksum = CLASSES[algorithm]() + checksum.update(data) + expected = CLASSES[algorithm]() + expected.update(data.tobytes()) + assert checksum.digest() == expected.digest() + + +@pytest.mark.parametrize("algorithm", ALGORITHMS) +def test_rejects_non_contiguous_buffers(algorithm: str) -> None: + with pytest.raises(TypeError): + CLASSES[algorithm]().update(memoryview(array.array("I", range(8)))[::2]) + + +@pytest.mark.parametrize("algorithm", ALGORITHMS) +def test_rejects_non_buffers(algorithm: str) -> None: + with pytest.raises(TypeError): + CLASSES[algorithm]().update("text") # type: ignore[arg-type] + + +@pytest.mark.parametrize("algorithm", ALGORITHMS) +def test_concurrent_checksums(algorithm: str) -> None: + data = pattern(16385) + + def compute(_: int) -> str: + checksum = CLASSES[algorithm]() + for offset in range(0, len(data), 4096): + checksum.update(data[offset : offset + 4096]) + return checksum.digest().hex() + + with ThreadPoolExecutor(max_workers=4) as executor: + results = set(executor.map(compute, range(8))) + assert results == {PATTERN_DIGESTS[algorithm][16385]} + + +@pytest.mark.parametrize( + ("name", "expected"), + [ + ("crc32", CRC32), + ("CRC32C", CRC32C), + ], +) +def test_new_returns_checksum_classes(name: str, expected: type[Checksum]) -> None: + assert type(new(name)) is expected + + +@pytest.mark.parametrize("name", ["SHA1", "sha256", "SHA512", "MD5"]) +def test_new_returns_hashlib_checksums(name: str) -> None: + checksum = new(name) + checksum.update(b"Hello world") + assert checksum.name == name.lower() + assert checksum.digest() == hashlib.new(name.lower(), b"Hello world").digest() + + +def test_new_rejects_unknown_algorithms() -> None: + with pytest.raises(ValueError, match="crc16"): + new("crc16") + + +def test_has_native_extension() -> None: + built = importlib.util.find_spec("smithy_checksums._native") is not None + assert smithy_checksums.HAS_NATIVE_EXTENSION is built diff --git a/packages/smithy-checksums/tests/unit/test_implementations.py b/packages/smithy-checksums/tests/unit/test_implementations.py new file mode 100644 index 000000000..a50b39184 --- /dev/null +++ b/packages/smithy-checksums/tests/unit/test_implementations.py @@ -0,0 +1,182 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Tests that run against both the native and pure-Python implementations. + +The native cases are skipped when the extension isn't built, but the pure-Python +cases always run, regardless of which implementation the package selected. +""" + +import os +import random +from collections.abc import Callable +from concurrent.futures import ThreadPoolExecutor +from types import ModuleType + +import pytest +from smithy_checksums._private import crc +from smithy_checksums._private.backend import ChecksumImpl + +from .vectors import ALGORITHMS, KNOWN_ANSWERS, PATTERN_DIGESTS, pattern + +type Factory = Callable[[], ChecksumImpl] + +PURE: dict[str, Factory] = { + "crc32": crc.Crc32, + "crc32c": crc.Crc32C, +} + + +def _load_native() -> ModuleType | None: + try: + from smithy_checksums import _native + except ImportError: + return None + return _native + + +_NATIVE_MODULE = _load_native() +NATIVE: dict[str, Factory] = ( + {} + if _NATIVE_MODULE is None + else { + "crc32": _NATIVE_MODULE.Crc32, + "crc32c": _NATIVE_MODULE.Crc32C, + } +) +requires_native = pytest.mark.skipif( + _NATIVE_MODULE is None, reason="native extension not built" +) + + +IMPLEMENTATIONS = [ + pytest.param(PURE, id="pure"), + pytest.param(NATIVE, id="native", marks=requires_native), +] + + +def _digest(factory: Factory, *chunks: bytes) -> str: + checksum = factory() + for chunk in chunks: + checksum.update(chunk) + return checksum.digest().hex() + + +@pytest.mark.parametrize("impls", IMPLEMENTATIONS) +@pytest.mark.parametrize("algorithm", ALGORITHMS) +def test_known_answers(impls: dict[str, Factory], algorithm: str) -> None: + for data, expected in KNOWN_ANSWERS[algorithm].items(): + assert _digest(impls[algorithm], data) == expected + + +@pytest.mark.parametrize("impls", IMPLEMENTATIONS) +@pytest.mark.parametrize("algorithm", ALGORITHMS) +def test_pattern_digests(impls: dict[str, Factory], algorithm: str) -> None: + for length, expected in PATTERN_DIGESTS[algorithm].items(): + assert _digest(impls[algorithm], pattern(length)) == expected, length + + +@pytest.mark.parametrize("impls", IMPLEMENTATIONS) +@pytest.mark.parametrize("algorithm", ALGORITHMS) +def test_streaming_matches_one_shot(impls: dict[str, Factory], algorithm: str) -> None: + data = pattern(16385) + expected = PATTERN_DIGESTS[algorithm][16385] + rng = random.Random(algorithm) + for _ in range(5): + chunks: list[bytes] = [] + offset = 0 + while offset < len(data): + size = rng.choice((1, 7, 63, 64, 65, 239, 240, 241, 1023, 1024, 4096)) + chunks.append(data[offset : offset + size]) + offset += size + assert _digest(impls[algorithm], *chunks) == expected + + +@pytest.mark.parametrize("impls", IMPLEMENTATIONS) +@pytest.mark.parametrize("algorithm", ALGORITHMS) +def test_copy_is_independent(impls: dict[str, Factory], algorithm: str) -> None: + data = pattern(2049) + original = impls[algorithm]() + original.update(data[:1000]) + copied = original.copy() + copied.update(data[1000:]) + assert copied.digest().hex() == _digest(impls[algorithm], data) + assert original.digest().hex() == _digest(impls[algorithm], data[:1000]) + + +@pytest.mark.parametrize("impls", IMPLEMENTATIONS) +@pytest.mark.parametrize("algorithm", ALGORITHMS) +def test_digest_does_not_reset(impls: dict[str, Factory], algorithm: str) -> None: + checksum = impls[algorithm]() + checksum.update(b"Hello ") + first = checksum.digest() + assert checksum.digest() == first + checksum.update(b"world") + assert checksum.digest().hex() == KNOWN_ANSWERS[algorithm][b"Hello world"] + + +@pytest.mark.parametrize("impls", IMPLEMENTATIONS) +@pytest.mark.parametrize("algorithm", ALGORITHMS) +@pytest.mark.parametrize( + "wrap", + [bytes, bytearray, memoryview], + ids=["bytes", "bytearray", "memoryview"], +) +def test_accepts_byte_buffers( + impls: dict[str, Factory], + algorithm: str, + wrap: Callable[[bytes], bytes | bytearray | memoryview], +) -> None: + # 4096 bytes is above the size at which the native extension detaches from + # the interpreter while hashing. + data = pattern(4096) + checksum = impls[algorithm]() + checksum.update(wrap(data)) + assert checksum.digest().hex() == _digest(impls[algorithm], data) + + +@pytest.mark.parametrize("impls", IMPLEMENTATIONS) +@pytest.mark.parametrize("algorithm", ALGORITHMS) +def test_shared_instance_updates_are_atomic( + impls: dict[str, Factory], algorithm: str +) -> None: + # Every update adds the same chunk, so the result doesn't depend on the order + # the threads run in, only on each update being applied whole. + chunk = pattern(4096) + threads, updates = 4, 8 + shared = impls[algorithm]() + + def run(_: int) -> None: + for _ in range(updates): + shared.update(chunk) + + with ThreadPoolExecutor(max_workers=threads) as executor: + list(executor.map(run, range(threads))) + assert shared.digest().hex() == _digest( + impls[algorithm], chunk * (threads * updates) + ) + + +@requires_native +@pytest.mark.parametrize("algorithm", ALGORITHMS) +def test_native_matches_pure(algorithm: str) -> None: + data = os.urandom(5000) + for length in [*range(300), 1023, 1024, 1025, 2047, 2048, 2049, 5000]: + assert _digest(NATIVE[algorithm], data[:length]) == _digest( + PURE[algorithm], data[:length] + ), length + + +@pytest.mark.parametrize("impls", IMPLEMENTATIONS) +@pytest.mark.parametrize("algorithm", ALGORITHMS) +def test_matches_crt(impls: dict[str, Factory], algorithm: str) -> None: + crt = pytest.importorskip("awscrt.checksums") + compute: dict[str, Callable[[bytes], bytes]] = { + "crc32": lambda data: crt.crc32(data).to_bytes(4, "big"), + "crc32c": lambda data: crt.crc32c(data).to_bytes(4, "big"), + } + data = os.urandom(3000) + for length in [*range(0, 300, 7), 1023, 1024, 1025, 3000]: + assert ( + _digest(impls[algorithm], data[:length]) + == compute[algorithm](data[:length]).hex() + ), length diff --git a/packages/smithy-checksums/tests/unit/vectors.py b/packages/smithy-checksums/tests/unit/vectors.py new file mode 100644 index 000000000..7697f57ce --- /dev/null +++ b/packages/smithy-checksums/tests/unit/vectors.py @@ -0,0 +1,67 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Expected digests shared by the checksum tests.""" + +ALGORITHMS = ("crc32", "crc32c") + +# Digests from independent sources: the standard check value of each CRC (the +# digest of "123456789"), the CRC32C test vectors from RFC 3720, and "Hello world" +# digests that match the ones the CRT produces. +KNOWN_ANSWERS: dict[str, dict[bytes, str]] = { + "crc32": {b"123456789": "cbf43926", b"Hello world": "8bd69e52"}, + "crc32c": { + b"123456789": "e3069283", + b"Hello world": "72b51f78", + # RFC 3720, appendix B.4. The RFC lists each CRC in transmission order, + # least significant byte first, so these are its values byte-reversed. + bytes(32): "8a9136aa", + b"\xff" * 32: "62a8ab43", + bytes(range(32)): "46dd794e", + bytes(range(31, -1, -1)): "113fdb5c", + }, +} + + +def pattern(length: int) -> bytes: + """Returns deterministic input that doesn't repeat on a power-of-two boundary.""" + return bytes(i % 251 for i in range(length)) + + +# Digests of pattern(length), from inputs shorter than a word to several +# kilobytes. +PATTERN_DIGESTS: dict[str, dict[int, str]] = { + "crc32": { + 0: "00000000", + 1: "d202ef8d", + 3: "0854897f", + 4: "8bb98613", + 8: "88aa689f", + 9: "bce14302", + 16: "cecee288", + 17: "2c183a19", + 128: "24650d57", + 129: "ca91cdf7", + 240: "a60b0b66", + 241: "cbc8d2f7", + 1024: "7be4dfd0", + 1025: "4e700dfb", + 16385: "e033bd2c", + }, + "crc32c": { + 0: "00000000", + 1: "527d5351", + 3: "92fd4bfa", + 4: "d9331aa3", + 8: "8a2cbc3b", + 9: "7144c5a8", + 16: "d9c908eb", + 17: "38435e17", + 128: "30d9c515", + 129: "f514629f", + 240: "9f4f71d6", + 241: "54fe7516", + 1024: "2af62c0c", + 1025: "c8d03add", + 16385: "896b2de8", + }, +} diff --git a/pyproject.toml b/pyproject.toml index 8c2cbe851..2ef6c6f0d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,6 +41,7 @@ smithy_xml = { workspace = true } smithy_aws_core = { workspace = true } smithy_aws_event_stream = { workspace = true } aws_sdk_signers = {workspace = true } +smithy_checksums = { workspace = true } smithy_python = { workspace = true } [tool.pyright] diff --git a/scripts/api-snapshot/api.json b/scripts/api-snapshot/api.json index 5382d5c2c..ca3a56342 100644 --- a/scripts/api-snapshot/api.json +++ b/scripts/api-snapshot/api.json @@ -677,6 +677,13 @@ "smithy_cbor.settings": { "CBORSettings": "class" }, + "smithy_checksums": { + "CRC32": "class", + "CRC32C": "class", + "Checksum": "protocol", + "HAS_NATIVE_EXTENSION": "bool", + "new": "function" + }, "smithy_core": { "HostType": "class", "SmithyError": "rex-exception", diff --git a/uv.lock b/uv.lock index f07a64d38..30d7b4359 100644 --- a/uv.lock +++ b/uv.lock @@ -8,6 +8,7 @@ members = [ "smithy-aws-core", "smithy-aws-event-stream", "smithy-cbor", + "smithy-checksums", "smithy-core", "smithy-http", "smithy-json", @@ -715,6 +716,11 @@ dependencies = [ [package.metadata] requires-dist = [{ name = "smithy-core", editable = "packages/smithy-core" }] +[[package]] +name = "smithy-checksums" +version = "0.0.0" +source = { editable = "packages/smithy-checksums" } + [[package]] name = "smithy-core" source = { editable = "packages/smithy-core" } From 4f9aa0e3b824379775f147629a538271db4d412e Mon Sep 17 00:00:00 2001 From: jonathan343 Date: Sat, 24 Oct 2026 23:48:40 -0400 Subject: [PATCH 2/9] Pin the Rust toolchain and cache Rust builds in CI Pins the toolchain used for development and CI in rust-toolchain.toml, so a new stable release can't introduce clippy lints that fail CI without a change here. Builds from a source distribution are unaffected and use the installed toolchain. CI now caches Rust builds with Swatinem/rust-cache, saving caches only on develop, and adds a job that checks each native extension builds with the minimum Rust version its Cargo.toml declares. The new jobs also get names and version comments on their pinned actions. --- .github/workflows/ci.yml | 60 ++++++++++++++++++++++++++++++++++------ rust-toolchain.toml | 7 +++++ 2 files changed, 58 insertions(+), 9 deletions(-) create mode 100644 rust-toolchain.toml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 505add17c..e073d4499 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,9 +53,17 @@ jobs: version: 3.8.2 - name: Set up Rust + # Installs the toolchain pinned in rust-toolchain.toml. run: | - rustup toolchain install stable --profile minimal --component clippy,rustfmt - rustup default stable + rustup toolchain install + + - name: Cache Rust builds + uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + with: + workspaces: packages/smithy-checksums + key: ${{ matrix.python-version }} + # Pull requests restore the caches develop saves, without adding their own. + save-if: ${{ github.ref == 'refs/heads/develop' }} - name: Setup workspace run: | @@ -68,7 +76,7 @@ jobs: - name: Install Rust dependency tools if: ${{ matrix.python-version == '3.12' }} - uses: taiki-e/install-action@1e29709c0ec63babbe8f266397a465f29a825f91 + uses: taiki-e/install-action@1e29709c0ec63babbe8f266397a465f29a825f91 # v2.87.31 with: tool: cargo-about@0.9.2,cargo-deny@0.20.2 @@ -112,6 +120,7 @@ jobs: # installed wheel. The release pipeline builds the published wheels; this # catches platform-specific build or test failures before then. native-wheels: + name: Native wheels (${{ matrix.package }}, ${{ matrix.os }}, Python ${{ matrix.python-version }}) runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -130,21 +139,29 @@ jobs: steps: - name: Checkout Repository - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - name: Install uv and set the Python version - uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 + uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 with: python-version: ${{ matrix.python-version }} activate-environment: true enable-cache: true - name: Set up Rust + # Installs the toolchain pinned in rust-toolchain.toml. run: | - rustup toolchain install stable --profile minimal - rustup default stable + rustup toolchain install + + - name: Cache Rust builds + uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + with: + workspaces: packages/${{ matrix.package }} + key: ${{ matrix.python-version }} + # Pull requests restore the caches develop saves, without adding their own. + save-if: ${{ github.ref == 'refs/heads/develop' }} - name: Build wheel run: | @@ -159,6 +176,7 @@ jobs: # Tests the pure-Python wheel, and installing from the sdist without a Rust # toolchain, which should fall back to the same pure-Python build. pure-python-fallback: + name: Pure-Python fallback (${{ matrix.package }}) runs-on: ubuntu-latest strategy: fail-fast: false @@ -169,12 +187,12 @@ jobs: steps: - name: Checkout Repository - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - name: Install uv and set the Python version - uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 + uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 with: python-version: "3.12" activate-environment: true @@ -200,3 +218,27 @@ jobs: uv pip install --python .venv-sdist --no-cache pytest awscrt dist/*.tar.gz .venv-sdist/bin/python -c "import smithy_checksums; assert not smithy_checksums.HAS_NATIVE_EXTENSION" .venv-sdist/bin/python -m pytest "packages/$PACKAGE/tests" --import-mode=importlib -o addopts="" + + # Checks each native extension builds with the minimum Rust version it declares, + # which source builds require. + rust-minimum-version: + name: Rust minimum version (${{ matrix.package }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + package: [smithy-checksums] + env: + PACKAGE: ${{ matrix.package }} + + steps: + - name: Checkout Repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Check with the minimum Rust version + run: | + version="$(sed -n 's/^rust-version = "\(.*\)"$/\1/p' "packages/$PACKAGE/Cargo.toml")" + rustup toolchain install "$version" --profile minimal + cargo "+$version" check --manifest-path "packages/$PACKAGE/Cargo.toml" --locked --all-targets diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 000000000..1bbb21e89 --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,7 @@ +# The Rust toolchain used to develop and test the native extensions in packages/. +# Builds from a source distribution use the installed toolchain instead, as long as +# it meets the package's rust-version. +[toolchain] +channel = "1.99.1" +components = ["clippy", "rustfmt"] +profile = "minimal" From f585c4f193fd3ddd4f7404af60e17b9b92c9f882 Mon Sep 17 00:00:00 2001 From: jonathan343 Date: Sat, 24 Oct 2026 23:59:15 -0400 Subject: [PATCH 3/9] Describe what smithy-checksums adds in its changelog entry --- ...ithy-checksums-feature-2c585fb742b2409690bb374cf08cd6cf.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/smithy-checksums/.changes/next-release/smithy-checksums-feature-2c585fb742b2409690bb374cf08cd6cf.json b/packages/smithy-checksums/.changes/next-release/smithy-checksums-feature-2c585fb742b2409690bb374cf08cd6cf.json index 6375c1caf..4a267588c 100644 --- a/packages/smithy-checksums/.changes/next-release/smithy-checksums-feature-2c585fb742b2409690bb374cf08cd6cf.json +++ b/packages/smithy-checksums/.changes/next-release/smithy-checksums-feature-2c585fb742b2409690bb374cf08cd6cf.json @@ -1,4 +1,4 @@ { "type": "feature", - "description": "Added the `smithy-checksums` package, which provides CRC32 and CRC32C checksums backed by a native Rust extension with a pure-Python fallback, and SHA-1, SHA-256, SHA-512, and MD5 checksums through `new()`." + "description": "Added CRC32 and CRC32C checksums, backed by a native Rust extension with a pure-Python fallback, and SHA-1, SHA-256, SHA-512, and MD5 checksums through `new()`." } \ No newline at end of file From 4a29f61516d02c356d720d3246260498f969183c Mon Sep 17 00:00:00 2001 From: jonathan343 Date: Sun, 25 Oct 2026 00:57:07 -0400 Subject: [PATCH 4/9] Make the Checksum protocol runtime-checkable --- packages/smithy-checksums/src/smithy_checksums/__init__.py | 3 ++- packages/smithy-checksums/tests/unit/test_checksums.py | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/smithy-checksums/src/smithy_checksums/__init__.py b/packages/smithy-checksums/src/smithy_checksums/__init__.py index d865b9e9d..88eb88d5d 100644 --- a/packages/smithy-checksums/src/smithy_checksums/__init__.py +++ b/packages/smithy-checksums/src/smithy_checksums/__init__.py @@ -11,7 +11,7 @@ import hashlib from collections.abc import Buffer from importlib.metadata import version -from typing import ClassVar, Final, Protocol, Self +from typing import ClassVar, Final, Protocol, Self, runtime_checkable from ._private.backend import IMPLEMENTATIONS as _IMPLEMENTATIONS from ._private.backend import NATIVE as _NATIVE @@ -34,6 +34,7 @@ """ +@runtime_checkable class Checksum(Protocol): """A streaming checksum. diff --git a/packages/smithy-checksums/tests/unit/test_checksums.py b/packages/smithy-checksums/tests/unit/test_checksums.py index 567ab4672..ba17d1a4d 100644 --- a/packages/smithy-checksums/tests/unit/test_checksums.py +++ b/packages/smithy-checksums/tests/unit/test_checksums.py @@ -109,6 +109,12 @@ def test_new_rejects_unknown_algorithms() -> None: new("crc16") +def test_checksum_protocol_supports_isinstance() -> None: + assert isinstance(CRC32C(), Checksum) + assert isinstance(hashlib.sha256(), Checksum) + assert not isinstance(object(), Checksum) + + def test_has_native_extension() -> None: built = importlib.util.find_spec("smithy_checksums._native") is not None assert smithy_checksums.HAS_NATIVE_EXTENSION is built From 7efa9aeb0ca9ed51ddfad83116fa8c0d77c789d9 Mon Sep 17 00:00:00 2001 From: jonathan343 Date: Tue, 27 Oct 2026 11:42:34 -0400 Subject: [PATCH 5/9] Speed up the pure-Python CRC32C by hashing chunks with big integers The byte-at-a-time table loop runs about a dozen bytecodes per byte, which caps it at roughly 15 MB/s however it's arranged; slicing-by-8 and 16-bit tables only reach 20-35 MB/s. A CRC without its initial value and final XOR is linear over GF(2), so each bit of the CRC of a fixed-size chunk is the parity of the chunk's bits under a fixed mask. Input is now buffered into 16 KiB chunks, each read as one integer, and every CRC bit is computed with an AND and bit_count(), which do the per-byte work in C. The masks are built lazily by doubling, in a few milliseconds. Remainders under 1 KiB still use the table. CRC32C goes from about 16 MB/s to about 210 MB/s, whatever the update size. --- .../src/smithy_checksums/_private/crc.py | 105 +++++++++++++++++- .../tests/unit/test_implementations.py | 36 ++++++ 2 files changed, 135 insertions(+), 6 deletions(-) diff --git a/packages/smithy-checksums/src/smithy_checksums/_private/crc.py b/packages/smithy-checksums/src/smithy_checksums/_private/crc.py index 92e5f3682..17819186c 100644 --- a/packages/smithy-checksums/src/smithy_checksums/_private/crc.py +++ b/packages/smithy-checksums/src/smithy_checksums/_private/crc.py @@ -3,10 +3,18 @@ """Pure-Python CRC implementations, used when the native extension is unavailable.""" import zlib +from functools import cache from typing import ClassVar, Self from .pure import PureChecksum +# Input is buffered and hashed this many bytes at a time with the masks from +# _chunk_masks. It must be a power of two. +_CHUNK_SIZE = 16384 +# Remainders shorter than this are hashed a byte at a time from the table, which +# is faster than a pass over the masks. +_MIN_MASKED_TAIL = 1024 + def _reflected_table(poly: int) -> tuple[int, ...]: table: list[int] = [] @@ -18,6 +26,50 @@ def _reflected_table(poly: int) -> tuple[int, ...]: return tuple(table) +def _apply(columns: list[int], value: int) -> int: + """Multiplies the GF(2) matrix with the given columns by ``value``.""" + result = 0 + for column in columns: + if value & 1: + result ^= column + value >>= 1 + return result + + +@cache +def _chunk_masks(table: tuple[int, ...], width: int) -> tuple[int, ...]: + """Returns masks that compute a CRC over a chunk with big-integer operations. + + Without its initial value and final XOR, a CRC is linear over GF(2), so each + bit of the CRC of a message is the parity of a fixed subset of the message's + bits. Bit ``r`` of the CRC of a ``_CHUNK_SIZE``-byte chunk, read as a + little-endian integer ``m``, is the parity of ``m & masks[r]``. This does the + per-byte work in C, which is many times faster than a table lookup per byte. + + The masks are built for one byte and then doubled: the CRC of ``a + b`` is the + CRC of ``a`` shifted through ``len(b)`` zero bytes, XORed with the CRC of + ``b``. The shift is linear too, and ``columns[s]`` holds its effect on bit + ``s`` of the CRC. + """ + masks = [0] * width + for bit in range(8): + for row in range(width): + if (table[1 << bit] >> row) & 1: + masks[row] |= 1 << bit + columns = [(table[(1 << s) & 0xFF] ^ ((1 << s) >> 8)) for s in range(width)] + length = 1 + while length < _CHUNK_SIZE: + shifted = [0] * width + for s, column in enumerate(columns): + for row in range(width): + if (column >> row) & 1: + shifted[row] ^= masks[s] + masks = [low | (high << (8 * length)) for low, high in zip(shifted, masks)] + columns = [_apply(columns, column) for column in columns] + length *= 2 + return tuple(masks) + + class Crc32(PureChecksum): """CRC-32 (ISO-HDLC), backed by :func:`zlib.crc32`.""" @@ -40,9 +92,14 @@ def _copy(self) -> Self: class _ReflectedCrc(PureChecksum): - """A table-driven CRC whose initial value and final XOR are all ones.""" + """A table-driven CRC whose initial value and final XOR are all ones. - __slots__ = ("_crc",) + Whole chunks are hashed with the masks from :func:`_chunk_masks`. Up to a + chunk of input is buffered, so that the speed doesn't depend on how the input + is split between updates. + """ + + __slots__ = ("_buffer", "_crc") _table: ClassVar[tuple[int, ...]] _mask: ClassVar[int] @@ -51,20 +108,56 @@ class _ReflectedCrc(PureChecksum): def __init__(self) -> None: super().__init__() self._crc = self._mask + self._buffer = bytearray() def _update(self, data: bytes | bytearray) -> None: - table = self._table + buffer = self._buffer + view = memoryview(data) + if buffer: + needed = _CHUNK_SIZE - len(buffer) + buffer += view[:needed] + if len(buffer) < _CHUNK_SIZE: + return + self._crc = self._hash_chunk(self._crc, buffer) + buffer.clear() + view = view[needed:] + end = len(view) - len(view) % _CHUNK_SIZE crc = self._crc - for byte in data: - crc = table[(crc ^ byte) & 0xFF] ^ (crc >> 8) + for offset in range(0, end, _CHUNK_SIZE): + crc = self._hash_chunk(crc, view[offset : offset + _CHUNK_SIZE]) self._crc = crc + buffer += view[end:] + + def _hash_chunk(self, crc: int, chunk: bytes | bytearray | memoryview) -> int: + """Returns the CRC after ``chunk``, which may be shorter than a chunk. + + A reflected CRC's state can be XORed into the next bytes of input and + cleared instead, and leading zero bytes don't change a CRC of zero. A short + chunk must still be at least as long as the CRC. + """ + message = (int.from_bytes(chunk, "little") ^ crc) << ( + 8 * (_CHUNK_SIZE - len(chunk)) + ) + result = 0 + for row, mask in enumerate(_chunk_masks(self._table, 8 * self._size)): + result |= ((message & mask).bit_count() & 1) << row + return result def _digest(self) -> bytes: - return (self._crc ^ self._mask).to_bytes(self._size, "big") + crc = self._crc + tail = self._buffer + if len(tail) >= _MIN_MASKED_TAIL: + crc = self._hash_chunk(crc, tail) + else: + table = self._table + for byte in tail: + crc = table[(crc ^ byte) & 0xFF] ^ (crc >> 8) + return (crc ^ self._mask).to_bytes(self._size, "big") def _copy(self) -> Self: new = type(self)() new._crc = self._crc + new._buffer = self._buffer.copy() return new diff --git a/packages/smithy-checksums/tests/unit/test_implementations.py b/packages/smithy-checksums/tests/unit/test_implementations.py index a50b39184..73975250f 100644 --- a/packages/smithy-checksums/tests/unit/test_implementations.py +++ b/packages/smithy-checksums/tests/unit/test_implementations.py @@ -180,3 +180,39 @@ def test_matches_crt(impls: dict[str, Factory], algorithm: str) -> None: _digest(impls[algorithm], data[:length]) == compute[algorithm](data[:length]).hex() ), length + + +def _bitwise_crc(poly: int, width: int, data: bytes) -> str: + """Computes a reflected CRC with all-ones init and final XOR, a bit at a time.""" + mask = (1 << width) - 1 + value = mask + for byte in data: + value ^= byte + for _ in range(8): + value = (value >> 1) ^ poly if value & 1 else value >> 1 + return (value ^ mask).to_bytes(width // 8, "big").hex() + + +@pytest.mark.parametrize( + ("algorithm", "poly", "width"), + [("crc32c", 0x82F63B78, 32)], +) +def test_chunked_crc_matches_bitwise(algorithm: str, poly: int, width: int) -> None: + # The pure CRCs hash whole chunks with masks, short tails with a table, and + # long tails with the masks again, so check lengths on either side of each. + chunk = crc._CHUNK_SIZE # pyright: ignore[reportPrivateUsage] + tail = crc._MIN_MASKED_TAIL # pyright: ignore[reportPrivateUsage] + data = os.urandom(2 * chunk + tail + 1) + for length in [0, 1, 8, tail - 1, tail, chunk - 1, chunk, chunk + 1, len(data)]: + assert _digest(PURE[algorithm], data[:length]) == _bitwise_crc( + poly, width, data[:length] + ), length + + rng = random.Random(algorithm) + chunks: list[bytes] = [] + offset = 0 + while offset < len(data): + size = rng.choice((1, 100, tail, chunk - 1, chunk, chunk + 1)) + chunks.append(data[offset : offset + size]) + offset += size + assert _digest(PURE[algorithm], *chunks) == _bitwise_crc(poly, width, data) From 2d2b1626e44c8a836fdf2be9ef0bf7953f210e71 Mon Sep 17 00:00:00 2001 From: jonathan343 Date: Tue, 27 Oct 2026 16:45:29 -0400 Subject: [PATCH 6/9] Keep bytecode out of native wheels maturin packages everything under python-source, so a native wheel built from a source tree that's been imported from picked up its __pycache__ directories, including bytecode for modules that no longer exist. The hatchling targets already exclude build artifacts; this gives maturin the same treatment. --- packages/smithy-checksums/pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/smithy-checksums/pyproject.toml b/packages/smithy-checksums/pyproject.toml index 91ed1fd03..b9c3336fe 100644 --- a/packages/smithy-checksums/pyproject.toml +++ b/packages/smithy-checksums/pyproject.toml @@ -47,6 +47,7 @@ backend-path = ["build_backend"] python-source = "src" module-name = "smithy_checksums._native" features = ["pyo3/extension-module"] +exclude = ["**/__pycache__/**", "**/*.pyc"] [tool.hatch.build.targets.wheel] packages = ["src/smithy_checksums"] From 7a7cbadd21e6377845d0e8b15bebcb0d1cdd0531 Mon Sep 17 00:00:00 2001 From: jonathan343 Date: Tue, 27 Oct 2026 16:20:37 -0400 Subject: [PATCH 7/9] Copy non-bytes buffers instead of reading them in place The native extension read read-only buffers in place through an unsafe slice, on the grounds that nothing could change them. The read-only flag only stops this reader from writing, though: a read-only memoryview of a bytearray still changes when another thread writes to the bytearray, which can happen while the interpreter is detached. Only bytes is immutable, so it's the only input that's still hashed in place; every other buffer is copied. Copying runs at about 40 GB/s, against about 100 GB/s in place. Also: - State::digest returns a fixed-size array, given by an associated Digest type, instead of a Vec. - Import pyo3 items explicitly instead of through the prelude. --- packages/smithy-checksums/rust/lib.rs | 51 ++++++++++++--------------- 1 file changed, 23 insertions(+), 28 deletions(-) diff --git a/packages/smithy-checksums/rust/lib.rs b/packages/smithy-checksums/rust/lib.rs index 4ed6086c1..7e94dc5c6 100644 --- a/packages/smithy-checksums/rust/lib.rs +++ b/packages/smithy-checksums/rust/lib.rs @@ -9,8 +9,8 @@ use std::sync::{Mutex, MutexGuard}; use pyo3::buffer::PyBuffer; -use pyo3::prelude::*; -use pyo3::types::PyBytes; +use pyo3::types::{PyAny, PyBytes, PyBytesMethods}; +use pyo3::{Bound, PyResult, Python, pyclass, pymethods, pymodule}; /// Inputs at least this large are hashed with the interpreter detached so /// other threads can run. Below it, detaching costs more than it saves. This @@ -19,17 +19,22 @@ const DETACH_THRESHOLD: usize = 2048; /// Streaming checksum state shared by every class in this module. trait State: Clone + Send { + /// The digest, a byte array as long as the algorithm's output. + type Digest: AsRef<[u8]>; + fn new() -> Self; fn update(&mut self, data: &[u8]); - fn digest(&self) -> Vec; + fn digest(&self) -> Self::Digest; } macro_rules! crc_state { - ($state:ident, $algorithm:ident, $size:literal) => { + ($state:ident, $algorithm:ident, $width:ty) => { #[derive(Clone)] struct $state(crc_fast::Digest); impl State for $state { + type Digest = [u8; size_of::<$width>()]; + fn new() -> Self { Self(crc_fast::Digest::new(crc_fast::CrcAlgorithm::$algorithm)) } @@ -38,16 +43,17 @@ macro_rules! crc_state { self.0.update(data); } - fn digest(&self) -> Vec { - // The digest is a u64 regardless of the CRC's width. - self.0.finalize().to_be_bytes()[8 - $size..].to_vec() + fn digest(&self) -> Self::Digest { + // `finalize` returns a u64 regardless of the CRC's width, and + // the CRC is in its low bits. + (self.0.finalize() as $width).to_be_bytes() } } }; } -crc_state!(Crc32State, Crc32IsoHdlc, 4); -crc_state!(Crc32cState, Crc32Iscsi, 4); +crc_state!(Crc32State, Crc32IsoHdlc, u32); +crc_state!(Crc32cState, Crc32Iscsi, u32); fn lock(state: &Mutex) -> MutexGuard<'_, S> { // A panic while holding the lock can't leave a checksum half-updated in a @@ -59,26 +65,15 @@ fn lock(state: &Mutex) -> MutexGuard<'_, S> { /// Feeds the bytes of `data` to `state`. /// -/// `bytes` and other read-only buffers are hashed in place. Writable buffers -/// (such as `bytearray`) are copied first, because another thread could -/// otherwise modify them while they're being read. +/// `bytes` is immutable, so it's hashed in place. Other buffers are copied +/// first, because another thread could modify them while they're being read. +/// That includes read-only buffers: the flag only stops this reader from +/// writing, and a read-only `memoryview` of a `bytearray` can still change. fn update(py: Python<'_>, state: &Mutex, data: &Bound<'_, PyAny>) -> PyResult<()> { - if let Ok(bytes) = data.cast::() { - hash_slice(py, state, bytes.as_bytes()); - return Ok(()); - } - - let buffer = PyBuffer::::get(data)?; - if buffer.readonly() && buffer.is_c_contiguous() { - // SAFETY: the buffer is read-only and C-contiguous, and `buffer` - // keeps the exporter alive and its memory pinned until it's dropped - // at the end of this function. - let slice = unsafe { - std::slice::from_raw_parts(buffer.buf_ptr().cast::(), buffer.len_bytes()) - }; - hash_slice(py, state, slice); + if let Ok(data) = data.cast::() { + hash_slice(py, state, data.as_bytes()); } else { - let copy = buffer.to_vec(py)?; + let copy = PyBuffer::::get(data)?.to_vec(py)?; hash_slice(py, state, ©); } Ok(()) @@ -113,7 +108,7 @@ macro_rules! checksum_class { } fn digest<'py>(&self, py: Python<'py>) -> Bound<'py, PyBytes> { - PyBytes::new(py, &lock(&self.state).digest()) + PyBytes::new(py, lock(&self.state).digest().as_ref()) } fn copy(&self) -> Self { From c41041689e24d745ee80112cc6e0fdef38622320 Mon Sep 17 00:00:00 2001 From: jonathan343 Date: Sat, 24 Oct 2026 22:06:27 -0400 Subject: [PATCH 8/9] Add CRC64NVME to smithy-checksums Adds CRC64NVME, the algorithm S3 uses for full-object checksums, backed by the native extension with a pure-Python fallback, and available through new("crc64nvme"). --- ...ture-6c167ff7220c4033b9eb3299d4627564.json | 4 +++ packages/smithy-checksums/README.md | 4 +-- .../build_backend/smithy_checksums_build.py | 2 +- packages/smithy-checksums/pyproject.toml | 2 +- packages/smithy-checksums/rust/lib.rs | 4 ++- .../src/smithy_checksums/__init__.py | 29 +++++++++++----- .../src/smithy_checksums/_native.pyi | 5 +++ .../src/smithy_checksums/_private/backend.py | 2 ++ .../src/smithy_checksums/_private/crc.py | 10 ++++++ .../tests/unit/test_checksums.py | 3 ++ .../tests/unit/test_implementations.py | 3 ++ .../smithy-checksums/tests/unit/vectors.py | 33 +++++++++++++++++-- scripts/api-snapshot/api.json | 1 + 13 files changed, 85 insertions(+), 17 deletions(-) create mode 100644 packages/smithy-checksums/.changes/next-release/smithy-checksums-feature-6c167ff7220c4033b9eb3299d4627564.json diff --git a/packages/smithy-checksums/.changes/next-release/smithy-checksums-feature-6c167ff7220c4033b9eb3299d4627564.json b/packages/smithy-checksums/.changes/next-release/smithy-checksums-feature-6c167ff7220c4033b9eb3299d4627564.json new file mode 100644 index 000000000..d7a00e7d1 --- /dev/null +++ b/packages/smithy-checksums/.changes/next-release/smithy-checksums-feature-6c167ff7220c4033b9eb3299d4627564.json @@ -0,0 +1,4 @@ +{ + "type": "feature", + "description": "Added CRC64NVME checksums." +} \ No newline at end of file diff --git a/packages/smithy-checksums/README.md b/packages/smithy-checksums/README.md index 227cf26c2..1e579b6d1 100644 --- a/packages/smithy-checksums/README.md +++ b/packages/smithy-checksums/README.md @@ -1,7 +1,7 @@ # smithy-checksums -This package provides the checksum algorithms used by Smithy clients: CRC32 and -CRC32C, and, through `new()`, SHA-1, SHA-256, SHA-512, and MD5. +This package provides the checksum algorithms used by Smithy clients: CRC32, +CRC32C, CRC64NVME, and, through `new()`, SHA-1, SHA-256, SHA-512, and MD5. ```python import smithy_checksums diff --git a/packages/smithy-checksums/build_backend/smithy_checksums_build.py b/packages/smithy-checksums/build_backend/smithy_checksums_build.py index c41cdc8ad..0eb5ca38f 100644 --- a/packages/smithy-checksums/build_backend/smithy_checksums_build.py +++ b/packages/smithy-checksums/build_backend/smithy_checksums_build.py @@ -36,7 +36,7 @@ _CARGO_TOML = Path(__file__).parent.parent / "Cargo.toml" _PURE_PYTHON_HINT = ( f"To install the pure-Python implementation instead, set {_ENV_VAR}=never. It " - "produces identical checksums, but CRC32C is much slower." + "produces identical checksums, but CRC32C and CRC64NVME are much slower." ) diff --git a/packages/smithy-checksums/pyproject.toml b/packages/smithy-checksums/pyproject.toml index b9c3336fe..3ba77d5c4 100644 --- a/packages/smithy-checksums/pyproject.toml +++ b/packages/smithy-checksums/pyproject.toml @@ -9,7 +9,7 @@ authors = [ description = "Checksum algorithms for Smithy tooling, with native acceleration." readme = "README.md" license = {text = "Apache License 2.0"} -keywords = ["smithy", "sdk", "checksum", "crc32c"] +keywords = ["smithy", "sdk", "checksum", "crc32c", "crc64nvme"] classifiers = [ "Development Status :: 2 - Pre-Alpha", "Intended Audience :: Developers", diff --git a/packages/smithy-checksums/rust/lib.rs b/packages/smithy-checksums/rust/lib.rs index 7e94dc5c6..4b2e76b3a 100644 --- a/packages/smithy-checksums/rust/lib.rs +++ b/packages/smithy-checksums/rust/lib.rs @@ -54,6 +54,7 @@ macro_rules! crc_state { crc_state!(Crc32State, Crc32IsoHdlc, u32); crc_state!(Crc32cState, Crc32Iscsi, u32); +crc_state!(Crc64NvmeState, Crc64Nvme, u64); fn lock(state: &Mutex) -> MutexGuard<'_, S> { // A panic while holding the lock can't leave a checksum half-updated in a @@ -122,9 +123,10 @@ macro_rules! checksum_class { checksum_class!(Crc32, Crc32State); checksum_class!(Crc32C, Crc32cState); +checksum_class!(Crc64Nvme, Crc64NvmeState); #[pymodule] mod _native { #[pymodule_export] - use super::{Crc32, Crc32C}; + use super::{Crc32, Crc32C, Crc64Nvme}; } diff --git a/packages/smithy-checksums/src/smithy_checksums/__init__.py b/packages/smithy-checksums/src/smithy_checksums/__init__.py index 88eb88d5d..74a7c6e9c 100644 --- a/packages/smithy-checksums/src/smithy_checksums/__init__.py +++ b/packages/smithy-checksums/src/smithy_checksums/__init__.py @@ -2,10 +2,11 @@ # SPDX-License-Identifier: Apache-2.0 """Checksum algorithms for Smithy clients. -CRC32 and CRC32C use a native extension when one is available for the platform -and fall back to pure Python otherwise. The fallback produces identical results -but is orders of magnitude slower, except for CRC32, which uses :mod:`zlib`. SHA-1, SHA-256, SHA-512, and MD5 are available -through :func:`new` and always use :mod:`hashlib`. +CRC32, CRC32C, and CRC64NVME use a native extension when one is available for the +platform and fall back to pure Python otherwise. The fallback produces identical +results but is orders of magnitude slower, except for CRC32, which uses :mod:`zlib`. +SHA-1, SHA-256, SHA-512, and MD5 are available through :func:`new` and always use +:mod:`hashlib`. """ import hashlib @@ -22,6 +23,7 @@ __all__ = ( "CRC32", "CRC32C", + "CRC64NVME", "HAS_NATIVE_EXTENSION", "Checksum", "new", @@ -30,7 +32,7 @@ HAS_NATIVE_EXTENSION: Final[bool] = _NATIVE """Whether the native extension is in use. -When it isn't, CRC32C is computed in pure Python. +When it isn't, CRC32C and CRC64NVME are computed in pure Python. """ @@ -121,7 +123,16 @@ class CRC32C(_BackendChecksum): digest_size = 4 -_CHECKSUMS: Final = {cls.name: cls for cls in (CRC32, CRC32C)} +class CRC64NVME(_BackendChecksum): + """CRC-64/NVME.""" + + __slots__ = () + + name = "crc64nvme" + digest_size = 8 + + +_CHECKSUMS: Final = {cls.name: cls for cls in (CRC32, CRC32C, CRC64NVME)} _HASHLIB_ALGORITHMS: Final = frozenset(("sha1", "sha256", "sha512")) @@ -129,10 +140,10 @@ def new(name: str) -> Checksum: """Creates a checksum from the name of its algorithm. Names are case-insensitive, so the values of a checksum algorithm enum such as - ``CRC32C`` or ``SHA256`` can be passed directly. + ``CRC64NVME`` or ``SHA256`` can be passed directly. - :param name: One of ``crc32``, ``crc32c``, ``sha1``, ``sha256``, ``sha512``, or - ``md5``. + :param name: One of ``crc32``, ``crc32c``, ``crc64nvme``, ``sha1``, ``sha256``, + ``sha512``, or ``md5``. :raises ValueError: If the algorithm isn't supported. """ key = name.lower() diff --git a/packages/smithy-checksums/src/smithy_checksums/_native.pyi b/packages/smithy-checksums/src/smithy_checksums/_native.pyi index 1163b9bb9..48214f330 100644 --- a/packages/smithy-checksums/src/smithy_checksums/_native.pyi +++ b/packages/smithy-checksums/src/smithy_checksums/_native.pyi @@ -15,3 +15,8 @@ class Crc32C: def update(self, data: _Bytes, /) -> None: ... def digest(self) -> bytes: ... def copy(self) -> Self: ... + +class Crc64Nvme: + def update(self, data: _Bytes, /) -> None: ... + def digest(self) -> bytes: ... + def copy(self) -> Self: ... diff --git a/packages/smithy-checksums/src/smithy_checksums/_private/backend.py b/packages/smithy-checksums/src/smithy_checksums/_private/backend.py index c542afd87..82b5d85a3 100644 --- a/packages/smithy-checksums/src/smithy_checksums/_private/backend.py +++ b/packages/smithy-checksums/src/smithy_checksums/_private/backend.py @@ -31,11 +31,13 @@ def _load() -> tuple[bool, Mapping[str, ImplFactory]]: return False, { "crc32": crc.Crc32, "crc32c": crc.Crc32C, + "crc64nvme": crc.Crc64Nvme, } return True, { "crc32": _native.Crc32, "crc32c": _native.Crc32C, + "crc64nvme": _native.Crc64Nvme, } diff --git a/packages/smithy-checksums/src/smithy_checksums/_private/crc.py b/packages/smithy-checksums/src/smithy_checksums/_private/crc.py index 17819186c..4d8d1be65 100644 --- a/packages/smithy-checksums/src/smithy_checksums/_private/crc.py +++ b/packages/smithy-checksums/src/smithy_checksums/_private/crc.py @@ -169,3 +169,13 @@ class Crc32C(_ReflectedCrc): _table = _reflected_table(0x82F63B78) _mask = 0xFFFFFFFF _size = 4 + + +class Crc64Nvme(_ReflectedCrc): + """CRC-64/NVME.""" + + __slots__ = () + + _table = _reflected_table(0x9A6C9329AC4BC9B5) + _mask = 0xFFFFFFFFFFFFFFFF + _size = 8 diff --git a/packages/smithy-checksums/tests/unit/test_checksums.py b/packages/smithy-checksums/tests/unit/test_checksums.py index ba17d1a4d..f29d5bab0 100644 --- a/packages/smithy-checksums/tests/unit/test_checksums.py +++ b/packages/smithy-checksums/tests/unit/test_checksums.py @@ -10,6 +10,7 @@ from smithy_checksums import ( CRC32, CRC32C, + CRC64NVME, Checksum, new, ) @@ -19,6 +20,7 @@ CLASSES = { "crc32": CRC32, "crc32c": CRC32C, + "crc64nvme": CRC64NVME, } @@ -90,6 +92,7 @@ def compute(_: int) -> str: [ ("crc32", CRC32), ("CRC32C", CRC32C), + ("CRC64NVME", CRC64NVME), ], ) def test_new_returns_checksum_classes(name: str, expected: type[Checksum]) -> None: diff --git a/packages/smithy-checksums/tests/unit/test_implementations.py b/packages/smithy-checksums/tests/unit/test_implementations.py index 73975250f..0ef0133a4 100644 --- a/packages/smithy-checksums/tests/unit/test_implementations.py +++ b/packages/smithy-checksums/tests/unit/test_implementations.py @@ -23,6 +23,7 @@ PURE: dict[str, Factory] = { "crc32": crc.Crc32, "crc32c": crc.Crc32C, + "crc64nvme": crc.Crc64Nvme, } @@ -41,6 +42,7 @@ def _load_native() -> ModuleType | None: else { "crc32": _NATIVE_MODULE.Crc32, "crc32c": _NATIVE_MODULE.Crc32C, + "crc64nvme": _NATIVE_MODULE.Crc64Nvme, } ) requires_native = pytest.mark.skipif( @@ -173,6 +175,7 @@ def test_matches_crt(impls: dict[str, Factory], algorithm: str) -> None: compute: dict[str, Callable[[bytes], bytes]] = { "crc32": lambda data: crt.crc32(data).to_bytes(4, "big"), "crc32c": lambda data: crt.crc32c(data).to_bytes(4, "big"), + "crc64nvme": lambda data: crt.crc64nvme(data).to_bytes(8, "big"), } data = os.urandom(3000) for length in [*range(0, 300, 7), 1023, 1024, 1025, 3000]: diff --git a/packages/smithy-checksums/tests/unit/vectors.py b/packages/smithy-checksums/tests/unit/vectors.py index 7697f57ce..a02f018aa 100644 --- a/packages/smithy-checksums/tests/unit/vectors.py +++ b/packages/smithy-checksums/tests/unit/vectors.py @@ -2,11 +2,12 @@ # SPDX-License-Identifier: Apache-2.0 """Expected digests shared by the checksum tests.""" -ALGORITHMS = ("crc32", "crc32c") +ALGORITHMS = ("crc32", "crc32c", "crc64nvme") # Digests from independent sources: the standard check value of each CRC (the -# digest of "123456789"), the CRC32C test vectors from RFC 3720, and "Hello world" -# digests that match the ones the CRT produces. +# digest of "123456789"), the CRC32C test vectors from RFC 3720, the CRC64NVME test +# cases from the NVMe NVM Command Set specification, and "Hello world" digests that +# match the ones the CRT produces. KNOWN_ANSWERS: dict[str, dict[bytes, str]] = { "crc32": {b"123456789": "cbf43926", b"Hello world": "8bd69e52"}, "crc32c": { @@ -19,6 +20,15 @@ bytes(range(32)): "46dd794e", bytes(range(31, -1, -1)): "113fdb5c", }, + "crc64nvme": { + b"123456789": "ae8b14860a799888", + b"Hello world": "38e259d03f312adb", + # The 64b CRC test cases in the NVMe NVM Command Set specification. + bytes(4096): "6482d367eb22b64e", + b"\xff" * 4096: "c0ddba7302eca3ac", + bytes(i % 256 for i in range(4096)): "3e729f5f6750449c", + bytes(255 - i % 256 for i in range(4096)): "9a2df64b8e9e517e", + }, } @@ -64,4 +74,21 @@ def pattern(length: int) -> bytes: 1025: "c8d03add", 16385: "896b2de8", }, + "crc64nvme": { + 0: "0000000000000000", + 1: "d5da5047efec8728", + 3: "3f3990291e6454d7", + 4: "29513a0b0028dea8", + 8: "7703cc98ac9209d7", + 9: "0eb6fdbb89d04d0b", + 16: "cf6454c75df95620", + 17: "5c34355fb173cefd", + 128: "a5296136625de4f7", + 129: "c3d556fcf23d6a24", + 240: "717e94e4fd87236d", + 241: "2737bbceef8467bc", + 1024: "fee95dc7adac063f", + 1025: "034aba81017791d2", + 16385: "9c0531959bc1eacf", + }, } diff --git a/scripts/api-snapshot/api.json b/scripts/api-snapshot/api.json index ca3a56342..75298cc55 100644 --- a/scripts/api-snapshot/api.json +++ b/scripts/api-snapshot/api.json @@ -680,6 +680,7 @@ "smithy_checksums": { "CRC32": "class", "CRC32C": "class", + "CRC64NVME": "class", "Checksum": "protocol", "HAS_NATIVE_EXTENSION": "bool", "new": "function" From eef1834945125d2c138f1e7e6aa476f785c21006 Mon Sep 17 00:00:00 2001 From: jonathan343 Date: Tue, 27 Oct 2026 11:42:57 -0400 Subject: [PATCH 9/9] Check the chunked pure-Python CRC64NVME against a bitwise CRC CRC64NVME shares the chunked big-integer path with CRC32C, which takes it from about 15 MB/s to about 135 MB/s. Add it to the test that checks that path against a bit-at-a-time CRC around the chunk and tail boundaries. --- packages/smithy-checksums/tests/unit/test_implementations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/smithy-checksums/tests/unit/test_implementations.py b/packages/smithy-checksums/tests/unit/test_implementations.py index 0ef0133a4..95c9fc4ac 100644 --- a/packages/smithy-checksums/tests/unit/test_implementations.py +++ b/packages/smithy-checksums/tests/unit/test_implementations.py @@ -198,7 +198,7 @@ def _bitwise_crc(poly: int, width: int, data: bytes) -> str: @pytest.mark.parametrize( ("algorithm", "poly", "width"), - [("crc32c", 0x82F63B78, 32)], + [("crc32c", 0x82F63B78, 32), ("crc64nvme", 0x9A6C9329AC4BC9B5, 64)], ) def test_chunked_crc_matches_bitwise(algorithm: str, poly: int, width: int) -> None: # The pure CRCs hash whole chunks with masks, short tails with a table, and