From 01af1cf147b29640b62d32c4e4922454fc1b93ef Mon Sep 17 00:00:00 2001 From: jonathan343 Date: Sat, 24 Oct 2026 22:05:05 -0400 Subject: [PATCH 01/12] Add smithy-checksums package with CRC32 and CRC32C Adds the smithy-checksums package, which provides the checksum algorithms Smithy clients need for flexible checksums. CRC32 and CRC32C are backed by a native Rust extension (PyO3, crc-fast) with a pure-Python fallback, and SHA-1, SHA-256, SHA-512, and MD5 come from hashlib through new(). Wheels are built by an in-tree PEP 517 backend: maturin when a Rust toolchain is available, hatchling (pure Python) otherwise. Source distributions are always built with hatchling. SMITHY_PYTHON_NATIVE_BUILD (auto, always, never) overrides the choice. If Rust is present but too old, or the native build fails, the build stops with instructions instead of silently falling back. Native wheels target abi3 for 3.12+, cp314t, and abi3t for 3.15t+. Native and pure-Python checksum objects are safe to share between threads. Repository setup: - CI builds the native extension in the main matrix, checks Rust formatting, lints, dependencies (cargo-deny), and third-party license attribution (cargo-about), builds and tests native wheels on Linux, macOS, and Windows, and tests the pure-Python wheel and an sdist install without Rust. - Adds make targets check-rust, lint-rust, and third-party-licenses. - License header checks cover .rs and .pyi files. - Dependabot updates Cargo dependencies. --- .github/dependabot.yml | 13 + .github/workflows/ci.yml | 112 ++ .licenserc.yaml | 3 + Makefile | 38 +- ...ture-2c585fb742b2409690bb374cf08cd6cf.json | 4 + packages/smithy-checksums/.gitignore | 3 + packages/smithy-checksums/CHANGELOG.md | 1 + packages/smithy-checksums/Cargo.lock | 190 +++ packages/smithy-checksums/Cargo.toml | 20 + packages/smithy-checksums/LICENSE-THIRD-PARTY | 1034 +++++++++++++++++ packages/smithy-checksums/NOTICE | 1 + packages/smithy-checksums/README.md | 31 + packages/smithy-checksums/about.hbs | 16 + packages/smithy-checksums/about.toml | 11 + .../build_backend/smithy_checksums_build.py | 151 +++ packages/smithy-checksums/deny.toml | 24 + packages/smithy-checksums/pyproject.toml | 81 ++ packages/smithy-checksums/rust/lib.rs | 135 +++ .../src/smithy_checksums/__init__.py | 144 +++ .../src/smithy_checksums/_native.pyi | 17 + .../src/smithy_checksums/_private/__init__.py | 2 + .../src/smithy_checksums/_private/backend.py | 42 + .../src/smithy_checksums/_private/buffers.py | 23 + .../src/smithy_checksums/_private/crc.py | 78 ++ .../src/smithy_checksums/_private/pure.py | 42 + .../src/smithy_checksums/py.typed | 1 + packages/smithy-checksums/tests/__init__.py | 2 + .../smithy-checksums/tests/unit/__init__.py | 2 + .../tests/unit/test_build_backend.py | 162 +++ .../tests/unit/test_checksums.py | 114 ++ .../tests/unit/test_implementations.py | 182 +++ .../smithy-checksums/tests/unit/vectors.py | 67 ++ pyproject.toml | 1 + scripts/api-snapshot/api.json | 7 + uv.lock | 6 + 35 files changed, 2759 insertions(+), 1 deletion(-) create mode 100644 packages/smithy-checksums/.changes/next-release/smithy-checksums-feature-2c585fb742b2409690bb374cf08cd6cf.json create mode 100644 packages/smithy-checksums/.gitignore create mode 100644 packages/smithy-checksums/CHANGELOG.md create mode 100644 packages/smithy-checksums/Cargo.lock create mode 100644 packages/smithy-checksums/Cargo.toml create mode 100644 packages/smithy-checksums/LICENSE-THIRD-PARTY create mode 100644 packages/smithy-checksums/NOTICE create mode 100644 packages/smithy-checksums/README.md create mode 100644 packages/smithy-checksums/about.hbs create mode 100644 packages/smithy-checksums/about.toml create mode 100644 packages/smithy-checksums/build_backend/smithy_checksums_build.py create mode 100644 packages/smithy-checksums/deny.toml create mode 100644 packages/smithy-checksums/pyproject.toml create mode 100644 packages/smithy-checksums/rust/lib.rs create mode 100644 packages/smithy-checksums/src/smithy_checksums/__init__.py create mode 100644 packages/smithy-checksums/src/smithy_checksums/_native.pyi create mode 100644 packages/smithy-checksums/src/smithy_checksums/_private/__init__.py create mode 100644 packages/smithy-checksums/src/smithy_checksums/_private/backend.py create mode 100644 packages/smithy-checksums/src/smithy_checksums/_private/buffers.py create mode 100644 packages/smithy-checksums/src/smithy_checksums/_private/crc.py create mode 100644 packages/smithy-checksums/src/smithy_checksums/_private/pure.py create mode 100644 packages/smithy-checksums/src/smithy_checksums/py.typed create mode 100644 packages/smithy-checksums/tests/__init__.py create mode 100644 packages/smithy-checksums/tests/unit/__init__.py create mode 100644 packages/smithy-checksums/tests/unit/test_build_backend.py create mode 100644 packages/smithy-checksums/tests/unit/test_checksums.py create mode 100644 packages/smithy-checksums/tests/unit/test_implementations.py create mode 100644 packages/smithy-checksums/tests/unit/vectors.py diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 6b54affa2..7c197c07a 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -12,6 +12,19 @@ updates: - "patch" cooldown: default-days: 7 + - package-ecosystem: "cargo" + directories: + - "/packages/smithy-checksums" + schedule: + interval: "weekly" + day: "wednesday" + groups: + cargo: + update-types: + - "minor" + - "patch" + cooldown: + default-days: 7 - package-ecosystem: "gradle" directory: "/codegen" schedule: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ea935848c..92803b6e1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,6 +21,9 @@ jobs: fail-fast: false matrix: python-version: ["3.12", "3.13", "3.14", "3.14t", "3.15"] + env: + # Fail rather than fall back to pure Python if a native extension can't be built. + SMITHY_PYTHON_NATIVE_BUILD: always steps: - name: Checkout Repository @@ -49,6 +52,11 @@ jobs: with: version: 3.8.2 + - name: Set up Rust + run: | + rustup toolchain install stable --profile minimal --component clippy,rustfmt + rustup default stable + - name: Setup workspace run: | make install @@ -58,6 +66,17 @@ jobs: run: | make check-py + - name: Install Rust dependency tools + if: ${{ matrix.python-version == '3.12' }} + uses: taiki-e/install-action@1e29709c0ec63babbe8f266397a465f29a825f91 + with: + tool: cargo-about@0.9.2,cargo-deny@0.20.2 + + - name: Check rust extensions + if: ${{ matrix.python-version == '3.12' }} + run: | + make check-rust + - name: Check API snapshot if: ${{ matrix.python-version == '3.12' }} run: | @@ -88,3 +107,96 @@ jobs: if: ${{ matrix.python-version == '3.12' }} run: | make typecheck-protocols + + # Builds each native extension as a wheel on every platform and tests the + # installed wheel. The release pipeline builds the published wheels; this + # catches platform-specific build or test failures before then. + native-wheels: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, ubuntu-24.04-arm, macos-latest, windows-latest] + # 3.12 builds the abi3 wheel, 3.14t a version-specific free-threaded + # wheel, and 3.15t the abi3t wheel. + python-version: ["3.12", "3.14t", "3.15t"] + package: [smithy-checksums] + env: + SMITHY_PYTHON_NATIVE_BUILD: always + PACKAGE: ${{ matrix.package }} + defaults: + run: + shell: bash + + steps: + - name: Checkout Repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + + - name: Install uv and set the Python version + uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 + with: + python-version: ${{ matrix.python-version }} + activate-environment: true + enable-cache: true + + - name: Set up Rust + run: | + rustup toolchain install stable --profile minimal + rustup default stable + + - name: Build wheel + run: | + uv build "packages/$PACKAGE" --wheel --out-dir dist + + - name: Test wheel + run: | + uv pip install pytest dist/*.whl + python -c "import smithy_checksums; assert smithy_checksums.HAS_NATIVE_EXTENSION" + python -m pytest "packages/$PACKAGE/tests" --import-mode=importlib -o addopts="" + + # Tests the pure-Python wheel, and installing from the sdist without a Rust + # toolchain, which should fall back to the same pure-Python build. + pure-python-fallback: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + package: [smithy-checksums] + env: + PACKAGE: ${{ matrix.package }} + + steps: + - name: Checkout Repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + + - name: Install uv and set the Python version + uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 + with: + python-version: "3.12" + activate-environment: true + enable-cache: true + + - name: Build sdist and pure-Python wheel + env: + SMITHY_PYTHON_NATIVE_BUILD: never + run: | + uv build "packages/$PACKAGE" --out-dir dist + + - name: Test pure-Python wheel + run: | + uv pip install pytest awscrt dist/*-py3-none-any.whl + python -c "import smithy_checksums; assert not smithy_checksums.HAS_NATIVE_EXTENSION" + python -m pytest "packages/$PACKAGE/tests" --import-mode=importlib -o addopts="" + + - name: Test sdist install without Rust + run: | + PATH="$(echo "$PATH" | tr ':' '\n' | grep -v '/\.cargo/bin' | paste -sd: -)" + if command -v cargo; then echo "cargo is still on the PATH"; exit 1; fi + uv venv .venv-sdist + uv pip install --python .venv-sdist --no-cache pytest awscrt dist/*.tar.gz + .venv-sdist/bin/python -c "import smithy_checksums; assert not smithy_checksums.HAS_NATIVE_EXTENSION" + .venv-sdist/bin/python -m pytest "packages/$PACKAGE/tests" --import-mode=importlib -o addopts="" diff --git a/.licenserc.yaml b/.licenserc.yaml index 012b76f65..d2e11f094 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -8,6 +8,8 @@ header: paths: - '**/*.py' + - '**/*.pyi' + - '**/*.rs' paths-ignore: - '**/*.md' @@ -22,3 +24,4 @@ header: Python: extensions: - ".py" + - ".pyi" diff --git a/Makefile b/Makefile index 2ee48a516..f1ab4102e 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ .DEFAULT_GOAL:=help -.PHONY: help install format-java check-java build-java test-protocols typecheck-protocols generate-protocol-tests generate-codegen-output-py api-snapshot aio-check ci lint-py check-py test-py test-integration benchmark build-py clean +.PHONY: help install format-java check-java build-java test-protocols typecheck-protocols generate-protocol-tests generate-codegen-output-py api-snapshot aio-check ci lint-py check-py lint-rust check-rust third-party-licenses test-py test-integration benchmark build-py clean help: ## Show this help. @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n\nTargets:\n"} /^[a-zA-Z_-]+:.*?##/ { printf " \033[36m%-25s\033[0m %s\n", $$1, $$2 }' $(MAKEFILE_LIST) @@ -94,8 +94,44 @@ check-py: ## Runs checks (formatting, lints, type-checking) on the python packag uv run pyright packages +# Packages with a native extension, each built from a Cargo.toml in the package root. +RUST_MANIFESTS := $(wildcard packages/*/Cargo.toml) + +lint-rust: ## Formats and fixes lints in the Rust extensions. + @set -e; for manifest in $(RUST_MANIFESTS); do \ + cargo fmt --manifest-path "$$manifest"; \ + cargo clippy --manifest-path "$$manifest" --all-targets --fix --allow-dirty --allow-staged; \ + done + + +check-rust: ## Checks formatting, lints, dependencies, and license attribution of the Rust extensions. + @set -e; for manifest in $(RUST_MANIFESTS); do \ + dir=$$(dirname "$$manifest"); \ + cargo fmt --manifest-path "$$manifest" --check; \ + cargo clippy --manifest-path "$$manifest" --all-targets --locked -- -D warnings; \ + cargo deny --manifest-path "$$manifest" check; \ + cargo about generate --manifest-path "$$manifest" \ + --output-file "$$dir/LICENSE-THIRD-PARTY.expected" "$$dir/about.hbs"; \ + if ! diff -u "$$dir/LICENSE-THIRD-PARTY" "$$dir/LICENSE-THIRD-PARTY.expected"; then \ + rm "$$dir/LICENSE-THIRD-PARTY.expected"; \ + echo "$$dir/LICENSE-THIRD-PARTY is out of date. Run 'make third-party-licenses'."; \ + exit 1; \ + fi; \ + rm "$$dir/LICENSE-THIRD-PARTY.expected"; \ + done + + +third-party-licenses: ## Regenerates the third-party license file of each Rust extension from its Cargo.lock. + @set -e; for manifest in $(RUST_MANIFESTS); do \ + dir=$$(dirname "$$manifest"); \ + cargo about generate --manifest-path "$$manifest" \ + --output-file "$$dir/LICENSE-THIRD-PARTY" "$$dir/about.hbs"; \ + done + + ci: ## Runs the full CI sequence locally (mirrors the CI workflow steps). make check-py + make check-rust make api-snapshot make test-py make build-py diff --git a/packages/smithy-checksums/.changes/next-release/smithy-checksums-feature-2c585fb742b2409690bb374cf08cd6cf.json b/packages/smithy-checksums/.changes/next-release/smithy-checksums-feature-2c585fb742b2409690bb374cf08cd6cf.json new file mode 100644 index 000000000..6375c1caf --- /dev/null +++ b/packages/smithy-checksums/.changes/next-release/smithy-checksums-feature-2c585fb742b2409690bb374cf08cd6cf.json @@ -0,0 +1,4 @@ +{ + "type": "feature", + "description": "Added the `smithy-checksums` package, which provides CRC32 and CRC32C checksums backed by a native Rust extension with a pure-Python fallback, and SHA-1, SHA-256, SHA-512, and MD5 checksums through `new()`." +} \ No newline at end of file diff --git a/packages/smithy-checksums/.gitignore b/packages/smithy-checksums/.gitignore new file mode 100644 index 000000000..f122d7c95 --- /dev/null +++ b/packages/smithy-checksums/.gitignore @@ -0,0 +1,3 @@ +# Native extension built in place by editable installs. +*.so +*.pyd diff --git a/packages/smithy-checksums/CHANGELOG.md b/packages/smithy-checksums/CHANGELOG.md new file mode 100644 index 000000000..5ddad421e --- /dev/null +++ b/packages/smithy-checksums/CHANGELOG.md @@ -0,0 +1 @@ +# Changelog \ No newline at end of file diff --git a/packages/smithy-checksums/Cargo.lock b/packages/smithy-checksums/Cargo.lock new file mode 100644 index 000000000..792443944 --- /dev/null +++ b/packages/smithy-checksums/Cargo.lock @@ -0,0 +1,190 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "crc-fast" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e75b2483e97a5a7da73ac68a05b629f9c53cff58d8ed1c77866079e18b00dba5" +dependencies = [ + "digest", + "spin", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "crypto-common", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "libc" +version = "0.2.191" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "pyo3" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "229ca13b8a15bb0d1487b81ab975bfcd97513416499a8312011b6036e42e0de4" +dependencies = [ + "libc", + "once_cell", + "portable-atomic", + "pyo3-build-config", + "pyo3-ffi", + "pyo3-macros", +] + +[[package]] +name = "pyo3-build-config" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70130e2b6dd1ee589a8788026a429561ca5c40fa7a2fa7914c2282eddae24700" +dependencies = [ + "target-lexicon", +] + +[[package]] +name = "pyo3-ffi" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e7ff73c555c286d212d4ce7e5fbda845a973c4f61bd5f160f5f2c3c2a04d1c4" +dependencies = [ + "libc", + "pyo3-build-config", +] + +[[package]] +name = "pyo3-macros" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c46d2e5862c7bb8207e3b639f713dda3c499933444784154c11d013c5a691b60" +dependencies = [ + "proc-macro2", + "pyo3-macros-backend", + "quote", + "syn", +] + +[[package]] +name = "pyo3-macros-backend" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38b89f066f2ae72a2aef52d1537ca2fcd104440ebfadee06e97aa90738a25b9b" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "smithy-checksums" +version = "0.0.0" +dependencies = [ + "crc-fast", + "pyo3", +] + +[[package]] +name = "spin" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" + +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "target-lexicon" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" diff --git a/packages/smithy-checksums/Cargo.toml b/packages/smithy-checksums/Cargo.toml new file mode 100644 index 000000000..7fb945a43 --- /dev/null +++ b/packages/smithy-checksums/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "smithy-checksums" +version = "0.0.0" +edition = "2024" +rust-version = "1.89" +publish = false +license = "Apache-2.0" + +[lib] +name = "_native" +crate-type = ["cdylib"] +path = "rust/lib.rs" + +[dependencies] +crc-fast = { version = "1.10", default-features = false, features = ["std"] } +pyo3 = { version = "0.30", features = ["abi3-py312", "abi3t-py315"] } + +[profile.release] +codegen-units = 1 +lto = true diff --git a/packages/smithy-checksums/LICENSE-THIRD-PARTY b/packages/smithy-checksums/LICENSE-THIRD-PARTY new file mode 100644 index 000000000..bf00306c1 --- /dev/null +++ b/packages/smithy-checksums/LICENSE-THIRD-PARTY @@ -0,0 +1,1034 @@ +Third-party software compiled into the smithy-checksums native extension. + +This file is generated from Cargo.lock by `make third-party-licenses`. Don't edit +it by hand. + +================================================================================ +Apache License 2.0 (Apache-2.0) + +Used by: + - crc-fast 1.10.0 (https://github.com/awesomized/crc-fast-rust) + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +================================================================================ +Apache License 2.0 (Apache-2.0) + +Used by: + - typenum 1.20.1 (https://github.com/paholg/typenum) + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright 2014 Paho Lurie-Gregg + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +================================================================================ +Apache License 2.0 (Apache-2.0) + +Used by: + - heck 0.5.0 (https://github.com/withoutboats/heck) + - once_cell 1.21.4 (https://github.com/matklad/once_cell) + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + +================================================================================ +Apache License 2.0 (Apache-2.0) + +Used by: + - crypto-common 0.1.7 (https://github.com/RustCrypto/traits) + - digest 0.10.7 (https://github.com/RustCrypto/traits) + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + +================================================================================ +Apache License 2.0 (Apache-2.0) + +Used by: + - libc 0.2.191 (https://github.com/rust-lang/libc) + - portable-atomic 1.15.0 (https://github.com/taiki-e/portable-atomic) + - proc-macro2 1.0.107 (https://github.com/dtolnay/proc-macro2) + - pyo3-ffi 0.30.1 (https://github.com/pyo3/pyo3) + - pyo3-macros-backend 0.30.1 (https://github.com/pyo3/pyo3) + - pyo3-macros 0.30.1 (https://github.com/pyo3/pyo3) + - pyo3 0.30.1 (https://github.com/pyo3/pyo3) + - quote 1.0.47 (https://github.com/dtolnay/quote) + - syn 3.0.9 (https://github.com/dtolnay/syn) + - unicode-ident 1.0.27 (https://github.com/dtolnay/unicode-ident) + +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. + +"Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: + + (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. + + You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + +To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + +================================================================================ +MIT License (MIT) + +Used by: + - spin 0.10.1 (https://github.com/mvdnes/spin-rs.git) + +The MIT License (MIT) + +Copyright (c) 2014 Mathijs van de Nes + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +================================================================================ +MIT License (MIT) + +Used by: + - generic-array 0.14.7 (https://github.com/fizyk20/generic-array.git) + +The MIT License (MIT) + +Copyright (c) 2015 Bartłomiej Kamiński + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +================================================================================ +Unicode License v3 (Unicode-3.0) + +Used by: + - unicode-ident 1.0.27 (https://github.com/dtolnay/unicode-ident) + +UNICODE LICENSE V3 + +COPYRIGHT AND PERMISSION NOTICE + +Copyright © 1991-2023 Unicode, Inc. + +NOTICE TO USER: Carefully read the following legal agreement. BY +DOWNLOADING, INSTALLING, COPYING OR OTHERWISE USING DATA FILES, AND/OR +SOFTWARE, YOU UNEQUIVOCALLY ACCEPT, AND AGREE TO BE BOUND BY, ALL OF THE +TERMS AND CONDITIONS OF THIS AGREEMENT. IF YOU DO NOT AGREE, DO NOT +DOWNLOAD, INSTALL, COPY, DISTRIBUTE OR USE THE DATA FILES OR SOFTWARE. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of data files and any associated documentation (the "Data Files") or +software and any associated documentation (the "Software") to deal in the +Data Files or Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, and/or sell +copies of the Data Files or Software, and to permit persons to whom the +Data Files or Software are furnished to do so, provided that either (a) +this copyright and permission notice appear with all copies of the Data +Files or Software, or (b) this copyright and permission notice appear in +associated Documentation. + +THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF +THIRD PARTY RIGHTS. + +IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS NOTICE +BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, +OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, +WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, +ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THE DATA +FILES OR SOFTWARE. + +Except as contained in this notice, the name of a copyright holder shall +not be used in advertising or otherwise to promote the sale, use or other +dealings in these Data Files or Software without prior written +authorization of the copyright holder. + diff --git a/packages/smithy-checksums/NOTICE b/packages/smithy-checksums/NOTICE new file mode 100644 index 000000000..616fc5889 --- /dev/null +++ b/packages/smithy-checksums/NOTICE @@ -0,0 +1 @@ +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. diff --git a/packages/smithy-checksums/README.md b/packages/smithy-checksums/README.md new file mode 100644 index 000000000..227cf26c2 --- /dev/null +++ b/packages/smithy-checksums/README.md @@ -0,0 +1,31 @@ +# smithy-checksums + +This package provides the checksum algorithms used by Smithy clients: CRC32 and +CRC32C, and, through `new()`, SHA-1, SHA-256, SHA-512, and MD5. + +```python +import smithy_checksums + +checksum = smithy_checksums.new("CRC32C") +checksum.update(b"hello world") +checksum.digest() +``` + +## Native acceleration + +The CRC algorithms are implemented in Rust and published as wheels for +common platforms. Where no wheel is available, the package installs as pure Python +and produces identical results, much more slowly. `HAS_NATIVE_EXTENSION` reports +which implementation is in use. CRC32 always uses `zlib` in pure Python, and the +SHA and MD5 algorithms always use `hashlib`. + +## Building + +Wheels are built with [maturin](https://www.maturin.rs/) when `cargo` is on the +`PATH`, and as pure Python when no Rust toolchain is installed. If Rust is installed +but can't build the extension, for example because it's older than the +`rust-version` in `Cargo.toml`, the build fails with instructions instead of +silently falling back. + +Set `SMITHY_PYTHON_NATIVE_BUILD` to `never` to build a pure-Python wheel, or to +`always` to require the native extension even when `cargo` isn't on the `PATH`. diff --git a/packages/smithy-checksums/about.hbs b/packages/smithy-checksums/about.hbs new file mode 100644 index 000000000..d23cd9c79 --- /dev/null +++ b/packages/smithy-checksums/about.hbs @@ -0,0 +1,16 @@ +Third-party software compiled into the smithy-checksums native extension. + +This file is generated from Cargo.lock by `make third-party-licenses`. Don't edit +it by hand. +{{#each licenses}} + +================================================================================ +{{name}} ({{id}}) + +Used by: +{{#each used_by}} + - {{crate.name}} {{crate.version}}{{#if crate.repository}} ({{crate.repository}}){{/if}} +{{/each}} + +{{text}} +{{/each}} diff --git a/packages/smithy-checksums/about.toml b/packages/smithy-checksums/about.toml new file mode 100644 index 000000000..9279ab1b3 --- /dev/null +++ b/packages/smithy-checksums/about.toml @@ -0,0 +1,11 @@ +# Configuration for cargo-about, which generates LICENSE-THIRD-PARTY. +accepted = [ + "Apache-2.0", + "Apache-2.0 WITH LLVM-exception", + "MIT", + "Unicode-3.0", +] +ignore-build-dependencies = true +ignore-dev-dependencies = true +# Our own crate is covered by the package license. +private = { ignore = true } diff --git a/packages/smithy-checksums/build_backend/smithy_checksums_build.py b/packages/smithy-checksums/build_backend/smithy_checksums_build.py new file mode 100644 index 000000000..c41cdc8ad --- /dev/null +++ b/packages/smithy-checksums/build_backend/smithy_checksums_build.py @@ -0,0 +1,151 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""PEP 517 build backend for smithy-checksums. + +Wheels are built with maturin, including the native extension, when a Rust +toolchain is available. Without one, they're built with hatchling and contain only +the pure-Python implementation, which the package also falls back to at runtime. +If a Rust toolchain is available but can't build the extension, for example because +it's too old, the build fails with instructions rather than silently falling back. + +Source distributions are always built with hatchling, since maturin needs cargo to +build them. The contents are the same either way. + +Set ``SMITHY_PYTHON_NATIVE_BUILD`` to control which backend builds wheels: + +* ``auto`` (the default): maturin if ``cargo`` is on the ``PATH``, else hatchling. +* ``always``: maturin, failing if the extension can't be built. +* ``never``: hatchling, producing a ``py3-none-any`` wheel. +""" + +import os +import re +import shutil +import subprocess +import sys +import tomllib +from importlib import import_module +from pathlib import Path + +type _ConfigSettings = dict[str, str | list[str]] | None +type _Version = tuple[int, ...] + +_ENV_VAR = "SMITHY_PYTHON_NATIVE_BUILD" +_MATURIN = "maturin>=1.10,<2" +_HATCHLING = "hatchling" +_CARGO_TOML = Path(__file__).parent.parent / "Cargo.toml" +_PURE_PYTHON_HINT = ( + f"To install the pure-Python implementation instead, set {_ENV_VAR}=never. It " + "produces identical checksums, but CRC32C is much slower." +) + + +def _build_native() -> bool: + mode = os.environ.get(_ENV_VAR, "auto").strip().lower() + match mode: + case "auto": + if shutil.which("cargo") is None: + return False + case "always": + pass + case "never": + return False + case _: + raise ValueError( + f"{_ENV_VAR} must be 'auto', 'always', or 'never', not {mode!r}" + ) + _check_rust_toolchain() + return True + + +def _check_rust_toolchain() -> None: + """Fails early, with instructions, if the Rust toolchain can't build the extension.""" + minimum = _minimum_rust_version() + required = ".".join(map(str, minimum)) + rustc = shutil.which("rustc") + if shutil.which("cargo") is None or rustc is None: + raise RuntimeError( + f"Building the smithy-checksums native extension requires Rust {required} " + f"or newer, but no Rust toolchain was found. Install one from " + f"https://rustup.rs. {_PURE_PYTHON_HINT}" + ) + found = _rustc_version(rustc) + # If the version can't be determined, let the build itself decide. + if found is not None and found < minimum: + raise RuntimeError( + f"Building the smithy-checksums native extension requires Rust {required} " + f"or newer, but found {'.'.join(map(str, found))}. Upgrade Rust, for " + f"example with `rustup update`. {_PURE_PYTHON_HINT}" + ) + + +def _minimum_rust_version() -> _Version: + with _CARGO_TOML.open("rb") as manifest: + rust_version: str = tomllib.load(manifest)["package"]["rust-version"] + return tuple(int(part) for part in rust_version.split(".")) + + +def _rustc_version(rustc: str) -> _Version | None: + try: + result = subprocess.run( # noqa: S603 + [rustc, "--version"], capture_output=True, text=True, timeout=60 + ) + except (OSError, subprocess.SubprocessError): + return None + return _parse_rustc_version(result.stdout) + + +def _parse_rustc_version(output: str) -> _Version | None: + match = re.match(r"rustc (\d+)\.(\d+)\.(\d+)", output) + return None if match is None else tuple(int(part) for part in match.groups()) + + +def _build(hook: str, wheel_directory: str, config_settings: _ConfigSettings) -> str: + if not _build_native(): + backend = import_module("hatchling.build") + return getattr(backend, hook)(wheel_directory, config_settings) + try: + return getattr(import_module("maturin"), hook)(wheel_directory, config_settings) + except (Exception, SystemExit): + # maturin prints the compiler output and exits the process on failure. + sys.stderr.write( + "\nBuilding the smithy-checksums native extension failed. " + f"{_PURE_PYTHON_HINT}\n" + ) + raise + + +def get_requires_for_build_wheel(config_settings: _ConfigSettings = None) -> list[str]: + return [_MATURIN] if _build_native() else [_HATCHLING] + + +def get_requires_for_build_editable( + config_settings: _ConfigSettings = None, +) -> list[str]: + return [_MATURIN] if _build_native() else [_HATCHLING, "editables"] + + +def get_requires_for_build_sdist(config_settings: _ConfigSettings = None) -> list[str]: + return [_HATCHLING] + + +def build_wheel( + wheel_directory: str, + config_settings: _ConfigSettings = None, + metadata_directory: str | None = None, +) -> str: + return _build("build_wheel", wheel_directory, config_settings) + + +def build_editable( + wheel_directory: str, + config_settings: _ConfigSettings = None, + metadata_directory: str | None = None, +) -> str: + return _build("build_editable", wheel_directory, config_settings) + + +def build_sdist(sdist_directory: str, config_settings: _ConfigSettings = None) -> str: + return import_module("hatchling.build").build_sdist( + sdist_directory, config_settings + ) diff --git a/packages/smithy-checksums/deny.toml b/packages/smithy-checksums/deny.toml new file mode 100644 index 000000000..b737758ca --- /dev/null +++ b/packages/smithy-checksums/deny.toml @@ -0,0 +1,24 @@ +# Configuration for cargo-deny, which checks the dependencies of the native extension. + +[advisories] +version = 2 +yanked = "deny" + +[licenses] +version = 2 +allow = [ + "Apache-2.0", + "Apache-2.0 WITH LLVM-exception", + "MIT", + "Unicode-3.0", +] +# Our own crate is covered by the package license. +private = { ignore = true } + +[bans] +multiple-versions = "warn" +wildcards = "deny" + +[sources] +unknown-registry = "deny" +unknown-git = "deny" diff --git a/packages/smithy-checksums/pyproject.toml b/packages/smithy-checksums/pyproject.toml new file mode 100644 index 000000000..91ed1fd03 --- /dev/null +++ b/packages/smithy-checksums/pyproject.toml @@ -0,0 +1,81 @@ +[project] +name = "smithy-checksums" +# The version must stay static: both build backends read it from here. +version = "0.0.0" +requires-python = ">=3.12" +authors = [ + {name = "Amazon Web Services"}, +] +description = "Checksum algorithms for Smithy tooling, with native acceleration." +readme = "README.md" +license = {text = "Apache License 2.0"} +keywords = ["smithy", "sdk", "checksum", "crc32c"] +classifiers = [ + "Development Status :: 2 - Pre-Alpha", + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + "Natural Language :: English", + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Programming Language :: Python :: 3.15", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Free Threading :: 2 - Beta", + "Programming Language :: Rust", + "Topic :: Software Development :: Libraries" +] +dependencies = [] + +[project.urls] +"Changelog" = "https://github.com/smithy-lang/smithy-python/blob/develop/packages/smithy-checksums/CHANGELOG.md" +"Code" = "https://github.com/smithy-lang/smithy-python/blob/develop/packages/smithy-checksums/" +"Issue tracker" = "https://github.com/smithy-lang/smithy-python/issues" + +# Wheels are built with maturin when a Rust toolchain is available and with +# hatchling (pure Python) otherwise. See build_backend/smithy_checksums_build.py. +[build-system] +requires = [] +build-backend = "smithy_checksums_build" +backend-path = ["build_backend"] + +[tool.maturin] +python-source = "src" +module-name = "smithy_checksums._native" +features = ["pyo3/extension-module"] + +[tool.hatch.build.targets.wheel] +packages = ["src/smithy_checksums"] +exclude = ["*.so", "*.pyd"] + +[tool.hatch.build.targets.sdist] +include = [ + "src", + "rust", + "build_backend", + "Cargo.toml", + "Cargo.lock", + "CHANGELOG.md", + "LICENSE-THIRD-PARTY", + "NOTICE", + "README.md", +] +exclude = ["*.so", "*.pyd"] + +[tool.uv] +# Rebuild the native extension in the workspace when the Rust sources change. +cache-keys = [ + { file = "pyproject.toml" }, + { file = "Cargo.toml" }, + { file = "Cargo.lock" }, + { file = "rust/**/*.rs" }, + { file = "build_backend/*.py" }, + { env = "SMITHY_PYTHON_NATIVE_BUILD" }, +] + +[tool.ruff] +src = ["src"] diff --git a/packages/smithy-checksums/rust/lib.rs b/packages/smithy-checksums/rust/lib.rs new file mode 100644 index 000000000..4ed6086c1 --- /dev/null +++ b/packages/smithy-checksums/rust/lib.rs @@ -0,0 +1,135 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Native checksum implementations for `smithy_checksums`. +//! +//! Each class mirrors the pure-Python implementation of the same name in +//! `smithy_checksums._private.crc`, so the two are interchangeable. + +use std::sync::{Mutex, MutexGuard}; + +use pyo3::buffer::PyBuffer; +use pyo3::prelude::*; +use pyo3::types::PyBytes; + +/// Inputs at least this large are hashed with the interpreter detached so +/// other threads can run. Below it, detaching costs more than it saves. This +/// matches the threshold `hashlib` uses. +const DETACH_THRESHOLD: usize = 2048; + +/// Streaming checksum state shared by every class in this module. +trait State: Clone + Send { + fn new() -> Self; + fn update(&mut self, data: &[u8]); + fn digest(&self) -> Vec; +} + +macro_rules! crc_state { + ($state:ident, $algorithm:ident, $size:literal) => { + #[derive(Clone)] + struct $state(crc_fast::Digest); + + impl State for $state { + fn new() -> Self { + Self(crc_fast::Digest::new(crc_fast::CrcAlgorithm::$algorithm)) + } + + fn update(&mut self, data: &[u8]) { + self.0.update(data); + } + + fn digest(&self) -> Vec { + // The digest is a u64 regardless of the CRC's width. + self.0.finalize().to_be_bytes()[8 - $size..].to_vec() + } + } + }; +} + +crc_state!(Crc32State, Crc32IsoHdlc, 4); +crc_state!(Crc32cState, Crc32Iscsi, 4); + +fn lock(state: &Mutex) -> MutexGuard<'_, S> { + // A panic while holding the lock can't leave a checksum half-updated in a + // way that matters more than the panic itself, so ignore poisoning. + state + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) +} + +/// Feeds the bytes of `data` to `state`. +/// +/// `bytes` and other read-only buffers are hashed in place. Writable buffers +/// (such as `bytearray`) are copied first, because another thread could +/// otherwise modify them while they're being read. +fn update(py: Python<'_>, state: &Mutex, data: &Bound<'_, PyAny>) -> PyResult<()> { + if let Ok(bytes) = data.cast::() { + hash_slice(py, state, bytes.as_bytes()); + return Ok(()); + } + + let buffer = PyBuffer::::get(data)?; + if buffer.readonly() && buffer.is_c_contiguous() { + // SAFETY: the buffer is read-only and C-contiguous, and `buffer` + // keeps the exporter alive and its memory pinned until it's dropped + // at the end of this function. + let slice = unsafe { + std::slice::from_raw_parts(buffer.buf_ptr().cast::(), buffer.len_bytes()) + }; + hash_slice(py, state, slice); + } else { + let copy = buffer.to_vec(py)?; + hash_slice(py, state, ©); + } + Ok(()) +} + +fn hash_slice(py: Python<'_>, state: &Mutex, data: &[u8]) { + if data.len() < DETACH_THRESHOLD { + lock(state).update(data); + } else { + py.detach(|| lock(state).update(data)); + } +} + +macro_rules! checksum_class { + ($name:ident, $state:ident) => { + #[pyclass(frozen, module = "smithy_checksums._native")] + struct $name { + state: Mutex<$state>, + } + + #[pymethods] + impl $name { + #[new] + fn new() -> Self { + Self { + state: Mutex::new($state::new()), + } + } + + fn update(&self, py: Python<'_>, data: &Bound<'_, PyAny>) -> PyResult<()> { + update(py, &self.state, data) + } + + fn digest<'py>(&self, py: Python<'py>) -> Bound<'py, PyBytes> { + PyBytes::new(py, &lock(&self.state).digest()) + } + + fn copy(&self) -> Self { + Self { + state: Mutex::new(lock(&self.state).clone()), + } + } + } + }; +} + +checksum_class!(Crc32, Crc32State); +checksum_class!(Crc32C, Crc32cState); + +#[pymodule] +mod _native { + #[pymodule_export] + use super::{Crc32, Crc32C}; +} diff --git a/packages/smithy-checksums/src/smithy_checksums/__init__.py b/packages/smithy-checksums/src/smithy_checksums/__init__.py new file mode 100644 index 000000000..d865b9e9d --- /dev/null +++ b/packages/smithy-checksums/src/smithy_checksums/__init__.py @@ -0,0 +1,144 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Checksum algorithms for Smithy clients. + +CRC32 and CRC32C use a native extension when one is available for the platform +and fall back to pure Python otherwise. The fallback produces identical results +but is orders of magnitude slower, except for CRC32, which uses :mod:`zlib`. SHA-1, SHA-256, SHA-512, and MD5 are available +through :func:`new` and always use :mod:`hashlib`. +""" + +import hashlib +from collections.abc import Buffer +from importlib.metadata import version +from typing import ClassVar, Final, Protocol, Self + +from ._private.backend import IMPLEMENTATIONS as _IMPLEMENTATIONS +from ._private.backend import NATIVE as _NATIVE +from ._private.backend import ChecksumImpl as _ChecksumImpl +from ._private.buffers import as_byte_view as _as_byte_view + +__version__ = version("smithy-checksums") +__all__ = ( + "CRC32", + "CRC32C", + "HAS_NATIVE_EXTENSION", + "Checksum", + "new", +) + +HAS_NATIVE_EXTENSION: Final[bool] = _NATIVE +"""Whether the native extension is in use. + +When it isn't, CRC32C is computed in pure Python. +""" + + +class Checksum(Protocol): + """A streaming checksum. + + Objects returned by :mod:`hashlib` also satisfy this protocol. + """ + + @property + def name(self) -> str: + """The lowercase name of the algorithm, such as ``crc32c``.""" + ... + + @property + def digest_size(self) -> int: + """The size of the digest in bytes.""" + ... + + def update(self, data: Buffer, /) -> None: + """Feeds bytes to the checksum. + + :param data: The bytes to add. Buffers that aren't C-contiguous are rejected. + """ + ... + + def digest(self) -> bytes: + """Returns the checksum of the data passed to :meth:`update` so far. + + The digest is big-endian, the byte order used by checksum headers such as + ``x-amz-checksum-crc32c``. Calling this doesn't reset the checksum. + """ + ... + + def copy(self) -> Self: + """Returns an independent copy of the checksum's current state.""" + ... + + +class _BackendChecksum: + __slots__ = ("_impl",) + + name: ClassVar[str] + digest_size: ClassVar[int] + + _impl: _ChecksumImpl + + def __init__(self) -> None: + self._impl = _IMPLEMENTATIONS[self.name]() + + def update(self, data: Buffer, /) -> None: + """Feeds bytes to the checksum. + + :param data: The bytes to add. Buffers that aren't C-contiguous are rejected. + """ + self._impl.update(_as_byte_view(data)) + + def digest(self) -> bytes: + """Returns the big-endian checksum of the data passed to :meth:`update` so far. + + Calling this doesn't reset the checksum. + """ + return self._impl.digest() + + def copy(self) -> Self: + """Returns an independent copy of the checksum's current state.""" + new = object.__new__(type(self)) + new._impl = self._impl.copy() + return new + + +class CRC32(_BackendChecksum): + """CRC-32 (ISO-HDLC), the variant used by zlib and gzip.""" + + __slots__ = () + + name = "crc32" + digest_size = 4 + + +class CRC32C(_BackendChecksum): + """CRC-32C (Castagnoli).""" + + __slots__ = () + + name = "crc32c" + digest_size = 4 + + +_CHECKSUMS: Final = {cls.name: cls for cls in (CRC32, CRC32C)} +_HASHLIB_ALGORITHMS: Final = frozenset(("sha1", "sha256", "sha512")) + + +def new(name: str) -> Checksum: + """Creates a checksum from the name of its algorithm. + + Names are case-insensitive, so the values of a checksum algorithm enum such as + ``CRC32C`` or ``SHA256`` can be passed directly. + + :param name: One of ``crc32``, ``crc32c``, ``sha1``, ``sha256``, ``sha512``, or + ``md5``. + :raises ValueError: If the algorithm isn't supported. + """ + key = name.lower() + if (checksum := _CHECKSUMS.get(key)) is not None: + return checksum() + if key in _HASHLIB_ALGORITHMS: + return hashlib.new(key) + if key == "md5": + return hashlib.md5(usedforsecurity=False) + raise ValueError(f"Unsupported checksum algorithm: {name!r}") diff --git a/packages/smithy-checksums/src/smithy_checksums/_native.pyi b/packages/smithy-checksums/src/smithy_checksums/_native.pyi new file mode 100644 index 000000000..1163b9bb9 --- /dev/null +++ b/packages/smithy-checksums/src/smithy_checksums/_native.pyi @@ -0,0 +1,17 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Type stubs for the native extension built from ``rust/lib.rs``.""" + +from typing import Self + +type _Bytes = bytes | bytearray | memoryview + +class Crc32: + def update(self, data: _Bytes, /) -> None: ... + def digest(self) -> bytes: ... + def copy(self) -> Self: ... + +class Crc32C: + def update(self, data: _Bytes, /) -> None: ... + def digest(self) -> bytes: ... + def copy(self) -> Self: ... diff --git a/packages/smithy-checksums/src/smithy_checksums/_private/__init__.py b/packages/smithy-checksums/src/smithy_checksums/_private/__init__.py new file mode 100644 index 000000000..33cbe867a --- /dev/null +++ b/packages/smithy-checksums/src/smithy_checksums/_private/__init__.py @@ -0,0 +1,2 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/packages/smithy-checksums/src/smithy_checksums/_private/backend.py b/packages/smithy-checksums/src/smithy_checksums/_private/backend.py new file mode 100644 index 000000000..c542afd87 --- /dev/null +++ b/packages/smithy-checksums/src/smithy_checksums/_private/backend.py @@ -0,0 +1,42 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Selects the native checksum implementations, or the pure-Python ones if unavailable.""" + +from collections.abc import Callable, Mapping +from typing import Protocol, Self + + +class ChecksumImpl(Protocol): + """The interface shared by the native and pure-Python implementations. + + Callers normalize input with :func:`.buffers.as_byte_view` first. + """ + + def update(self, data: bytes | bytearray | memoryview, /) -> None: ... + + def digest(self) -> bytes: ... + + def copy(self) -> Self: ... + + +type ImplFactory = Callable[[], ChecksumImpl] + + +def _load() -> tuple[bool, Mapping[str, ImplFactory]]: + try: + from smithy_checksums import _native + except ImportError: + from smithy_checksums._private import crc + + return False, { + "crc32": crc.Crc32, + "crc32c": crc.Crc32C, + } + + return True, { + "crc32": _native.Crc32, + "crc32c": _native.Crc32C, + } + + +NATIVE, IMPLEMENTATIONS = _load() diff --git a/packages/smithy-checksums/src/smithy_checksums/_private/buffers.py b/packages/smithy-checksums/src/smithy_checksums/_private/buffers.py new file mode 100644 index 000000000..ee21e54be --- /dev/null +++ b/packages/smithy-checksums/src/smithy_checksums/_private/buffers.py @@ -0,0 +1,23 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +from collections.abc import Buffer + + +def as_bytes(data: Buffer) -> bytes | bytearray: + """Returns the bytes of ``data``, copying only if it isn't already bytes-like.""" + if isinstance(data, bytes | bytearray): + return data + return memoryview(data).tobytes() + + +def as_byte_view(data: Buffer) -> bytes | bytearray | memoryview: + """Returns ``data`` as an object the native extension accepts, without copying. + + The native extension reads any buffer with a byte format. Other formats (such + as an ``array('I')``) are recast to bytes, which requires a C-contiguous buffer, + the same requirement :mod:`hashlib` has. + """ + if isinstance(data, bytes | bytearray): + return data + view = memoryview(data) + return view if view.format == "B" else view.cast("B") diff --git a/packages/smithy-checksums/src/smithy_checksums/_private/crc.py b/packages/smithy-checksums/src/smithy_checksums/_private/crc.py new file mode 100644 index 000000000..92e5f3682 --- /dev/null +++ b/packages/smithy-checksums/src/smithy_checksums/_private/crc.py @@ -0,0 +1,78 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Pure-Python CRC implementations, used when the native extension is unavailable.""" + +import zlib +from typing import ClassVar, Self + +from .pure import PureChecksum + + +def _reflected_table(poly: int) -> tuple[int, ...]: + table: list[int] = [] + for byte in range(256): + crc = byte + for _ in range(8): + crc = (crc >> 1) ^ poly if crc & 1 else crc >> 1 + table.append(crc) + return tuple(table) + + +class Crc32(PureChecksum): + """CRC-32 (ISO-HDLC), backed by :func:`zlib.crc32`.""" + + __slots__ = ("_crc",) + + def __init__(self) -> None: + super().__init__() + self._crc = 0 + + def _update(self, data: bytes | bytearray) -> None: + self._crc = zlib.crc32(data, self._crc) + + def _digest(self) -> bytes: + return self._crc.to_bytes(4, "big") + + def _copy(self) -> Self: + new = type(self)() + new._crc = self._crc + return new + + +class _ReflectedCrc(PureChecksum): + """A table-driven CRC whose initial value and final XOR are all ones.""" + + __slots__ = ("_crc",) + + _table: ClassVar[tuple[int, ...]] + _mask: ClassVar[int] + _size: ClassVar[int] + + def __init__(self) -> None: + super().__init__() + self._crc = self._mask + + def _update(self, data: bytes | bytearray) -> None: + table = self._table + crc = self._crc + for byte in data: + crc = table[(crc ^ byte) & 0xFF] ^ (crc >> 8) + self._crc = crc + + def _digest(self) -> bytes: + return (self._crc ^ self._mask).to_bytes(self._size, "big") + + def _copy(self) -> Self: + new = type(self)() + new._crc = self._crc + return new + + +class Crc32C(_ReflectedCrc): + """CRC-32C (Castagnoli, also called iSCSI).""" + + __slots__ = () + + _table = _reflected_table(0x82F63B78) + _mask = 0xFFFFFFFF + _size = 4 diff --git a/packages/smithy-checksums/src/smithy_checksums/_private/pure.py b/packages/smithy-checksums/src/smithy_checksums/_private/pure.py new file mode 100644 index 000000000..e42bac3dd --- /dev/null +++ b/packages/smithy-checksums/src/smithy_checksums/_private/pure.py @@ -0,0 +1,42 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +from collections.abc import Buffer +from threading import Lock +from typing import Self + +from .buffers import as_bytes + + +class PureChecksum: + """Base class for the pure-Python implementations. + + Each method holds a lock, so an instance can be shared between threads like the + native implementations and :mod:`hashlib` objects can. Subclasses implement the + underscored methods, which are only called with the lock held. + """ + + __slots__ = ("_lock",) + + def __init__(self) -> None: + self._lock = Lock() + + def update(self, data: Buffer, /) -> None: + with self._lock: + self._update(as_bytes(data)) + + def digest(self) -> bytes: + with self._lock: + return self._digest() + + def copy(self) -> Self: + with self._lock: + return self._copy() + + def _update(self, data: bytes | bytearray) -> None: + raise NotImplementedError + + def _digest(self) -> bytes: + raise NotImplementedError + + def _copy(self) -> Self: + raise NotImplementedError diff --git a/packages/smithy-checksums/src/smithy_checksums/py.typed b/packages/smithy-checksums/src/smithy_checksums/py.typed new file mode 100644 index 000000000..f5642f79f --- /dev/null +++ b/packages/smithy-checksums/src/smithy_checksums/py.typed @@ -0,0 +1 @@ +Marker diff --git a/packages/smithy-checksums/tests/__init__.py b/packages/smithy-checksums/tests/__init__.py new file mode 100644 index 000000000..04f8b7b76 --- /dev/null +++ b/packages/smithy-checksums/tests/__init__.py @@ -0,0 +1,2 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/packages/smithy-checksums/tests/unit/__init__.py b/packages/smithy-checksums/tests/unit/__init__.py new file mode 100644 index 000000000..04f8b7b76 --- /dev/null +++ b/packages/smithy-checksums/tests/unit/__init__.py @@ -0,0 +1,2 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/packages/smithy-checksums/tests/unit/test_build_backend.py b/packages/smithy-checksums/tests/unit/test_build_backend.py new file mode 100644 index 000000000..85c6b9c7d --- /dev/null +++ b/packages/smithy-checksums/tests/unit/test_build_backend.py @@ -0,0 +1,162 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +import importlib.util +import shutil +from pathlib import Path +from types import ModuleType, SimpleNamespace + +import pytest + +_BACKEND_PATH = ( + Path(__file__).parents[2] / "build_backend" / "smithy_checksums_build.py" +) +_ENV_VAR = "SMITHY_PYTHON_NATIVE_BUILD" + + +@pytest.fixture +def backend() -> ModuleType: + spec = importlib.util.spec_from_file_location( + "smithy_checksums_build", _BACKEND_PATH + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _set_toolchain( + monkeypatch: pytest.MonkeyPatch, + backend: ModuleType, + *, + installed: bool = True, + version: tuple[int, ...] | None = (1, 93, 0), +) -> None: + tools = {"cargo": "/bin/cargo", "rustc": "/bin/rustc"} if installed else {} + monkeypatch.setattr(shutil, "which", tools.get) + monkeypatch.setattr(backend, "_rustc_version", lambda _: version) # type: ignore + + +def _set_mode(monkeypatch: pytest.MonkeyPatch, mode: str | None) -> None: + if mode is None: + monkeypatch.delenv(_ENV_VAR, raising=False) + else: + monkeypatch.setenv(_ENV_VAR, mode) + + +def _requires_native(backend: ModuleType) -> bool: + requires = backend.get_requires_for_build_wheel() + native = any(req.startswith("maturin") for req in requires) + assert ("hatchling" in requires) is not native + + editable_requires = backend.get_requires_for_build_editable() + assert ("editables" in editable_requires) is not native + return native + + +@pytest.mark.parametrize( + ("mode", "installed", "native"), + [ + (None, True, True), + (None, False, False), + ("auto", True, True), + ("AUTO", False, False), + ("always", True, True), + ("never", True, False), + ("never", False, False), + ], +) +def test_selects_backend( + backend: ModuleType, + monkeypatch: pytest.MonkeyPatch, + mode: str | None, + installed: bool, + native: bool, +) -> None: + _set_mode(monkeypatch, mode) + _set_toolchain(monkeypatch, backend, installed=installed) + assert _requires_native(backend) is native + + +@pytest.mark.parametrize("mode", [None, "never", "always"]) +def test_sdist_never_needs_rust( + backend: ModuleType, monkeypatch: pytest.MonkeyPatch, mode: str | None +) -> None: + _set_mode(monkeypatch, mode) + _set_toolchain(monkeypatch, backend, installed=False) + assert backend.get_requires_for_build_sdist() == ["hatchling"] + + +@pytest.mark.parametrize("mode", [None, "always"]) +def test_rejects_old_rust( + backend: ModuleType, monkeypatch: pytest.MonkeyPatch, mode: str | None +) -> None: + _set_mode(monkeypatch, mode) + _set_toolchain(monkeypatch, backend, version=(1, 70, 0)) + with pytest.raises(RuntimeError, match=r"requires Rust 1\.89.*found 1\.70\.0") as e: + backend.get_requires_for_build_wheel() + assert f"{_ENV_VAR}=never" in str(e.value) + + +def test_always_requires_rust( + backend: ModuleType, monkeypatch: pytest.MonkeyPatch +) -> None: + _set_mode(monkeypatch, "always") + _set_toolchain(monkeypatch, backend, installed=False) + with pytest.raises(RuntimeError, match="no Rust toolchain was found") as e: + backend.get_requires_for_build_wheel() + assert f"{_ENV_VAR}=never" in str(e.value) + + +def test_unknown_rust_version_builds_natively( + backend: ModuleType, monkeypatch: pytest.MonkeyPatch +) -> None: + _set_mode(monkeypatch, None) + _set_toolchain(monkeypatch, backend, version=None) + assert _requires_native(backend) + + +def test_failed_native_build_suggests_pure_python( + backend: ModuleType, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + def fail(wheel_directory: str, config_settings: object) -> str: + raise SystemExit(1) + + _set_mode(monkeypatch, None) + _set_toolchain(monkeypatch, backend) + monkeypatch.setattr( + backend, + "import_module", + lambda _: SimpleNamespace(build_wheel=fail), # type: ignore + ) + with pytest.raises(SystemExit): + backend.build_wheel("dist") + assert f"{_ENV_VAR}=never" in capsys.readouterr().err + + +def test_rejects_unknown_mode( + backend: ModuleType, monkeypatch: pytest.MonkeyPatch +) -> None: + _set_mode(monkeypatch, "sometimes") + with pytest.raises(ValueError, match="sometimes"): + backend.get_requires_for_build_wheel() + + +def test_minimum_rust_version_matches_cargo_toml(backend: ModuleType) -> None: + assert backend._minimum_rust_version() == (1, 89) + + +@pytest.mark.parametrize( + ("output", "version"), + [ + ("rustc 1.93.0 (254b59607 2026-01-19)\n", (1, 93, 0)), + ("rustc 1.95.0-nightly (0123abcde 2026-02-01)\n", (1, 95, 0)), + ("error: no default toolchain configured\n", None), + ("", None), + ], +) +def test_parses_rustc_version( + backend: ModuleType, output: str, version: tuple[int, ...] | None +) -> None: + assert backend._parse_rustc_version(output) == version diff --git a/packages/smithy-checksums/tests/unit/test_checksums.py b/packages/smithy-checksums/tests/unit/test_checksums.py new file mode 100644 index 000000000..567ab4672 --- /dev/null +++ b/packages/smithy-checksums/tests/unit/test_checksums.py @@ -0,0 +1,114 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +import array +import hashlib +import importlib.util +from concurrent.futures import ThreadPoolExecutor + +import pytest +import smithy_checksums +from smithy_checksums import ( + CRC32, + CRC32C, + Checksum, + new, +) + +from .vectors import ALGORITHMS, KNOWN_ANSWERS, PATTERN_DIGESTS, pattern + +CLASSES = { + "crc32": CRC32, + "crc32c": CRC32C, +} + + +@pytest.mark.parametrize("algorithm", ALGORITHMS) +def test_class_attributes(algorithm: str) -> None: + checksum = CLASSES[algorithm]() + assert checksum.name == algorithm + assert checksum.digest_size == len(checksum.digest()) + + +@pytest.mark.parametrize("algorithm", ALGORITHMS) +def test_known_answers(algorithm: str) -> None: + for data, expected in KNOWN_ANSWERS[algorithm].items(): + checksum = CLASSES[algorithm]() + checksum.update(data) + assert checksum.digest().hex() == expected + + +@pytest.mark.parametrize("algorithm", ALGORITHMS) +def test_copy_returns_same_type(algorithm: str) -> None: + checksum = CLASSES[algorithm]() + checksum.update(b"Hello ") + copied = checksum.copy() + copied.update(b"world") + assert type(copied) is type(checksum) + assert copied.digest().hex() == KNOWN_ANSWERS[algorithm][b"Hello world"] + assert checksum.digest().hex() != copied.digest().hex() + + +@pytest.mark.parametrize("algorithm", ALGORITHMS) +def test_accepts_non_byte_formats(algorithm: str) -> None: + data = array.array("I", range(1024)) + checksum = CLASSES[algorithm]() + checksum.update(data) + expected = CLASSES[algorithm]() + expected.update(data.tobytes()) + assert checksum.digest() == expected.digest() + + +@pytest.mark.parametrize("algorithm", ALGORITHMS) +def test_rejects_non_contiguous_buffers(algorithm: str) -> None: + with pytest.raises(TypeError): + CLASSES[algorithm]().update(memoryview(array.array("I", range(8)))[::2]) + + +@pytest.mark.parametrize("algorithm", ALGORITHMS) +def test_rejects_non_buffers(algorithm: str) -> None: + with pytest.raises(TypeError): + CLASSES[algorithm]().update("text") # type: ignore[arg-type] + + +@pytest.mark.parametrize("algorithm", ALGORITHMS) +def test_concurrent_checksums(algorithm: str) -> None: + data = pattern(16385) + + def compute(_: int) -> str: + checksum = CLASSES[algorithm]() + for offset in range(0, len(data), 4096): + checksum.update(data[offset : offset + 4096]) + return checksum.digest().hex() + + with ThreadPoolExecutor(max_workers=4) as executor: + results = set(executor.map(compute, range(8))) + assert results == {PATTERN_DIGESTS[algorithm][16385]} + + +@pytest.mark.parametrize( + ("name", "expected"), + [ + ("crc32", CRC32), + ("CRC32C", CRC32C), + ], +) +def test_new_returns_checksum_classes(name: str, expected: type[Checksum]) -> None: + assert type(new(name)) is expected + + +@pytest.mark.parametrize("name", ["SHA1", "sha256", "SHA512", "MD5"]) +def test_new_returns_hashlib_checksums(name: str) -> None: + checksum = new(name) + checksum.update(b"Hello world") + assert checksum.name == name.lower() + assert checksum.digest() == hashlib.new(name.lower(), b"Hello world").digest() + + +def test_new_rejects_unknown_algorithms() -> None: + with pytest.raises(ValueError, match="crc16"): + new("crc16") + + +def test_has_native_extension() -> None: + built = importlib.util.find_spec("smithy_checksums._native") is not None + assert smithy_checksums.HAS_NATIVE_EXTENSION is built diff --git a/packages/smithy-checksums/tests/unit/test_implementations.py b/packages/smithy-checksums/tests/unit/test_implementations.py new file mode 100644 index 000000000..a50b39184 --- /dev/null +++ b/packages/smithy-checksums/tests/unit/test_implementations.py @@ -0,0 +1,182 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Tests that run against both the native and pure-Python implementations. + +The native cases are skipped when the extension isn't built, but the pure-Python +cases always run, regardless of which implementation the package selected. +""" + +import os +import random +from collections.abc import Callable +from concurrent.futures import ThreadPoolExecutor +from types import ModuleType + +import pytest +from smithy_checksums._private import crc +from smithy_checksums._private.backend import ChecksumImpl + +from .vectors import ALGORITHMS, KNOWN_ANSWERS, PATTERN_DIGESTS, pattern + +type Factory = Callable[[], ChecksumImpl] + +PURE: dict[str, Factory] = { + "crc32": crc.Crc32, + "crc32c": crc.Crc32C, +} + + +def _load_native() -> ModuleType | None: + try: + from smithy_checksums import _native + except ImportError: + return None + return _native + + +_NATIVE_MODULE = _load_native() +NATIVE: dict[str, Factory] = ( + {} + if _NATIVE_MODULE is None + else { + "crc32": _NATIVE_MODULE.Crc32, + "crc32c": _NATIVE_MODULE.Crc32C, + } +) +requires_native = pytest.mark.skipif( + _NATIVE_MODULE is None, reason="native extension not built" +) + + +IMPLEMENTATIONS = [ + pytest.param(PURE, id="pure"), + pytest.param(NATIVE, id="native", marks=requires_native), +] + + +def _digest(factory: Factory, *chunks: bytes) -> str: + checksum = factory() + for chunk in chunks: + checksum.update(chunk) + return checksum.digest().hex() + + +@pytest.mark.parametrize("impls", IMPLEMENTATIONS) +@pytest.mark.parametrize("algorithm", ALGORITHMS) +def test_known_answers(impls: dict[str, Factory], algorithm: str) -> None: + for data, expected in KNOWN_ANSWERS[algorithm].items(): + assert _digest(impls[algorithm], data) == expected + + +@pytest.mark.parametrize("impls", IMPLEMENTATIONS) +@pytest.mark.parametrize("algorithm", ALGORITHMS) +def test_pattern_digests(impls: dict[str, Factory], algorithm: str) -> None: + for length, expected in PATTERN_DIGESTS[algorithm].items(): + assert _digest(impls[algorithm], pattern(length)) == expected, length + + +@pytest.mark.parametrize("impls", IMPLEMENTATIONS) +@pytest.mark.parametrize("algorithm", ALGORITHMS) +def test_streaming_matches_one_shot(impls: dict[str, Factory], algorithm: str) -> None: + data = pattern(16385) + expected = PATTERN_DIGESTS[algorithm][16385] + rng = random.Random(algorithm) + for _ in range(5): + chunks: list[bytes] = [] + offset = 0 + while offset < len(data): + size = rng.choice((1, 7, 63, 64, 65, 239, 240, 241, 1023, 1024, 4096)) + chunks.append(data[offset : offset + size]) + offset += size + assert _digest(impls[algorithm], *chunks) == expected + + +@pytest.mark.parametrize("impls", IMPLEMENTATIONS) +@pytest.mark.parametrize("algorithm", ALGORITHMS) +def test_copy_is_independent(impls: dict[str, Factory], algorithm: str) -> None: + data = pattern(2049) + original = impls[algorithm]() + original.update(data[:1000]) + copied = original.copy() + copied.update(data[1000:]) + assert copied.digest().hex() == _digest(impls[algorithm], data) + assert original.digest().hex() == _digest(impls[algorithm], data[:1000]) + + +@pytest.mark.parametrize("impls", IMPLEMENTATIONS) +@pytest.mark.parametrize("algorithm", ALGORITHMS) +def test_digest_does_not_reset(impls: dict[str, Factory], algorithm: str) -> None: + checksum = impls[algorithm]() + checksum.update(b"Hello ") + first = checksum.digest() + assert checksum.digest() == first + checksum.update(b"world") + assert checksum.digest().hex() == KNOWN_ANSWERS[algorithm][b"Hello world"] + + +@pytest.mark.parametrize("impls", IMPLEMENTATIONS) +@pytest.mark.parametrize("algorithm", ALGORITHMS) +@pytest.mark.parametrize( + "wrap", + [bytes, bytearray, memoryview], + ids=["bytes", "bytearray", "memoryview"], +) +def test_accepts_byte_buffers( + impls: dict[str, Factory], + algorithm: str, + wrap: Callable[[bytes], bytes | bytearray | memoryview], +) -> None: + # 4096 bytes is above the size at which the native extension detaches from + # the interpreter while hashing. + data = pattern(4096) + checksum = impls[algorithm]() + checksum.update(wrap(data)) + assert checksum.digest().hex() == _digest(impls[algorithm], data) + + +@pytest.mark.parametrize("impls", IMPLEMENTATIONS) +@pytest.mark.parametrize("algorithm", ALGORITHMS) +def test_shared_instance_updates_are_atomic( + impls: dict[str, Factory], algorithm: str +) -> None: + # Every update adds the same chunk, so the result doesn't depend on the order + # the threads run in, only on each update being applied whole. + chunk = pattern(4096) + threads, updates = 4, 8 + shared = impls[algorithm]() + + def run(_: int) -> None: + for _ in range(updates): + shared.update(chunk) + + with ThreadPoolExecutor(max_workers=threads) as executor: + list(executor.map(run, range(threads))) + assert shared.digest().hex() == _digest( + impls[algorithm], chunk * (threads * updates) + ) + + +@requires_native +@pytest.mark.parametrize("algorithm", ALGORITHMS) +def test_native_matches_pure(algorithm: str) -> None: + data = os.urandom(5000) + for length in [*range(300), 1023, 1024, 1025, 2047, 2048, 2049, 5000]: + assert _digest(NATIVE[algorithm], data[:length]) == _digest( + PURE[algorithm], data[:length] + ), length + + +@pytest.mark.parametrize("impls", IMPLEMENTATIONS) +@pytest.mark.parametrize("algorithm", ALGORITHMS) +def test_matches_crt(impls: dict[str, Factory], algorithm: str) -> None: + crt = pytest.importorskip("awscrt.checksums") + compute: dict[str, Callable[[bytes], bytes]] = { + "crc32": lambda data: crt.crc32(data).to_bytes(4, "big"), + "crc32c": lambda data: crt.crc32c(data).to_bytes(4, "big"), + } + data = os.urandom(3000) + for length in [*range(0, 300, 7), 1023, 1024, 1025, 3000]: + assert ( + _digest(impls[algorithm], data[:length]) + == compute[algorithm](data[:length]).hex() + ), length diff --git a/packages/smithy-checksums/tests/unit/vectors.py b/packages/smithy-checksums/tests/unit/vectors.py new file mode 100644 index 000000000..7697f57ce --- /dev/null +++ b/packages/smithy-checksums/tests/unit/vectors.py @@ -0,0 +1,67 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Expected digests shared by the checksum tests.""" + +ALGORITHMS = ("crc32", "crc32c") + +# Digests from independent sources: the standard check value of each CRC (the +# digest of "123456789"), the CRC32C test vectors from RFC 3720, and "Hello world" +# digests that match the ones the CRT produces. +KNOWN_ANSWERS: dict[str, dict[bytes, str]] = { + "crc32": {b"123456789": "cbf43926", b"Hello world": "8bd69e52"}, + "crc32c": { + b"123456789": "e3069283", + b"Hello world": "72b51f78", + # RFC 3720, appendix B.4. The RFC lists each CRC in transmission order, + # least significant byte first, so these are its values byte-reversed. + bytes(32): "8a9136aa", + b"\xff" * 32: "62a8ab43", + bytes(range(32)): "46dd794e", + bytes(range(31, -1, -1)): "113fdb5c", + }, +} + + +def pattern(length: int) -> bytes: + """Returns deterministic input that doesn't repeat on a power-of-two boundary.""" + return bytes(i % 251 for i in range(length)) + + +# Digests of pattern(length), from inputs shorter than a word to several +# kilobytes. +PATTERN_DIGESTS: dict[str, dict[int, str]] = { + "crc32": { + 0: "00000000", + 1: "d202ef8d", + 3: "0854897f", + 4: "8bb98613", + 8: "88aa689f", + 9: "bce14302", + 16: "cecee288", + 17: "2c183a19", + 128: "24650d57", + 129: "ca91cdf7", + 240: "a60b0b66", + 241: "cbc8d2f7", + 1024: "7be4dfd0", + 1025: "4e700dfb", + 16385: "e033bd2c", + }, + "crc32c": { + 0: "00000000", + 1: "527d5351", + 3: "92fd4bfa", + 4: "d9331aa3", + 8: "8a2cbc3b", + 9: "7144c5a8", + 16: "d9c908eb", + 17: "38435e17", + 128: "30d9c515", + 129: "f514629f", + 240: "9f4f71d6", + 241: "54fe7516", + 1024: "2af62c0c", + 1025: "c8d03add", + 16385: "896b2de8", + }, +} diff --git a/pyproject.toml b/pyproject.toml index cb3a1372e..729167a0f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,6 +42,7 @@ smithy_xml = { workspace = true } smithy_aws_core = { workspace = true } smithy_aws_event_stream = { workspace = true } aws_sdk_signers = {workspace = true } +smithy_checksums = { workspace = true } smithy_python = { workspace = true } [tool.pyright] diff --git a/scripts/api-snapshot/api.json b/scripts/api-snapshot/api.json index b904a65cf..9c312e6b2 100644 --- a/scripts/api-snapshot/api.json +++ b/scripts/api-snapshot/api.json @@ -677,6 +677,13 @@ "smithy_cbor.settings": { "CBORSettings": "class" }, + "smithy_checksums": { + "CRC32": "class", + "CRC32C": "class", + "Checksum": "protocol", + "HAS_NATIVE_EXTENSION": "bool", + "new": "function" + }, "smithy_core": { "HostType": "class", "SmithyError": "rex-exception", diff --git a/uv.lock b/uv.lock index 4cbf599d5..26a7be6e5 100644 --- a/uv.lock +++ b/uv.lock @@ -8,6 +8,7 @@ members = [ "smithy-aws-core", "smithy-aws-event-stream", "smithy-cbor", + "smithy-checksums", "smithy-core", "smithy-http", "smithy-json", @@ -715,6 +716,11 @@ dependencies = [ [package.metadata] requires-dist = [{ name = "smithy-core", editable = "packages/smithy-core" }] +[[package]] +name = "smithy-checksums" +version = "0.0.0" +source = { editable = "packages/smithy-checksums" } + [[package]] name = "smithy-core" source = { editable = "packages/smithy-core" } From 747250ea6d15d5ecf445878f096a58e8398298d8 Mon Sep 17 00:00:00 2001 From: jonathan343 Date: Sat, 24 Oct 2026 23:48:40 -0400 Subject: [PATCH 02/12] Pin the Rust toolchain and cache Rust builds in CI Pins the toolchain used for development and CI in rust-toolchain.toml, so a new stable release can't introduce clippy lints that fail CI without a change here. Builds from a source distribution are unaffected and use the installed toolchain. CI now caches Rust builds with Swatinem/rust-cache, saving caches only on develop, and adds a job that checks each native extension builds with the minimum Rust version its Cargo.toml declares. The new jobs also get names and version comments on their pinned actions. --- .github/workflows/ci.yml | 60 ++++++++++++++++++++++++++++++++++------ rust-toolchain.toml | 7 +++++ 2 files changed, 58 insertions(+), 9 deletions(-) create mode 100644 rust-toolchain.toml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 92803b6e1..4791a215a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,9 +53,17 @@ jobs: version: 3.8.2 - name: Set up Rust + # Installs the toolchain pinned in rust-toolchain.toml. run: | - rustup toolchain install stable --profile minimal --component clippy,rustfmt - rustup default stable + rustup toolchain install + + - name: Cache Rust builds + uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + with: + workspaces: packages/smithy-checksums + key: ${{ matrix.python-version }} + # Pull requests restore the caches develop saves, without adding their own. + save-if: ${{ github.ref == 'refs/heads/develop' }} - name: Setup workspace run: | @@ -68,7 +76,7 @@ jobs: - name: Install Rust dependency tools if: ${{ matrix.python-version == '3.12' }} - uses: taiki-e/install-action@1e29709c0ec63babbe8f266397a465f29a825f91 + uses: taiki-e/install-action@1e29709c0ec63babbe8f266397a465f29a825f91 # v2.87.31 with: tool: cargo-about@0.9.2,cargo-deny@0.20.2 @@ -112,6 +120,7 @@ jobs: # installed wheel. The release pipeline builds the published wheels; this # catches platform-specific build or test failures before then. native-wheels: + name: Native wheels (${{ matrix.package }}, ${{ matrix.os }}, Python ${{ matrix.python-version }}) runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -130,21 +139,29 @@ jobs: steps: - name: Checkout Repository - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - name: Install uv and set the Python version - uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 + uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 with: python-version: ${{ matrix.python-version }} activate-environment: true enable-cache: true - name: Set up Rust + # Installs the toolchain pinned in rust-toolchain.toml. run: | - rustup toolchain install stable --profile minimal - rustup default stable + rustup toolchain install + + - name: Cache Rust builds + uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + with: + workspaces: packages/${{ matrix.package }} + key: ${{ matrix.python-version }} + # Pull requests restore the caches develop saves, without adding their own. + save-if: ${{ github.ref == 'refs/heads/develop' }} - name: Build wheel run: | @@ -159,6 +176,7 @@ jobs: # Tests the pure-Python wheel, and installing from the sdist without a Rust # toolchain, which should fall back to the same pure-Python build. pure-python-fallback: + name: Pure-Python fallback (${{ matrix.package }}) runs-on: ubuntu-latest strategy: fail-fast: false @@ -169,12 +187,12 @@ jobs: steps: - name: Checkout Repository - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - name: Install uv and set the Python version - uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 + uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 with: python-version: "3.12" activate-environment: true @@ -200,3 +218,27 @@ jobs: uv pip install --python .venv-sdist --no-cache pytest awscrt dist/*.tar.gz .venv-sdist/bin/python -c "import smithy_checksums; assert not smithy_checksums.HAS_NATIVE_EXTENSION" .venv-sdist/bin/python -m pytest "packages/$PACKAGE/tests" --import-mode=importlib -o addopts="" + + # Checks each native extension builds with the minimum Rust version it declares, + # which source builds require. + rust-minimum-version: + name: Rust minimum version (${{ matrix.package }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + package: [smithy-checksums] + env: + PACKAGE: ${{ matrix.package }} + + steps: + - name: Checkout Repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Check with the minimum Rust version + run: | + version="$(sed -n 's/^rust-version = "\(.*\)"$/\1/p' "packages/$PACKAGE/Cargo.toml")" + rustup toolchain install "$version" --profile minimal + cargo "+$version" check --manifest-path "packages/$PACKAGE/Cargo.toml" --locked --all-targets diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 000000000..1bbb21e89 --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,7 @@ +# The Rust toolchain used to develop and test the native extensions in packages/. +# Builds from a source distribution use the installed toolchain instead, as long as +# it meets the package's rust-version. +[toolchain] +channel = "1.99.1" +components = ["clippy", "rustfmt"] +profile = "minimal" From 0a78d46a75f835e900c61906dce4d110a6a34e31 Mon Sep 17 00:00:00 2001 From: jonathan343 Date: Sat, 24 Oct 2026 23:59:15 -0400 Subject: [PATCH 03/12] Describe what smithy-checksums adds in its changelog entry --- ...ithy-checksums-feature-2c585fb742b2409690bb374cf08cd6cf.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/smithy-checksums/.changes/next-release/smithy-checksums-feature-2c585fb742b2409690bb374cf08cd6cf.json b/packages/smithy-checksums/.changes/next-release/smithy-checksums-feature-2c585fb742b2409690bb374cf08cd6cf.json index 6375c1caf..4a267588c 100644 --- a/packages/smithy-checksums/.changes/next-release/smithy-checksums-feature-2c585fb742b2409690bb374cf08cd6cf.json +++ b/packages/smithy-checksums/.changes/next-release/smithy-checksums-feature-2c585fb742b2409690bb374cf08cd6cf.json @@ -1,4 +1,4 @@ { "type": "feature", - "description": "Added the `smithy-checksums` package, which provides CRC32 and CRC32C checksums backed by a native Rust extension with a pure-Python fallback, and SHA-1, SHA-256, SHA-512, and MD5 checksums through `new()`." + "description": "Added CRC32 and CRC32C checksums, backed by a native Rust extension with a pure-Python fallback, and SHA-1, SHA-256, SHA-512, and MD5 checksums through `new()`." } \ No newline at end of file From 4ff33b9d9030a0d183d491a3d4b54965f11a8649 Mon Sep 17 00:00:00 2001 From: jonathan343 Date: Sun, 25 Oct 2026 00:57:07 -0400 Subject: [PATCH 04/12] Make the Checksum protocol runtime-checkable --- packages/smithy-checksums/src/smithy_checksums/__init__.py | 3 ++- packages/smithy-checksums/tests/unit/test_checksums.py | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/smithy-checksums/src/smithy_checksums/__init__.py b/packages/smithy-checksums/src/smithy_checksums/__init__.py index d865b9e9d..88eb88d5d 100644 --- a/packages/smithy-checksums/src/smithy_checksums/__init__.py +++ b/packages/smithy-checksums/src/smithy_checksums/__init__.py @@ -11,7 +11,7 @@ import hashlib from collections.abc import Buffer from importlib.metadata import version -from typing import ClassVar, Final, Protocol, Self +from typing import ClassVar, Final, Protocol, Self, runtime_checkable from ._private.backend import IMPLEMENTATIONS as _IMPLEMENTATIONS from ._private.backend import NATIVE as _NATIVE @@ -34,6 +34,7 @@ """ +@runtime_checkable class Checksum(Protocol): """A streaming checksum. diff --git a/packages/smithy-checksums/tests/unit/test_checksums.py b/packages/smithy-checksums/tests/unit/test_checksums.py index 567ab4672..ba17d1a4d 100644 --- a/packages/smithy-checksums/tests/unit/test_checksums.py +++ b/packages/smithy-checksums/tests/unit/test_checksums.py @@ -109,6 +109,12 @@ def test_new_rejects_unknown_algorithms() -> None: new("crc16") +def test_checksum_protocol_supports_isinstance() -> None: + assert isinstance(CRC32C(), Checksum) + assert isinstance(hashlib.sha256(), Checksum) + assert not isinstance(object(), Checksum) + + def test_has_native_extension() -> None: built = importlib.util.find_spec("smithy_checksums._native") is not None assert smithy_checksums.HAS_NATIVE_EXTENSION is built From 922e7a111341c41393bc74f63049ebe0b854d2ac Mon Sep 17 00:00:00 2001 From: jonathan343 Date: Tue, 27 Oct 2026 11:42:34 -0400 Subject: [PATCH 05/12] Speed up the pure-Python CRC32C by hashing chunks with big integers The byte-at-a-time table loop runs about a dozen bytecodes per byte, which caps it at roughly 15 MB/s however it's arranged; slicing-by-8 and 16-bit tables only reach 20-35 MB/s. A CRC without its initial value and final XOR is linear over GF(2), so each bit of the CRC of a fixed-size chunk is the parity of the chunk's bits under a fixed mask. Input is now buffered into 16 KiB chunks, each read as one integer, and every CRC bit is computed with an AND and bit_count(), which do the per-byte work in C. The masks are built lazily by doubling, in a few milliseconds. Remainders under 1 KiB still use the table. CRC32C goes from about 16 MB/s to about 210 MB/s, whatever the update size. --- .../src/smithy_checksums/_private/crc.py | 105 +++++++++++++++++- .../tests/unit/test_implementations.py | 36 ++++++ 2 files changed, 135 insertions(+), 6 deletions(-) diff --git a/packages/smithy-checksums/src/smithy_checksums/_private/crc.py b/packages/smithy-checksums/src/smithy_checksums/_private/crc.py index 92e5f3682..17819186c 100644 --- a/packages/smithy-checksums/src/smithy_checksums/_private/crc.py +++ b/packages/smithy-checksums/src/smithy_checksums/_private/crc.py @@ -3,10 +3,18 @@ """Pure-Python CRC implementations, used when the native extension is unavailable.""" import zlib +from functools import cache from typing import ClassVar, Self from .pure import PureChecksum +# Input is buffered and hashed this many bytes at a time with the masks from +# _chunk_masks. It must be a power of two. +_CHUNK_SIZE = 16384 +# Remainders shorter than this are hashed a byte at a time from the table, which +# is faster than a pass over the masks. +_MIN_MASKED_TAIL = 1024 + def _reflected_table(poly: int) -> tuple[int, ...]: table: list[int] = [] @@ -18,6 +26,50 @@ def _reflected_table(poly: int) -> tuple[int, ...]: return tuple(table) +def _apply(columns: list[int], value: int) -> int: + """Multiplies the GF(2) matrix with the given columns by ``value``.""" + result = 0 + for column in columns: + if value & 1: + result ^= column + value >>= 1 + return result + + +@cache +def _chunk_masks(table: tuple[int, ...], width: int) -> tuple[int, ...]: + """Returns masks that compute a CRC over a chunk with big-integer operations. + + Without its initial value and final XOR, a CRC is linear over GF(2), so each + bit of the CRC of a message is the parity of a fixed subset of the message's + bits. Bit ``r`` of the CRC of a ``_CHUNK_SIZE``-byte chunk, read as a + little-endian integer ``m``, is the parity of ``m & masks[r]``. This does the + per-byte work in C, which is many times faster than a table lookup per byte. + + The masks are built for one byte and then doubled: the CRC of ``a + b`` is the + CRC of ``a`` shifted through ``len(b)`` zero bytes, XORed with the CRC of + ``b``. The shift is linear too, and ``columns[s]`` holds its effect on bit + ``s`` of the CRC. + """ + masks = [0] * width + for bit in range(8): + for row in range(width): + if (table[1 << bit] >> row) & 1: + masks[row] |= 1 << bit + columns = [(table[(1 << s) & 0xFF] ^ ((1 << s) >> 8)) for s in range(width)] + length = 1 + while length < _CHUNK_SIZE: + shifted = [0] * width + for s, column in enumerate(columns): + for row in range(width): + if (column >> row) & 1: + shifted[row] ^= masks[s] + masks = [low | (high << (8 * length)) for low, high in zip(shifted, masks)] + columns = [_apply(columns, column) for column in columns] + length *= 2 + return tuple(masks) + + class Crc32(PureChecksum): """CRC-32 (ISO-HDLC), backed by :func:`zlib.crc32`.""" @@ -40,9 +92,14 @@ def _copy(self) -> Self: class _ReflectedCrc(PureChecksum): - """A table-driven CRC whose initial value and final XOR are all ones.""" + """A table-driven CRC whose initial value and final XOR are all ones. - __slots__ = ("_crc",) + Whole chunks are hashed with the masks from :func:`_chunk_masks`. Up to a + chunk of input is buffered, so that the speed doesn't depend on how the input + is split between updates. + """ + + __slots__ = ("_buffer", "_crc") _table: ClassVar[tuple[int, ...]] _mask: ClassVar[int] @@ -51,20 +108,56 @@ class _ReflectedCrc(PureChecksum): def __init__(self) -> None: super().__init__() self._crc = self._mask + self._buffer = bytearray() def _update(self, data: bytes | bytearray) -> None: - table = self._table + buffer = self._buffer + view = memoryview(data) + if buffer: + needed = _CHUNK_SIZE - len(buffer) + buffer += view[:needed] + if len(buffer) < _CHUNK_SIZE: + return + self._crc = self._hash_chunk(self._crc, buffer) + buffer.clear() + view = view[needed:] + end = len(view) - len(view) % _CHUNK_SIZE crc = self._crc - for byte in data: - crc = table[(crc ^ byte) & 0xFF] ^ (crc >> 8) + for offset in range(0, end, _CHUNK_SIZE): + crc = self._hash_chunk(crc, view[offset : offset + _CHUNK_SIZE]) self._crc = crc + buffer += view[end:] + + def _hash_chunk(self, crc: int, chunk: bytes | bytearray | memoryview) -> int: + """Returns the CRC after ``chunk``, which may be shorter than a chunk. + + A reflected CRC's state can be XORed into the next bytes of input and + cleared instead, and leading zero bytes don't change a CRC of zero. A short + chunk must still be at least as long as the CRC. + """ + message = (int.from_bytes(chunk, "little") ^ crc) << ( + 8 * (_CHUNK_SIZE - len(chunk)) + ) + result = 0 + for row, mask in enumerate(_chunk_masks(self._table, 8 * self._size)): + result |= ((message & mask).bit_count() & 1) << row + return result def _digest(self) -> bytes: - return (self._crc ^ self._mask).to_bytes(self._size, "big") + crc = self._crc + tail = self._buffer + if len(tail) >= _MIN_MASKED_TAIL: + crc = self._hash_chunk(crc, tail) + else: + table = self._table + for byte in tail: + crc = table[(crc ^ byte) & 0xFF] ^ (crc >> 8) + return (crc ^ self._mask).to_bytes(self._size, "big") def _copy(self) -> Self: new = type(self)() new._crc = self._crc + new._buffer = self._buffer.copy() return new diff --git a/packages/smithy-checksums/tests/unit/test_implementations.py b/packages/smithy-checksums/tests/unit/test_implementations.py index a50b39184..73975250f 100644 --- a/packages/smithy-checksums/tests/unit/test_implementations.py +++ b/packages/smithy-checksums/tests/unit/test_implementations.py @@ -180,3 +180,39 @@ def test_matches_crt(impls: dict[str, Factory], algorithm: str) -> None: _digest(impls[algorithm], data[:length]) == compute[algorithm](data[:length]).hex() ), length + + +def _bitwise_crc(poly: int, width: int, data: bytes) -> str: + """Computes a reflected CRC with all-ones init and final XOR, a bit at a time.""" + mask = (1 << width) - 1 + value = mask + for byte in data: + value ^= byte + for _ in range(8): + value = (value >> 1) ^ poly if value & 1 else value >> 1 + return (value ^ mask).to_bytes(width // 8, "big").hex() + + +@pytest.mark.parametrize( + ("algorithm", "poly", "width"), + [("crc32c", 0x82F63B78, 32)], +) +def test_chunked_crc_matches_bitwise(algorithm: str, poly: int, width: int) -> None: + # The pure CRCs hash whole chunks with masks, short tails with a table, and + # long tails with the masks again, so check lengths on either side of each. + chunk = crc._CHUNK_SIZE # pyright: ignore[reportPrivateUsage] + tail = crc._MIN_MASKED_TAIL # pyright: ignore[reportPrivateUsage] + data = os.urandom(2 * chunk + tail + 1) + for length in [0, 1, 8, tail - 1, tail, chunk - 1, chunk, chunk + 1, len(data)]: + assert _digest(PURE[algorithm], data[:length]) == _bitwise_crc( + poly, width, data[:length] + ), length + + rng = random.Random(algorithm) + chunks: list[bytes] = [] + offset = 0 + while offset < len(data): + size = rng.choice((1, 100, tail, chunk - 1, chunk, chunk + 1)) + chunks.append(data[offset : offset + size]) + offset += size + assert _digest(PURE[algorithm], *chunks) == _bitwise_crc(poly, width, data) From 5f3c8a28f516749b749778a24370f1305485b3c4 Mon Sep 17 00:00:00 2001 From: jonathan343 Date: Tue, 27 Oct 2026 16:45:29 -0400 Subject: [PATCH 06/12] Keep bytecode out of native wheels maturin packages everything under python-source, so a native wheel built from a source tree that's been imported from picked up its __pycache__ directories, including bytecode for modules that no longer exist. The hatchling targets already exclude build artifacts; this gives maturin the same treatment. --- packages/smithy-checksums/pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/smithy-checksums/pyproject.toml b/packages/smithy-checksums/pyproject.toml index 91ed1fd03..b9c3336fe 100644 --- a/packages/smithy-checksums/pyproject.toml +++ b/packages/smithy-checksums/pyproject.toml @@ -47,6 +47,7 @@ backend-path = ["build_backend"] python-source = "src" module-name = "smithy_checksums._native" features = ["pyo3/extension-module"] +exclude = ["**/__pycache__/**", "**/*.pyc"] [tool.hatch.build.targets.wheel] packages = ["src/smithy_checksums"] From 3b402e1a8cc289c0f127b63a76e975da7050adf6 Mon Sep 17 00:00:00 2001 From: jonathan343 Date: Tue, 27 Oct 2026 16:20:37 -0400 Subject: [PATCH 07/12] Copy non-bytes buffers instead of reading them in place The native extension read read-only buffers in place through an unsafe slice, on the grounds that nothing could change them. The read-only flag only stops this reader from writing, though: a read-only memoryview of a bytearray still changes when another thread writes to the bytearray, which can happen while the interpreter is detached. Only bytes is immutable, so it's the only input that's still hashed in place; every other buffer is copied. Copying runs at about 40 GB/s, against about 100 GB/s in place. Also: - State::digest returns a fixed-size array, given by an associated Digest type, instead of a Vec. - Import pyo3 items explicitly instead of through the prelude. --- packages/smithy-checksums/rust/lib.rs | 51 ++++++++++++--------------- 1 file changed, 23 insertions(+), 28 deletions(-) diff --git a/packages/smithy-checksums/rust/lib.rs b/packages/smithy-checksums/rust/lib.rs index 4ed6086c1..7e94dc5c6 100644 --- a/packages/smithy-checksums/rust/lib.rs +++ b/packages/smithy-checksums/rust/lib.rs @@ -9,8 +9,8 @@ use std::sync::{Mutex, MutexGuard}; use pyo3::buffer::PyBuffer; -use pyo3::prelude::*; -use pyo3::types::PyBytes; +use pyo3::types::{PyAny, PyBytes, PyBytesMethods}; +use pyo3::{Bound, PyResult, Python, pyclass, pymethods, pymodule}; /// Inputs at least this large are hashed with the interpreter detached so /// other threads can run. Below it, detaching costs more than it saves. This @@ -19,17 +19,22 @@ const DETACH_THRESHOLD: usize = 2048; /// Streaming checksum state shared by every class in this module. trait State: Clone + Send { + /// The digest, a byte array as long as the algorithm's output. + type Digest: AsRef<[u8]>; + fn new() -> Self; fn update(&mut self, data: &[u8]); - fn digest(&self) -> Vec; + fn digest(&self) -> Self::Digest; } macro_rules! crc_state { - ($state:ident, $algorithm:ident, $size:literal) => { + ($state:ident, $algorithm:ident, $width:ty) => { #[derive(Clone)] struct $state(crc_fast::Digest); impl State for $state { + type Digest = [u8; size_of::<$width>()]; + fn new() -> Self { Self(crc_fast::Digest::new(crc_fast::CrcAlgorithm::$algorithm)) } @@ -38,16 +43,17 @@ macro_rules! crc_state { self.0.update(data); } - fn digest(&self) -> Vec { - // The digest is a u64 regardless of the CRC's width. - self.0.finalize().to_be_bytes()[8 - $size..].to_vec() + fn digest(&self) -> Self::Digest { + // `finalize` returns a u64 regardless of the CRC's width, and + // the CRC is in its low bits. + (self.0.finalize() as $width).to_be_bytes() } } }; } -crc_state!(Crc32State, Crc32IsoHdlc, 4); -crc_state!(Crc32cState, Crc32Iscsi, 4); +crc_state!(Crc32State, Crc32IsoHdlc, u32); +crc_state!(Crc32cState, Crc32Iscsi, u32); fn lock(state: &Mutex) -> MutexGuard<'_, S> { // A panic while holding the lock can't leave a checksum half-updated in a @@ -59,26 +65,15 @@ fn lock(state: &Mutex) -> MutexGuard<'_, S> { /// Feeds the bytes of `data` to `state`. /// -/// `bytes` and other read-only buffers are hashed in place. Writable buffers -/// (such as `bytearray`) are copied first, because another thread could -/// otherwise modify them while they're being read. +/// `bytes` is immutable, so it's hashed in place. Other buffers are copied +/// first, because another thread could modify them while they're being read. +/// That includes read-only buffers: the flag only stops this reader from +/// writing, and a read-only `memoryview` of a `bytearray` can still change. fn update(py: Python<'_>, state: &Mutex, data: &Bound<'_, PyAny>) -> PyResult<()> { - if let Ok(bytes) = data.cast::() { - hash_slice(py, state, bytes.as_bytes()); - return Ok(()); - } - - let buffer = PyBuffer::::get(data)?; - if buffer.readonly() && buffer.is_c_contiguous() { - // SAFETY: the buffer is read-only and C-contiguous, and `buffer` - // keeps the exporter alive and its memory pinned until it's dropped - // at the end of this function. - let slice = unsafe { - std::slice::from_raw_parts(buffer.buf_ptr().cast::(), buffer.len_bytes()) - }; - hash_slice(py, state, slice); + if let Ok(data) = data.cast::() { + hash_slice(py, state, data.as_bytes()); } else { - let copy = buffer.to_vec(py)?; + let copy = PyBuffer::::get(data)?.to_vec(py)?; hash_slice(py, state, ©); } Ok(()) @@ -113,7 +108,7 @@ macro_rules! checksum_class { } fn digest<'py>(&self, py: Python<'py>) -> Bound<'py, PyBytes> { - PyBytes::new(py, &lock(&self.state).digest()) + PyBytes::new(py, lock(&self.state).digest().as_ref()) } fn copy(&self) -> Self { From a083f55486d7fd5f1f495b64339763f055156f4c Mon Sep 17 00:00:00 2001 From: jonathan343 Date: Tue, 27 Oct 2026 17:20:22 -0400 Subject: [PATCH 08/12] Stop calling the checksum implementations a backend "Backend" is the PEP 517 term for the build backend, and it doesn't suggest that the native extension is optional. The module that picks the native or pure-Python implementations is now _private/selection.py, and the public classes' base is _BaseChecksum. --- .../src/smithy_checksums/__init__.py | 12 ++++++------ .../_private/{backend.py => selection.py} | 0 .../tests/unit/test_implementations.py | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) rename packages/smithy-checksums/src/smithy_checksums/_private/{backend.py => selection.py} (100%) diff --git a/packages/smithy-checksums/src/smithy_checksums/__init__.py b/packages/smithy-checksums/src/smithy_checksums/__init__.py index 88eb88d5d..323d13e68 100644 --- a/packages/smithy-checksums/src/smithy_checksums/__init__.py +++ b/packages/smithy-checksums/src/smithy_checksums/__init__.py @@ -13,9 +13,9 @@ from importlib.metadata import version from typing import ClassVar, Final, Protocol, Self, runtime_checkable -from ._private.backend import IMPLEMENTATIONS as _IMPLEMENTATIONS -from ._private.backend import NATIVE as _NATIVE -from ._private.backend import ChecksumImpl as _ChecksumImpl +from ._private.selection import IMPLEMENTATIONS as _IMPLEMENTATIONS +from ._private.selection import NATIVE as _NATIVE +from ._private.selection import ChecksumImpl as _ChecksumImpl from ._private.buffers import as_byte_view as _as_byte_view __version__ = version("smithy-checksums") @@ -71,7 +71,7 @@ def copy(self) -> Self: ... -class _BackendChecksum: +class _BaseChecksum: __slots__ = ("_impl",) name: ClassVar[str] @@ -103,7 +103,7 @@ def copy(self) -> Self: return new -class CRC32(_BackendChecksum): +class CRC32(_BaseChecksum): """CRC-32 (ISO-HDLC), the variant used by zlib and gzip.""" __slots__ = () @@ -112,7 +112,7 @@ class CRC32(_BackendChecksum): digest_size = 4 -class CRC32C(_BackendChecksum): +class CRC32C(_BaseChecksum): """CRC-32C (Castagnoli).""" __slots__ = () diff --git a/packages/smithy-checksums/src/smithy_checksums/_private/backend.py b/packages/smithy-checksums/src/smithy_checksums/_private/selection.py similarity index 100% rename from packages/smithy-checksums/src/smithy_checksums/_private/backend.py rename to packages/smithy-checksums/src/smithy_checksums/_private/selection.py diff --git a/packages/smithy-checksums/tests/unit/test_implementations.py b/packages/smithy-checksums/tests/unit/test_implementations.py index 73975250f..a0a00c663 100644 --- a/packages/smithy-checksums/tests/unit/test_implementations.py +++ b/packages/smithy-checksums/tests/unit/test_implementations.py @@ -14,7 +14,7 @@ import pytest from smithy_checksums._private import crc -from smithy_checksums._private.backend import ChecksumImpl +from smithy_checksums._private.selection import ChecksumImpl from .vectors import ALGORITHMS, KNOWN_ANSWERS, PATTERN_DIGESTS, pattern From 2f880d8c69a14562d4076cd42587433862ceed9e Mon Sep 17 00:00:00 2001 From: jonathan343 Date: Wed, 28 Oct 2026 13:55:10 -0400 Subject: [PATCH 09/12] Stop HTML-escaping the third-party license file Handlebars escapes {{...}} for HTML, so LICENSE-THIRD-PARTY had " and ' in place of quotes. The template now uses {{{...}}} for every field, since the output is plain text. --- packages/smithy-checksums/LICENSE-THIRD-PARTY | 204 +++++++++--------- packages/smithy-checksums/about.hbs | 6 +- 2 files changed, 105 insertions(+), 105 deletions(-) diff --git a/packages/smithy-checksums/LICENSE-THIRD-PARTY b/packages/smithy-checksums/LICENSE-THIRD-PARTY index bf00306c1..52b2ee121 100644 --- a/packages/smithy-checksums/LICENSE-THIRD-PARTY +++ b/packages/smithy-checksums/LICENSE-THIRD-PARTY @@ -17,38 +17,38 @@ Used by: 1. Definitions. - "License" shall mean the terms and conditions for use, reproduction, + "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. - "Licensor" shall mean the copyright owner or entity authorized by + "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. - "Legal Entity" shall mean the union of the acting entity and all + "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the + "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. - "You" (or "Your") shall mean an individual or Legal Entity + "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. - "Source" form shall mean the preferred form for making modifications, + "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. - "Object" form shall mean any form resulting from mechanical + "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. - "Work" shall mean the work of authorship, whether in Source or + "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). - "Derivative Works" shall mean any work, whether in Source or Object + "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes @@ -56,21 +56,21 @@ Used by: separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. - "Contribution" shall mean any work of authorship, including + "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" + the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." + designated in writing by the copyright owner as "Not a Contribution." - "Contributor" shall mean Licensor and any individual or Legal Entity + "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. @@ -114,7 +114,7 @@ Used by: excluding those notices that do not pertain to any part of the Derivative Works; and - (d) If the Work includes a "NOTICE" text file as part of its + (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not @@ -153,7 +153,7 @@ Used by: 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, + Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A @@ -189,24 +189,24 @@ Used by: APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier + same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] - Licensed under the Apache License, Version 2.0 (the "License"); + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. @@ -226,38 +226,38 @@ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. - "License" shall mean the terms and conditions for use, reproduction, + "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. - "Licensor" shall mean the copyright owner or entity authorized by + "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. - "Legal Entity" shall mean the union of the acting entity and all + "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the + "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. - "You" (or "Your") shall mean an individual or Legal Entity + "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. - "Source" form shall mean the preferred form for making modifications, + "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. - "Object" form shall mean any form resulting from mechanical + "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. - "Work" shall mean the work of authorship, whether in Source or + "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). - "Derivative Works" shall mean any work, whether in Source or Object + "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes @@ -265,21 +265,21 @@ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. - "Contribution" shall mean any work of authorship, including + "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" + the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." + designated in writing by the copyright owner as "Not a Contribution." - "Contributor" shall mean Licensor and any individual or Legal Entity + "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. @@ -323,7 +323,7 @@ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION excluding those notices that do not pertain to any part of the Derivative Works; and - (d) If the Work includes a "NOTICE" text file as part of its + (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not @@ -362,7 +362,7 @@ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, + Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A @@ -398,24 +398,24 @@ END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier + same "printed page" as the copyright notice for easier identification within third-party archives. Copyright 2014 Paho Lurie-Gregg -Licensed under the Apache License, Version 2.0 (the "License"); +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, +distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. @@ -435,38 +435,38 @@ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. - "License" shall mean the terms and conditions for use, reproduction, + "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. - "Licensor" shall mean the copyright owner or entity authorized by + "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. - "Legal Entity" shall mean the union of the acting entity and all + "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the + "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. - "You" (or "Your") shall mean an individual or Legal Entity + "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. - "Source" form shall mean the preferred form for making modifications, + "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. - "Object" form shall mean any form resulting from mechanical + "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. - "Work" shall mean the work of authorship, whether in Source or + "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). - "Derivative Works" shall mean any work, whether in Source or Object + "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes @@ -474,21 +474,21 @@ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. - "Contribution" shall mean any work of authorship, including + "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" + the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." + designated in writing by the copyright owner as "Not a Contribution." - "Contributor" shall mean Licensor and any individual or Legal Entity + "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. @@ -532,7 +532,7 @@ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION excluding those notices that do not pertain to any part of the Derivative Works; and - (d) If the Work includes a "NOTICE" text file as part of its + (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not @@ -571,7 +571,7 @@ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, + Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A @@ -607,24 +607,24 @@ END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier + same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] -Licensed under the Apache License, Version 2.0 (the "License"); +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, +distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. @@ -645,38 +645,38 @@ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. - "License" shall mean the terms and conditions for use, reproduction, + "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. - "Licensor" shall mean the copyright owner or entity authorized by + "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. - "Legal Entity" shall mean the union of the acting entity and all + "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the + "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. - "You" (or "Your") shall mean an individual or Legal Entity + "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. - "Source" form shall mean the preferred form for making modifications, + "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. - "Object" form shall mean any form resulting from mechanical + "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. - "Work" shall mean the work of authorship, whether in Source or + "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). - "Derivative Works" shall mean any work, whether in Source or Object + "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes @@ -684,21 +684,21 @@ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. - "Contribution" shall mean any work of authorship, including + "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" + the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." + designated in writing by the copyright owner as "Not a Contribution." - "Contributor" shall mean Licensor and any individual or Legal Entity + "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. @@ -742,7 +742,7 @@ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION excluding those notices that do not pertain to any part of the Derivative Works; and - (d) If the Work includes a "NOTICE" text file as part of its + (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not @@ -781,7 +781,7 @@ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, + Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A @@ -817,24 +817,24 @@ END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier + same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] -Licensed under the Apache License, Version 2.0 (the "License"); +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, +distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. @@ -863,25 +863,25 @@ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. -"License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. +"License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. -"Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. +"Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. -"Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. +"Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. -"You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. +"You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. -"Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. +"Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. -"Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. +"Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. -"Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). +"Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). -"Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. +"Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. -"Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." +"Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." -"Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. @@ -895,7 +895,7 @@ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and - (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. + (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. @@ -903,7 +903,7 @@ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. -7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. +7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. @@ -913,18 +913,18 @@ END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. -To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. +To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] -Licensed under the Apache License, Version 2.0 (the "License"); +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, +distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. @@ -941,7 +941,7 @@ The MIT License (MIT) Copyright (c) 2014 Mathijs van de Nes Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal +of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is @@ -950,7 +950,7 @@ furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER @@ -969,7 +969,7 @@ The MIT License (MIT) Copyright (c) 2015 Bartłomiej Kamiński Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal +of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is @@ -978,7 +978,7 @@ furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER @@ -1005,8 +1005,8 @@ TERMS AND CONDITIONS OF THIS AGREEMENT. IF YOU DO NOT AGREE, DO NOT DOWNLOAD, INSTALL, COPY, DISTRIBUTE OR USE THE DATA FILES OR SOFTWARE. Permission is hereby granted, free of charge, to any person obtaining a -copy of data files and any associated documentation (the "Data Files") or -software and any associated documentation (the "Software") to deal in the +copy of data files and any associated documentation (the "Data Files") or +software and any associated documentation (the "Software") to deal in the Data Files or Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, and/or sell copies of the Data Files or Software, and to permit persons to whom the @@ -1015,7 +1015,7 @@ this copyright and permission notice appear with all copies of the Data Files or Software, or (b) this copyright and permission notice appear in associated Documentation. -THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS. diff --git a/packages/smithy-checksums/about.hbs b/packages/smithy-checksums/about.hbs index d23cd9c79..ad87eb115 100644 --- a/packages/smithy-checksums/about.hbs +++ b/packages/smithy-checksums/about.hbs @@ -5,12 +5,12 @@ it by hand. {{#each licenses}} ================================================================================ -{{name}} ({{id}}) +{{{name}}} ({{{id}}}) Used by: {{#each used_by}} - - {{crate.name}} {{crate.version}}{{#if crate.repository}} ({{crate.repository}}){{/if}} + - {{{crate.name}}} {{{crate.version}}}{{#if crate.repository}} ({{{crate.repository}}}){{/if}} {{/each}} -{{text}} +{{{text}}} {{/each}} From 34c1c82964584103f93cefdc89e97ef477579f22 Mon Sep 17 00:00:00 2001 From: jonathan343 Date: Thu, 29 Oct 2026 10:51:39 -0400 Subject: [PATCH 10/12] Sort the smithy_checksums imports Renaming _private/backend.py to _private/selection.py moved its imports out of order. --- packages/smithy-checksums/src/smithy_checksums/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/smithy-checksums/src/smithy_checksums/__init__.py b/packages/smithy-checksums/src/smithy_checksums/__init__.py index 323d13e68..3597a8353 100644 --- a/packages/smithy-checksums/src/smithy_checksums/__init__.py +++ b/packages/smithy-checksums/src/smithy_checksums/__init__.py @@ -13,10 +13,10 @@ from importlib.metadata import version from typing import ClassVar, Final, Protocol, Self, runtime_checkable +from ._private.buffers import as_byte_view as _as_byte_view from ._private.selection import IMPLEMENTATIONS as _IMPLEMENTATIONS from ._private.selection import NATIVE as _NATIVE from ._private.selection import ChecksumImpl as _ChecksumImpl -from ._private.buffers import as_byte_view as _as_byte_view __version__ = version("smithy-checksums") __all__ = ( From 7120603f83472b49f645ab41e5beaad65d0c6b7a Mon Sep 17 00:00:00 2001 From: jonathan343 Date: Thu, 29 Oct 2026 11:42:40 -0400 Subject: [PATCH 11/12] Reject non-contiguous buffers like hashlib does Strided byte views, such as memoryview(b"abcdefgh")[::2], were hashed while hashlib.sha256() rejects them, and views with other formats failed with a TypeError from the cast instead. new() returns hashlib objects for the SHA and MD5 algorithms, so the same update() call worked for some algorithms and raised for others. Every non-contiguous buffer now raises the same BufferError hashlib does. --- .../src/smithy_checksums/_private/buffers.py | 11 +++++++---- .../smithy-checksums/tests/unit/test_checksums.py | 14 +++++++++++--- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/packages/smithy-checksums/src/smithy_checksums/_private/buffers.py b/packages/smithy-checksums/src/smithy_checksums/_private/buffers.py index ee21e54be..11f6bba48 100644 --- a/packages/smithy-checksums/src/smithy_checksums/_private/buffers.py +++ b/packages/smithy-checksums/src/smithy_checksums/_private/buffers.py @@ -11,13 +11,16 @@ def as_bytes(data: Buffer) -> bytes | bytearray: def as_byte_view(data: Buffer) -> bytes | bytearray | memoryview: - """Returns ``data`` as an object the native extension accepts, without copying. + """Returns ``data`` as a C-contiguous buffer of bytes, without copying. - The native extension reads any buffer with a byte format. Other formats (such - as an ``array('I')``) are recast to bytes, which requires a C-contiguous buffer, - the same requirement :mod:`hashlib` has. + Both implementations accept the result. Buffers with other formats (such as an + ``array('I')``) are recast to bytes. + + :raises BufferError: If ``data`` isn't C-contiguous, as :mod:`hashlib` does. """ if isinstance(data, bytes | bytearray): return data view = memoryview(data) + if not view.c_contiguous: + raise BufferError("memoryview: underlying buffer is not C-contiguous") return view if view.format == "B" else view.cast("B") diff --git a/packages/smithy-checksums/tests/unit/test_checksums.py b/packages/smithy-checksums/tests/unit/test_checksums.py index ba17d1a4d..4898530aa 100644 --- a/packages/smithy-checksums/tests/unit/test_checksums.py +++ b/packages/smithy-checksums/tests/unit/test_checksums.py @@ -59,9 +59,17 @@ def test_accepts_non_byte_formats(algorithm: str) -> None: @pytest.mark.parametrize("algorithm", ALGORITHMS) -def test_rejects_non_contiguous_buffers(algorithm: str) -> None: - with pytest.raises(TypeError): - CLASSES[algorithm]().update(memoryview(array.array("I", range(8)))[::2]) +@pytest.mark.parametrize( + "data", + [memoryview(b"abcdefgh")[::2], memoryview(array.array("I", range(8)))[::2]], + ids=["bytes", "array"], +) +def test_rejects_non_contiguous_buffers(algorithm: str, data: memoryview) -> None: + # Matching hashlib keeps every algorithm from new() consistent. + with pytest.raises(BufferError): + hashlib.sha256().update(data) + with pytest.raises(BufferError, match="C-contiguous"): + CLASSES[algorithm]().update(data) @pytest.mark.parametrize("algorithm", ALGORITHMS) From 6a7a84f98200279e97e5a1219406c59330f3a226 Mon Sep 17 00:00:00 2001 From: jonathan343 Date: Thu, 29 Oct 2026 11:42:40 -0400 Subject: [PATCH 12/12] Pass buffers to the pure-Python checksums without copying The pure-Python update() copied every memoryview to bytes before hashing, which as_byte_view had just produced without copying. zlib.crc32 and the chunked CRC read any byte buffer, so the view is passed through, and the as_bytes helper is gone. --- .../src/smithy_checksums/_private/buffers.py | 7 ------- .../src/smithy_checksums/_private/crc.py | 4 ++-- .../src/smithy_checksums/_private/pure.py | 9 +++------ 3 files changed, 5 insertions(+), 15 deletions(-) diff --git a/packages/smithy-checksums/src/smithy_checksums/_private/buffers.py b/packages/smithy-checksums/src/smithy_checksums/_private/buffers.py index 11f6bba48..175ab15e7 100644 --- a/packages/smithy-checksums/src/smithy_checksums/_private/buffers.py +++ b/packages/smithy-checksums/src/smithy_checksums/_private/buffers.py @@ -3,13 +3,6 @@ from collections.abc import Buffer -def as_bytes(data: Buffer) -> bytes | bytearray: - """Returns the bytes of ``data``, copying only if it isn't already bytes-like.""" - if isinstance(data, bytes | bytearray): - return data - return memoryview(data).tobytes() - - def as_byte_view(data: Buffer) -> bytes | bytearray | memoryview: """Returns ``data`` as a C-contiguous buffer of bytes, without copying. diff --git a/packages/smithy-checksums/src/smithy_checksums/_private/crc.py b/packages/smithy-checksums/src/smithy_checksums/_private/crc.py index 17819186c..22de6bcc7 100644 --- a/packages/smithy-checksums/src/smithy_checksums/_private/crc.py +++ b/packages/smithy-checksums/src/smithy_checksums/_private/crc.py @@ -79,7 +79,7 @@ def __init__(self) -> None: super().__init__() self._crc = 0 - def _update(self, data: bytes | bytearray) -> None: + def _update(self, data: bytes | bytearray | memoryview) -> None: self._crc = zlib.crc32(data, self._crc) def _digest(self) -> bytes: @@ -110,7 +110,7 @@ def __init__(self) -> None: self._crc = self._mask self._buffer = bytearray() - def _update(self, data: bytes | bytearray) -> None: + def _update(self, data: bytes | bytearray | memoryview) -> None: buffer = self._buffer view = memoryview(data) if buffer: diff --git a/packages/smithy-checksums/src/smithy_checksums/_private/pure.py b/packages/smithy-checksums/src/smithy_checksums/_private/pure.py index e42bac3dd..b82e56737 100644 --- a/packages/smithy-checksums/src/smithy_checksums/_private/pure.py +++ b/packages/smithy-checksums/src/smithy_checksums/_private/pure.py @@ -1,11 +1,8 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -from collections.abc import Buffer from threading import Lock from typing import Self -from .buffers import as_bytes - class PureChecksum: """Base class for the pure-Python implementations. @@ -20,9 +17,9 @@ class PureChecksum: def __init__(self) -> None: self._lock = Lock() - def update(self, data: Buffer, /) -> None: + def update(self, data: bytes | bytearray | memoryview, /) -> None: with self._lock: - self._update(as_bytes(data)) + self._update(data) def digest(self) -> bytes: with self._lock: @@ -32,7 +29,7 @@ def copy(self) -> Self: with self._lock: return self._copy() - def _update(self, data: bytes | bytearray) -> None: + def _update(self, data: bytes | bytearray | memoryview) -> None: raise NotImplementedError def _digest(self) -> bytes: