diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 661d8b3..040c2a0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -83,10 +83,10 @@ jobs: # would have. Pinned rather than floating: a green run should mean this # code passed against a known compiler, not against whatever was newest # that morning. - - name: Install twill v1.9.0 + - name: Install twill v1.12.0 run: | curl -fsSL -o twill \ - https://github.com/twill-lang/twill/releases/download/v1.9.0/twill-v1.9.0-linux-amd64 + https://github.com/twill-lang/twill/releases/download/v1.12.0/twill-v1.12.0-linux-amd64 chmod +x twill ./twill --version diff --git a/CHANGELOG.md b/CHANGELOG.md index 24af986..cf365df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,7 +41,10 @@ Added: SHA-256 test vectors including the padding boundaries at 55, 56 and 64 bytes. This began as `src/sha256.tw`, a SHA-256 written here in twill; it moved into the standard library so the toolchain has one implementation of a digest that - everything must agree on byte for byte. + everything must agree on byte for byte. As of twill 1.11 that implementation + is the `sha256` builtin, and `src/pkghash.tw` calls it: 16 MB in 6.5 ms on + one machine, where `std/hash` did about 100 kB/s. The pin, the README's + install line and CI move to 1.12.0 with it. - A package content hash over a length-prefixed serialisation of the file tree, excluding VCS metadata, and verification of it on every install. - Vendoring into `twill_modules/`, which is where twill's `import` can reach it. diff --git a/README.md b/README.md index fa8a574..9ba310c 100644 --- a/README.md +++ b/README.md @@ -46,8 +46,8 @@ ok tests/ui_test.tw 6 file(s): 6 passed, 0 failed ``` -You need twill 1.7.0 or newer. Everything shown in this file was run on twill -1.8.0, which is what CI pins. `docs/needs.md` is still worth reading -- it is +You need twill 1.12.0 or newer. Everything shown in this file was run on twill +1.12.0, which is what CI pins. `docs/needs.md` is still worth reading -- it is the list of what this library asked the language for, and it now records which of those arrived and which are still open. @@ -58,12 +58,12 @@ is a twill binary: ```bash curl -fsSL -o twill \ - https://github.com/twill-lang/twill/releases/download/v1.8.0/twill-v1.8.0-linux-amd64 + https://github.com/twill-lang/twill/releases/download/v1.12.0/twill-v1.12.0-linux-amd64 chmod +x twill ./twill --version ``` -The v1.8.0 assets are `twill-v1.8.0-linux-amd64`, `-linux-arm64`, +The v1.12.0 assets are `twill-v1.12.0-linux-amd64`, `-linux-arm64`, `-darwin-amd64`, `-darwin-arm64` and `-windows-amd64.exe`. Then clone this repository and run `main.tw`: @@ -121,7 +121,7 @@ variable. | Version parsing and `^` constraints | runs, tested by `tests/semver_test.tw` | | Dependency resolver, pure and network-free | runs, tested by `tests/resolve_test.tw` | | `spool.lock` writer and reader, deterministic | runs, tested by `tests/lockfile_test.tw` | -| SHA-256, in twill, verified against published vectors | moved to `std/hash`; `tests/sha256_test.tw` still checks it against the vectors | +| SHA-256, in twill, verified against published vectors | moved to `std/hash`, then to the `sha256` builtin; `tests/sha256_test.tw` still checks it against the vectors | | Package content hashing and verification | runs, tested by `tests/sha256_test.tw` | | Vendoring into `twill_modules/` | runs: clones, checks out the resolved tag, verifies the content hash, writes the tree | | `init` / `list` / `remove` | run; they touch no network | @@ -324,7 +324,7 @@ src/toml.tw the spool.toml reader src/semver.tw versions and `^` constraints src/manifest.tw spool.toml model, parse, render, add/remove a dependency src/resolve.tw the resolver: pure, no IO, testable from a literal table -src/pkghash.tw the package content hash and its verification, over std/hash +src/pkghash.tw the package content hash and its verification, over the sha256 builtin src/lockfile.tw spool.lock render and parse src/vendor.tw git, the filesystem, and nothing else in spool touches them src/commands.tw add, install, list, remove, init @@ -344,9 +344,11 @@ None. Not "few". None. No third-party twill packages, no Go, no shell scripts doing the real work, no vendored anything. SHA-256 used to be in this repository, in twill, for exactly that reason. It is -now twill's `std/hash`, which is the same argument arriving at the right place: -a digest the whole toolchain has to agree on byte for byte should have one -implementation, and the standard library is not a third-party dependency. +now twill's `sha256` builtin, held to `std/hash` by a test in twill, which is +the same argument arriving at the right place: a digest the whole toolchain has +to agree on byte for byte should have one implementation, and the language is +not a third-party dependency. The builtin also runs at machine speed, 16 MB in +6.5 ms on one machine, where `std/hash` did about 100 kB/s on the same one. `src/pkghash.tw` is what remains here, and it is the part that is spool's: the canonical serialisation of a file tree that gets hashed. diff --git a/docs/needs.md b/docs/needs.md index 074e703..cb5d6a3 100644 --- a/docs/needs.md +++ b/docs/needs.md @@ -321,9 +321,10 @@ on byte for byte should have one implementation and not one per repository. `warp` wanted the same function for cache keys, and two transcriptions of SHA-256 that must agree is the worse risk. -It is `std/hash` now. `src/sha256.tw` is deleted, `src/pkghash.tw` calls -`sha.hash_str`, and `tests/sha256_test.tw` still checks the published vectors, -including the padding boundaries at 55, 56 and 64 bytes, through `std/hash`. The +It is the `sha256` builtin now, from twill 1.11, which is `std/hash` at machine +speed and is held to it by a test in twill. `src/sha256.tw` is deleted, +`src/pkghash.tw` calls `sha256`, and `tests/sha256_test.tw` still checks the +published vectors, including the padding boundaries at 55, 56 and 64 bytes. The part that stayed here is the part that is actually spool's: the canonical, length-prefixed serialisation of a file tree in `src/pkghash.tw`. diff --git a/spool.toml b/spool.toml index 2cf7e0a..2d7a6c2 100644 --- a/spool.toml +++ b/spool.toml @@ -18,4 +18,4 @@ entry = "src/commands.tw" # this source was written against. spool has no notion of a toolchain dependency, # so it is declared as an ordinary git dependency, the same way every other # package in the ecosystem declares it. -twill = { version = "^1.9.0", git = "https://github.com/twill-lang/twill" } +twill = { version = "^1.12.0", git = "https://github.com/twill-lang/twill" } diff --git a/src/pkghash.tw b/src/pkghash.tw index 23b721a..2192130 100644 --- a/src/pkghash.tw +++ b/src/pkghash.tw @@ -8,7 +8,6 @@ mode systems # over different bytes under the same commit, this is what notices. import "strutil.tw" as s -import "std/hash" as sha import "std/text" as text let HASH_PREFIX = "sha256:" @@ -54,7 +53,7 @@ fn canonical(paths: Arr[Str], contents: Arr[Str]) -> Str { # `/` as the separator on every platform, so the same package hashes the same # on Windows and Linux; sort_tree below is what callers use to guarantee it. fn hash_tree(paths: Arr[Str], contents: Arr[Str]) -> Str { - HASH_PREFIX + sha.hash_str(canonical(paths, contents)) + HASH_PREFIX + sha256(canonical(paths, contents)) } # is_excluded drops VCS metadata, which differs between a clone and an archive diff --git a/tests/sha256_test.tw b/tests/sha256_test.tw index e8517e6..1b8b025 100644 --- a/tests/sha256_test.tw +++ b/tests/sha256_test.tw @@ -2,8 +2,10 @@ mode systems # SHA-256 against the published test vectors. # -# These are the reason src/sha256.tw can be trusted at all. A hash function that -# is only tested against itself is a checksum with delusions. +# These are the reason the `sha256` builtin src/pkghash.tw calls can be trusted +# at all. A hash function that is only tested against itself is a checksum with +# delusions. `std/hash` is still imported for `padded_len`, which is where the +# block boundaries are stated. import "harness.tw" as t import "../src/strutil.tw" as s @@ -22,45 +24,46 @@ fn refusal(r: Res[Unit, Str]) -> Str { fn the_empty_string_hashes_to_the_published_vector() { t.equal_str("sha256 of the empty string", - sha.hash_str(""), + sha256(""), "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855") } fn abc_hashes_to_the_published_vector() { t.equal_str("sha256 of abc", - sha.hash_str("abc"), + sha256("abc"), "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad") } fn a_message_spanning_two_blocks_hashes_to_the_published_vector() { t.equal_str("sha256 of the 56-byte two-block vector", - sha.hash_str("abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"), + sha256("abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"), "248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1") } fn a_million_a_characters_is_not_tested_here_but_the_boundary_lengths_are() { - # The 1,000,000-character vector is skipped deliberately: at the speed this - # implementation runs in an interpreter it would dominate the suite. The - # lengths that actually exercise the padding logic are 55, 56 and 64 bytes, - # which straddle the point where padding spills into an extra block. + # The 1,000,000-character vector is skipped deliberately: the builtin hashes + # it in under a millisecond, but s.repeat takes twenty seconds to build it, + # and that would dominate the suite. The lengths that actually exercise the + # padding logic are 55, 56 and 64 bytes, which straddle the point where + # padding spills into an extra block. t.equal_i64("55 bytes pads into one block", sha.padded_len(55), 64) t.equal_i64("56 bytes pads into two blocks", sha.padded_len(56), 128) t.equal_i64("64 bytes pads into two blocks", sha.padded_len(64), 128) t.equal_i64("0 bytes pads into one block", sha.padded_len(0), 64) t.equal_str("sha256 of 55 a characters", - sha.hash_str(s.repeat("a", 55)), + sha256(s.repeat("a", 55)), "9f4390f8d30c2dd92ec9f095b65e2b9ae9b0a925a5258e241c9f1e910f734318") t.equal_str("sha256 of 56 a characters", - sha.hash_str(s.repeat("a", 56)), + sha256(s.repeat("a", 56)), "b35439a4ac6f0948b6d6f9e3c6af0f5f590ce20f1bde7090ef7970686ec6738a") t.equal_str("sha256 of 64 a characters", - sha.hash_str(s.repeat("a", 64)), + sha256(s.repeat("a", 64)), "ffe054fe7ae0cb6dc65c3af9b61d5209f439851db43d0ba5997337df154668eb") } fn every_digest_is_sixty_four_lowercase_hex_characters() { - let d = sha.hash_str("anything") + let d = sha256("anything") t.equal_i64("a digest is 64 characters", len(d), 64) let i = 0 let ok = true