diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b56f459..7f6d602 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -83,10 +83,10 @@ jobs: # would have. Pinned rather than floating: a green run should mean this # code passed against a known compiler, not against whatever was newest # that morning. - - name: Install twill v1.7.1 + - name: Install twill v1.8.0 run: | curl -fsSL -o twill \ - https://github.com/twill-lang/twill/releases/download/v1.7.1/twill-v1.7.1-linux-amd64 + https://github.com/twill-lang/twill/releases/download/v1.8.0/twill-v1.8.0-linux-amd64 chmod +x twill ./twill --version diff --git a/CHANGELOG.md b/CHANGELOG.md index 3870429..24af986 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,12 +4,25 @@ First cut of spool, the package manager for twill, written in twill. -It runs, on twill 1.7.1. `init`, `list` and `remove` do their whole job; -`install` writes the lockfile and the vendor directory; `add` writes the -dependency into `spool.toml` and then stops where it would fetch, because twill -has no process interface and spool fetches by running `git`. That is the one -thing still missing, and `docs/needs.md` entry 1 is the whole of it. `README.md` -has the status table. +It runs, and it fetches. `init`, `list`, `remove` and `add` do their whole job, +and `install` resolves a git dependency, clones it, vendors it into +`twill_modules/` and writes a `spool.lock` carrying the commit and the content +hash. Fetching waited on `docs/needs.md` entry 1, a process interface in twill, +which now exists as `run(program, argv, dir) -> Res[Str, Str]` -- the signature +that entry asked for. All fourteen entries in that file are delivered. +`README.md` has the status table. + +Two things changed here to meet it: + +- `git()` in `src/vendor.tw` is one line. It used to unwrap a status byte the + old `run` put in front of its output, because the language could not return + two values; `docs/needs.md` entry 10 called that the ugliest thing in spool + and this was its last hiding place. +- `published_versions` answers a `Res` rather than an `Arr`. A repository that + clones and carries no readable tag publishes no versions; one that cannot be + reached at all -- no `git` on PATH, `TWILL_NO_EXEC` set, a URL nobody can + clone -- is a different failure, and folding both into an empty list made + spool blame the repository for the local problem. Added: diff --git a/README.md b/README.md index 9338ee9..fa8a574 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ ok tests/ui_test.tw ``` You need twill 1.7.0 or newer. Everything shown in this file was run on twill -1.7.1, which is what CI pins. `docs/needs.md` is still worth reading -- it is +1.8.0, which is what CI pins. `docs/needs.md` is still worth reading -- it is the list of what this library asked the language for, and it now records which of those arrived and which are still open. @@ -58,12 +58,12 @@ is a twill binary: ```bash curl -fsSL -o twill \ - https://github.com/twill-lang/twill/releases/download/v1.7.1/twill-v1.7.1-linux-amd64 + https://github.com/twill-lang/twill/releases/download/v1.8.0/twill-v1.8.0-linux-amd64 chmod +x twill ./twill --version ``` -The v1.7.1 assets are `twill-v1.7.1-linux-amd64`, `-linux-arm64`, +The v1.8.0 assets are `twill-v1.8.0-linux-amd64`, `-linux-arm64`, `-darwin-amd64`, `-darwin-arm64` and `-windows-amd64.exe`. Then clone this repository and run `main.tw`: @@ -97,14 +97,23 @@ There is no `spool` binary yet, so every command below is spelled ## Status -The source runs and its tests pass. "Blocked" below means one thing only: -twill 1.7.1 has no process interface, so spool cannot shell out to `git`, and -vendoring from a git source is the one feature that needs to. That is verified -rather than assumed: there is no `run`, `exec` or `spawn` among the builtins -twill 1.7.1 defines, and `src/vendor.tw` calls `run`, so the moment a command -reaches git it dies with `undefined variable "run"`. It is a decision about the -security surface of running a `.tw` file rather than a missing builtin, and it -is the last thing between spool and doing its whole job. +The source runs, its tests pass, and **it fetches**. The one thing this section +called blocked was that twill had no process interface, so spool could not shell +out to `git` and vendoring was the single feature that needed to. twill now +defines `run(program, argv, dir) -> Res[Str, Str]`, with the signature +`docs/needs.md` entry 1 asked for, and `src/vendor.tw` calls it directly: the +status-byte encoding that entry called the ugliest thing in spool is gone with +it. + +That is verified by doing it rather than by reading the builtin list. Against a +local git repository tagged `v1.2.0`, `spool install` clones it, resolves the +constraint, vendors the package into `twill_modules/` and writes a `spool.lock` +carrying the commit and the content hash. + +Requires `git` on `PATH`, which spool reports plainly rather than failing +obscurely. Setting `TWILL_NO_EXEC` turns twill's process interface off +wholesale, and spool then fails to fetch with a message that names the +variable. | Piece | State | | --- | --- | @@ -114,10 +123,10 @@ is the last thing between spool and doing its whole job. | `spool.lock` writer and reader, deterministic | runs, tested by `tests/lockfile_test.tw` | | SHA-256, in twill, verified against published vectors | moved to `std/hash`; `tests/sha256_test.tw` still checks it against the vectors | | Package content hashing and verification | runs, tested by `tests/sha256_test.tw` | -| Vendoring into `twill_modules/` | blocked: needs a process interface for git, which twill 1.7.1 does not have | +| Vendoring into `twill_modules/` | runs: clones, checks out the resolved tag, verifies the content hash, writes the tree | | `init` / `list` / `remove` | run; they touch no network | -| `add` | writes the dependency into `spool.toml`, then stops where it would fetch | -| `install` | writes `spool.lock` and the vendor directory; stops where it would fetch a declared git dependency | +| `add` | writes the dependency into `spool.toml` | +| `install` | resolves, fetches, vendors, and writes `spool.lock` | | Tests | 6 suites, 6 passed, run by CI on every push | | A registry | not planned for v0.1. Git sources only | | Publishing packages | not in scope. spool consumes, it does not publish | @@ -266,10 +275,10 @@ myproject/ spool add tensorstats https://github.com/example/tensorstats ``` -This is the one step that does not work on twill 1.7.1. `add` writes the -dependency into `spool.toml` and then stops at git, so the rest of this section -describes the tree spool produces once there is a process interface, not one it -can produce today. +This works from twill 1.8.0, which is the release that added the process +interface. On 1.7.1 and earlier `add` wrote the dependency into `spool.toml` and +then stopped at git, and the rest of this section described a tree spool could +not yet produce. **2. Gitignore the vendor directory, commit the two spool files.** diff --git a/docs/needs.md b/docs/needs.md index bfe20b1..6a30810 100644 --- a/docs/needs.md +++ b/docs/needs.md @@ -5,13 +5,12 @@ the list of language and runtime features the source uses that `mode systems` did not provide, with the file that needs each one and what spool did in the meantime. -Most of it is now a record rather than a queue. Of the fourteen entries below, -thirteen are delivered as of twill 1.7.1, which is the release `spool.toml` and -CI pin. **One is still open, and it is entry 1: there is no process interface.** -That is verified rather than assumed: no builtin in twill 1.7.1 starts a -subprocess, `src/vendor.tw` calls `run`, and any command that reaches git dies -with `undefined variable "run"`. Vendoring from a git source is the one spool -feature that waits on it. +This is now a record rather than a queue. **All fourteen entries below are +delivered.** Thirteen were delivered as of twill 1.7.1, the release `spool.toml` +and CI pin, and entry 1 -- the process interface, the one thing between spool +and fetching a package -- landed after it. Verified by fetching: against a git +repository tagged `v1.2.0`, `spool install` clones, resolves, vendors into +`twill_modules/` and writes a `spool.lock` with the commit and content hash. It was meant to be read as a work queue for the language, not as a complaint. Every entry was reached by writing real code and hitting the wall, which is the @@ -34,11 +33,16 @@ Entry 1 still is. **Needs:** `run(program: Str, argv: Arr[Str], dir: Str) -> Res[Str, Str]` **Used by:** `src/vendor.tw` -**Status:** **still open on twill 1.7.1.** The only entry here that is. - -The `Res` in the signature is the one change since this was written. The -`"!"`-flag encoding entry 10 describes is gone everywhere else in spool, and a -process interface should not be the last place it survives. +**Status:** **done.** The signature is the one asked for here, `Res` included, +so the `"!"`-flag encoding entry 10 describes did not survive in its last place +either -- `git()` in `src/vendor.tw` is now one line. + +Two things came with it that this entry asked for without naming. It takes an +argument vector and never a shell, which is what makes it safe to hand a tag or +a URL out of somebody else's manifest straight to git. And the security note in +the last paragraph below is answered rather than deferred: `TWILL_NO_EXEC`, set +to anything non-empty, refuses every `run` and returns an `Err` saying so, which +is how spool now reports it instead of blaming the repository. This is the largest gap and it is not a small one. spool fetches packages by running `git clone`, `git fetch`, `git tag`, `git rev-list`, `git show` and diff --git a/spool.toml b/spool.toml index a54445e..be1fd5e 100644 --- a/spool.toml +++ b/spool.toml @@ -18,4 +18,4 @@ entry = "src/commands.tw" # this source was written against. spool has no notion of a toolchain dependency, # so it is declared as an ordinary git dependency, the same way every other # package in the ecosystem declares it. -twill = { version = "^1.7.0", git = "https://github.com/twill-lang/twill" } +twill = { version = "^1.8.0", git = "https://github.com/twill-lang/twill" } diff --git a/src/commands.tw b/src/commands.tw index 9c67c64..f68e3ad 100644 --- a/src/commands.tw +++ b/src/commands.tw @@ -243,7 +243,7 @@ fn cmd_add(dir: Str, name: Str, url: Str, requested: Str) -> Res[Unit, Str] { # With no version given, pin to the highest published version with a caret. # Defaulting to "*" would be less code and would silently opt the project # into the next major release of a dependency. - let versions = vendor.published_versions(p.root, url) + let versions: Arr[Str] = vendor.published_versions(p.root, url)? if len(versions) == 0 { return Err(url + " publishes no version tags that spool can read; pass a version explicitly") } diff --git a/src/vendor.tw b/src/vendor.tw index 05a8ecd..5255485 100644 --- a/src/vendor.tw +++ b/src/vendor.tw @@ -81,18 +81,18 @@ fn short_hash(v: Str) -> Str { # failing obscurely. # `Ok` carries git's output, `Err` carries git's complaint. # -# `run` answers with a status BYTE in front of its output -- `"!"` for a -# non-zero exit, `" "` otherwise -- because the subset could not return two -# values. That convention leaked into every caller in this file, along with a -# `git_ok`/`git_out` pair whose only job was to make it survivable; docs/needs.md -# entry 10 calls it the ugliest thing in spool. It stops here now: `run`'s -# encoding is unwrapped once, at the one place that calls it. +# `run` used to be described here as answering with a status BYTE in front of +# its output -- `"!"` for a non-zero exit, `" "` otherwise -- because the subset +# could not return two values, and docs/needs.md entry 10 called that the +# ugliest thing in spool. The builtin that arrived answers a `Res` instead, so +# there is no encoding left to unwrap and this function is the shape it always +# wanted to be: git's own two answers, renamed. +# +# `run` takes an argument vector and never a shell, which is what makes it safe +# to pass a tag or a URL out of somebody else's manifest straight through to +# git. fn git(dir: Str, argv: Arr[Str]) -> Res[Str, Str] { - let r = run("git", argv, dir) - if r[0] == s.B_SPACE { - return Ok(r[1:len(r)]) - } - Err(r[1:len(r)]) + run("git", argv, dir) } # ensure_clone clones a repository into the cache or updates an existing clone. @@ -120,16 +120,18 @@ fn ensure_clone(root: Str, url: Str) -> Res[Str, Str] { # plain major.minor.patch is skipped rather than reported: repositories carry # all kinds of tags, and refusing to resolve because someone tagged "nightly" # would be obstructive. -fn published_versions(root: Str, url: Str) -> Arr[Str] { +# +# It answers a `Res` because there are two different empty answers and callers +# were reporting the wrong one. A repository that clones and carries no readable +# tag really does publish no versions, and that is `Ok([])`. A repository that +# could not be reached, or a git that could not be started at all -- no `git` on +# PATH, `TWILL_NO_EXEC` set, a URL nobody can clone -- has published nothing we +# know about, and saying "publishes no versions" about it names the wrong thing +# as the problem. Before this it swallowed both into an empty list. +fn published_versions(root: Str, url: Str) -> Res[Arr[Str], Str] { let out: Arr[Str] = [] - let dir: Str = match ensure_clone(root, url) { - Ok(at) => at, - Err(_) => return out, - } - let listed: Str = match git(dir, ["tag", "--list"]) { - Ok(text) => text, - Err(_) => return out, - } + let dir: Str = ensure_clone(root, url)? + let listed: Str = git(dir, ["tag", "--list"])? let lines = s.split_lines(listed) let i = 0 while i < len(lines) { @@ -140,7 +142,7 @@ fn published_versions(root: Str, url: Str) -> Arr[Str] { } i = i + 1 } - out + Ok(out) } # tag_for finds the spelling, "1.2.3" or "v1.2.3", that the repository actually @@ -205,7 +207,7 @@ fn build_catalog(root: Str, m: manifest.Manifest, cat: resolve.Catalog) -> Res[U if dict_has(cat.versions, url) { continue } - let versions = published_versions(root, url) + let versions: Arr[Str] = published_versions(root, url)? cat.versions[url] = s.join(versions, ",") let v = 0 while v < len(versions) {