From 3ab2715c3838cd05bbf4d8b8f74e7f3c083f5276 Mon Sep 17 00:00:00 2001 From: Raghu Betina Date: Tue, 22 Sep 2026 19:36:11 -0500 Subject: [PATCH 1/2] Simplify plugin release coordination Publish through protected release tags and trusted publishing, without promotion-only credentials or repeated package checks. Keep catalog selection checks focused and bind references to release provenance. --- .github/workflows/ci.yml | 32 +- .github/workflows/promote.yml | 84 --- .github/workflows/publish.yml | 7 +- README.md | 10 +- RELEASING.md | 19 +- docs/README.md | 2 +- docs/npm-promotion.md | 170 +---- .../2026-09-22-npm-promotion-retirement.md | 22 + evidence/README.md | 1 + script/check | 2 +- script/check-catalog.mjs | 38 ++ script/ci-scope.mjs | 39 ++ script/npm-promotion.mjs | 416 ------------- .../references/diagnostics-and-recovery.md | 7 +- .../references/foundation-plan-020.md | 8 +- test/catalog-ci.test.mjs | 77 +++ test/npm-promotion-cli-cache.test.mjs | 65 -- test/npm-promotion.test.mjs | 582 ------------------ test/release-compatibility.test.mjs | 17 +- test/repository.test.mjs | 48 +- 20 files changed, 287 insertions(+), 1359 deletions(-) delete mode 100644 .github/workflows/promote.yml create mode 100644 evidence/2026-09-22-npm-promotion-retirement.md create mode 100644 script/check-catalog.mjs create mode 100644 script/ci-scope.mjs delete mode 100644 script/npm-promotion.mjs create mode 100644 test/catalog-ci.test.mjs delete mode 100644 test/npm-promotion-cli-cache.test.mjs delete mode 100644 test/npm-promotion.test.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 61d7bdc..636548f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,30 +31,48 @@ jobs: run: | test "$(node --version)" = "v24.18.0" test "$(npm --version)" = "11.16.0" + - name: Select CI scope + id: scope + env: + BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.before }} + run: node script/ci-scope.mjs "$BASE_SHA" >> "$GITHUB_OUTPUT" + - name: Validate catalog metadata + if: steps.scope.outputs.catalog_only == 'true' && matrix.node == '22.0.0' + run: node script/check-release-compatibility.mjs + - name: Verify published catalog selection + if: steps.scope.outputs.catalog_only == 'true' && matrix.node == '24.18.0' + run: node script/check-catalog.mjs - run: npm ci --ignore-scripts + if: steps.scope.outputs.catalog_only != 'true' - name: Audit dependencies - if: matrix.node == '24.18.0' + if: steps.scope.outputs.catalog_only != 'true' && matrix.node == '24.18.0' run: npm audit - - run: sh script/check - name: Rehearse release ordering - if: matrix.node == '24.18.0' + if: steps.scope.outputs.catalog_only != 'true' && matrix.node == '24.18.0' run: | git fetch --force --no-tags origin \ "+refs/tags/claude-v*:refs/release-check/tags/claude-v*" node script/check-plugin-release-order.mjs --prospective - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + if: steps.scope.outputs.catalog_only != 'true' with: repository: firstdraft/cli ref: main fetch-depth: 0 path: tmp/firstdraft-cli persist-credentials: false - - run: git -C tmp/firstdraft-cli merge-base --is-ancestor 660c02e46cdf36ec76dd556de8c96ef67ed3b035 HEAD - - run: git -C tmp/firstdraft-cli checkout --detach 660c02e46cdf36ec76dd556de8c96ef67ed3b035 + - name: Select the pinned CLI + if: steps.scope.outputs.catalog_only != 'true' + run: | + cli_revision="$(node --input-type=module -e 'import { cliRevision } from "./script/cli-contract/config.mjs"; console.log(cliRevision)')" + git -C tmp/firstdraft-cli merge-base --is-ancestor "$cli_revision" HEAD + git -C tmp/firstdraft-cli checkout --detach "$cli_revision" + - run: sh script/check --cli-root tmp/firstdraft-cli + if: steps.scope.outputs.catalog_only != 'true' - run: node script/check-cli-contract.mjs tmp/firstdraft-cli - - run: node script/check-claude-plugin-package.mjs --cli-root tmp/firstdraft-cli + if: steps.scope.outputs.catalog_only != 'true' - name: Check Codex installation and Skill discovery - if: matrix.node == '24.18.0' + if: steps.scope.outputs.catalog_only != 'true' && matrix.node == '24.18.0' run: | npm install --prefix "$RUNNER_TEMP/codex" --ignore-scripts --no-audit --no-fund @openai/codex@0.154.0 node script/claude-plugin-package.mjs stage "$RUNNER_TEMP/firstdraft" --cli-root tmp/firstdraft-cli diff --git a/.github/workflows/promote.yml b/.github/workflows/promote.yml deleted file mode 100644 index bf3d44d..0000000 --- a/.github/workflows/promote.yml +++ /dev/null @@ -1,84 +0,0 @@ -name: Promote npm defaults - -on: - push: - tags: ["promote-v*"] - workflow_dispatch: - inputs: - cleanup_run_id: - description: Reconciled prior probe run to clean up; leave blank for a new credential check - required: false - type: string - -permissions: {} - -concurrency: - group: claude-plugin-npm-publish - cancel-in-progress: false - -jobs: - verify: - name: Verify published packages - runs-on: ubuntu-latest - timeout-minutes: 5 - permissions: - contents: read - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - fetch-depth: 0 - persist-credentials: false - - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 - with: - node-version: 24.18.0 - package-manager-cache: false - - name: Verify release toolchain - run: | - test "$(node --version)" = "v24.18.0" - test "$(npm --version)" = "11.16.0" - - run: node script/npm-promotion.mjs inspect - - promote: - name: Promote or check npm access - needs: verify - runs-on: ubuntu-latest - timeout-minutes: 5 - environment: npm-promotion - permissions: - contents: read - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - fetch-depth: 0 - persist-credentials: false - - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 - with: - node-version: 24.18.0 - package-manager-cache: false - registry-url: https://registry.npmjs.org/ - - name: Verify release toolchain - run: | - test "$(node --version)" = "v24.18.0" - test "$(npm --version)" = "11.16.0" - - name: Promote the qualified pair - if: github.event_name == 'push' - run: node script/npm-promotion.mjs promote - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_PROMOTION_TOKEN }} - - name: Check token with temporary tags - if: github.event_name == 'workflow_dispatch' && inputs.cleanup_run_id == '' - run: node script/npm-promotion.mjs verify-token - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_PROMOTION_TOKEN }} - - name: Clean up a reconciled probe - if: github.event_name == 'workflow_dispatch' && inputs.cleanup_run_id != '' - run: node script/npm-promotion.mjs cleanup-probe - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_PROMOTION_TOKEN }} - NPM_PROMOTION_CLEANUP_RUN_ID: ${{ inputs.cleanup_run_id }} - - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - if: always() - with: - name: npm-promotion-${{ github.run_id }}-${{ github.run_attempt }} - path: ${{ runner.temp }}/npm-promotion/receipt.json - if-no-files-found: ignore diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 0c31153..70e6f76 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -114,8 +114,11 @@ jobs: fetch-depth: 0 path: tmp/firstdraft-cli persist-credentials: false - - run: git -C tmp/firstdraft-cli merge-base --is-ancestor 660c02e46cdf36ec76dd556de8c96ef67ed3b035 HEAD - - run: git -C tmp/firstdraft-cli checkout --detach 660c02e46cdf36ec76dd556de8c96ef67ed3b035 + - name: Select the pinned CLI + run: | + cli_revision="$(node --input-type=module -e 'import { cliRevision } from "./script/cli-contract/config.mjs"; console.log(cliRevision)')" + git -C tmp/firstdraft-cli merge-base --is-ancestor "$cli_revision" HEAD + git -C tmp/firstdraft-cli checkout --detach "$cli_revision" - run: node script/check-cli-registry-package.mjs --cli-root tmp/firstdraft-cli - run: node script/claude-plugin-package.mjs pack "$RUNNER_TEMP/plugin" --cli-root tmp/firstdraft-cli - name: Verify publication bytes diff --git a/README.md b/README.md index 9082d97..19615a6 100644 --- a/README.md +++ b/README.md @@ -9,9 +9,9 @@ actually receives. Source candidate `0.4.0` requires CLI `0.4.0`, Service API `0.4`, and Foundation Plan `0.20`. It preserves the planning workspace under `.firstdraft/design/` and defaults to current-folder local output. Use explicit `--github` for server Publication. Its identities belong in -[release compatibility](release/compatibility.json); it is unpublished. -The public catalog remains on plugin `0.2.5`. Only `create-full-stack-app` is packaged; the UI Skill auditions remain -deferred source. +[release compatibility](release/compatibility.json). The [public catalog](.claude-plugin/marketplace.json) owns the +installed version; source compatibility does not establish publication. Only `create-full-stack-app` is packaged; +the UI Skill auditions remain deferred source. Trying First Draft as a tester? Start with the [local development guide](https://gist.github.com/raghubetina/3d424a97a1eaa6de8c406e67f32a237e). Start in an empty @@ -49,8 +49,8 @@ Historical pins and release chronology in the evidence archive are receipts, not ## Using Codex -The catalog manifest selects shared plugin `0.2.5`, which includes the compatible CLI and discovers it in either -agent. Drawing Board supplies its own project wrapper and installed CLI. The +The [catalog manifest](.claude-plugin/marketplace.json) selects the shared plugin, which includes the compatible CLI +and discovers it in either agent. Drawing Board supplies its own project wrapper and installed CLI. The [release evidence](evidence/2026-09-15-account-authoring-0.2.5.md) distinguishes package checks from public catalog installation. A fresh authenticated student Codespace journey remains unproved. diff --git a/RELEASING.md b/RELEASING.md index 94b9d6a..0e46a3a 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -9,8 +9,8 @@ longer part of an ordinary release. Coordinate the service, CLI, and Skills thro [`release/compatibility.json`](release/compatibility.json) owns the candidate version, compatible CLI/API/Plan identities, and deterministic package SHA-256. The current source candidate is `@firstdraft.com/claude-code@0.4.0` with CLI `0.4.0`, API `>= 0.4.0`, `< 0.5.0`, and Plan `sketch/0.20`. -It is unpublished. The [marketplace manifest](.claude-plugin/marketplace.json) still selects the published plugin -`0.2.5`; keep it there until the intended new version is actually published. Source compatibility is not public +The [marketplace manifest](.claude-plugin/marketplace.json) independently selects a published plugin version; +retain its selection until the intended new version is actually published. Source compatibility is not public catalog selection. Query npm when releasing rather than treating a dated distribution snapshot as current. Use an ordinary pre-1.0 minor bump for a breaking compatibility change and a patch bump for a compatible change. @@ -28,7 +28,8 @@ The real GitHub `npm` environment protection still applies; do not bypass it or ## 1. Use the checks already completed 1. Resolve the candidate commits for the service, CLI, and Skills. Confirm the Skills commit is on `main`, and the - CLI pin and compatibility metadata match the intended release. + CLI pin and compatibility metadata match the intended release. `script/cli-contract/config.mjs` owns the exact + CLI revision and runtime digest; workflows read that configuration rather than copying its values. 2. Reuse successful hosted CI for the exact release commit. CI already runs repository tests, the pinned CLI contract, deterministic package checks, and Codex discovery. Do not rerun that suite, dependency audit, both client installations, or behavioral evaluation sessions merely because the release is about to publish. @@ -40,9 +41,8 @@ For development or a failed check, the relevant reproduction commands are: ```sh npm ci --ignore-scripts -sh script/check +sh script/check --cli-root /path/to/exact/cli node script/check-cli-contract.mjs /path/to/exact/cli -node script/check-claude-plugin-package.mjs --cli-root /path/to/exact/cli ``` These are troubleshooting and pre-merge commands, not a second post-merge release checklist. @@ -92,7 +92,8 @@ A public package verification is a read-only reconciliation, not another live ap ## 4. Select the published version in the catalog Update `.claude-plugin/marketplace.json` to the exact published version. Keep its version and npm source version -aligned, run normal PR checks, and merge under the already authorized release scope. Never point the live catalog +aligned and merge under the already authorized release scope. A version-selection-only change uses the catalog +metadata and published-version checks; any other change runs the full CI matrix. Never point the live catalog at an unpublished candidate. Catalog CI validates this small change; do not add a second product smoke. A normal package release does not require installing both Claude and Codex again after the catalog merge. Verify a @@ -111,6 +112,6 @@ start again without reconciliation. A validated retained Compilation ID permits `plan compile --github`, the documented unchanged-byte, same-singleton Publication replay remains available after the prior invocation exits; it never applies to an ambiguous Plan push or direct Compilation start. -The [npm-default repair workflow](docs/npm-promotion.md) remains available for explicitly requested repairs of -already-published versions. It is not part of the ordinary release. Record new release observations in -[`evidence/`](evidence/README.md) without rewriting historical receipts. +Use the short [npm-default repair procedure](docs/npm-promotion.md) for an approved change to an already-published +version. It uses standard npm dist-tags, without a separate promotion workflow or credential probe. Record new +release observations in [`evidence/`](evidence/README.md) without rewriting historical receipts. diff --git a/docs/README.md b/docs/README.md index f8089cc..03ed23e 100644 --- a/docs/README.md +++ b/docs/README.md @@ -10,7 +10,7 @@ historical observations intentionally have different owners. | What package is the source candidate compatible with? | [`release/compatibility.json`](../release/compatibility.json) | | What does the shared Claude/Codex catalog select? | [`.claude-plugin/marketplace.json`](../.claude-plugin/marketplace.json) | | What is the current release procedure? | [`RELEASING.md`](../RELEASING.md) | -| How do I repair an existing npm default? | [npm promotion](npm-promotion.md) | +| How do I repair an existing npm default? | [npm default repair](npm-promotion.md) | | What does the agent execute? | [`SKILL.md`](../skills/create-full-stack-app/SKILL.md) | | How does an agent extend or review generated UI? | The app's `UI.md`; [deferred Skill status](../README.md#ui-continuation) | | What is exact Plan syntax? | [JSON Schema](../skills/create-full-stack-app/references/foundation-plan-0.20.schema.json) | diff --git a/docs/npm-promotion.md b/docs/npm-promotion.md index 702a99f..3f65e21 100644 --- a/docs/npm-promotion.md +++ b/docs/npm-promotion.md @@ -1,153 +1,35 @@ -# Repair npm defaults through GitHub +# Repairing an npm default -Ordinary releases publish directly to `latest` through OIDC; follow [RELEASING.md](../RELEASING.md). This legacy -workflow is only for an explicitly requested repair of an already-published compatible pair. It is not a publication -prerequisite and does not justify a token setup, credential probe, or separate approval during a normal release. +Ordinary releases publish directly to `latest`. Use this procedure only to repair a dist-tag on an +already-published version, including an explicitly approved rollback. The former `next` promotion workflow and +credential probes are retired; their [credential cleanup is complete](../evidence/2026-09-22-npm-promotion-retirement.md). +Ordinary publication uses GitHub Actions trusted publishing, without a local npm login. -The retained `promote-v` workflow changes the compatible CLI and plugin `latest` tags in that order. -It requires its own configured `npm-promotion` environment. It neither publishes package bytes nor deploys the -service. The historical helper expects the matching `next` tags and catalog; use it only when those preconditions -already hold. Do not move `next` merely to make an ordinary release fit this retired sequence. - -npm trusted publishing authenticates new-version publication. Existing-version dist-tag repair may require npm -authentication or the retained scoped token. Keep one operator and reconcile ambiguous writes read-only. - -## Initial setup and renewal - -Configure these controls before using [the workflow](../.github/workflows/promote.yml): - -- Active tag rulesets cover `refs/tags/promote-v*`. Restrict creation to organization administrators; disallow - updates and deletion without a bypass actor. Keep the existing `claude-v*` publication protections. -- The `npm-promotion` environment requires the release owner's review, with administrator bypass disabled. Its - deployment policies permit `promote-v*` tags and the `main` branch for the credential check. A self-review is - allowed, matching the existing single-operator publication environment. -- Create one granular npm token restricted to **only** `@firstdraft.com/cli` and `@firstdraft.com/claude-code`, - with **stage-only** access and **Bypass two-factor authentication** enabled. Grant no organization-management - access. Choose an expiry and arrange renewal before it expires. The stage-only token UI also lists staging, - deprecation, and unpublishing; npm does not offer a dist-tag-only permission. -- Store it only as `NPM_PROMOTION_TOKEN` in that environment. Pass the secret through stdin or the GitHub UI; - never put it in a command argument, receipt, repository file, or log. Do not add it to the publication environment. -- Verify both packages permit granular tokens with bypass 2FA. The npm setting - [“Require two-factor authentication and disallow tokens”](https://docs.npmjs.com/requiring-2fa-for-package-publishing-and-settings-modification/) - blocks this workflow. If a package uses that setting, its owner must explicitly allow scoped token operations. - Trusted publication remains configured separately. - -Before a new credential check, reconcile and remove any retained `promotion-check-*` tags from either package using -the [cleanup procedure](#partial-results-and-recovery). Supply `cleanup_run_id` for that recovery dispatch; leaving it -blank creates a new probe. The check rejects retained probes on either package before adding another. - -First check existing write receipts against the proof requirements below. The -[current token's CLI probe deletion returned 403](../evidence/2026-09-13-npm-token-write-verification.md#deletion-boundary). -Its writes are already proved; do not repeat that rejected probe. A full check with that behavior stops before the -plugin and leaves a CLI probe requiring [interactive cleanup](#partial-results-and-recovery). - -When new write proof is needed, after setup or renewal and any cleanup, dispatch `Promote npm defaults` on current -`main` with `cleanup_run_id` blank and be prepared for interactive cleanup if the rejection persists. -This is a credential check, not a release promotion. It requires `next`, `latest`, and the catalog to select the -qualified pair already. For each package it -adds `promotion-check-` selecting that same version, verifies the result, removes that tag, and verifies the -original tags are restored. Approve the environment job after inspecting its verification job. A successful no-op -release promotion alone would not prove token write access. -This probe does not prove least privilege. At creation and each renewal, inspect the token's two-package list, -stage-only permission, lack of organization access, and expiry in npm; retain its name and expiry in setup evidence. - -Promotion needs a successful tag write on each package with the configured token, with the resulting selections -independently verified. A no-op does not count. Receipts from separate runs may establish those writes if secret -metadata confirms the token was not replaced between them. Recheck its current npm permissions and both packages' -token policies; each package's write must postdate the latest relevant policy or credential change. A new token -cannot inherit the old token's write proof. All probes must also be reconciled and removed before -declaring setup ready. Record that combined proof explicitly; never relabel a failed credential-check run as passed. -The full check additionally exercises tag deletion, which normal promotion does not use. If deletion is rejected, -follow [recovery](#partial-results-and-recovery) and retain that limitation separately. A package the failed check -never reached remains unexercised until a separately approved tag write supplies that proof. Repeating the same -rejected cleanup or broadening the token solely to make the probe pass is not required for promotion. - -## Repair a qualified release - -Obtain release approval; an existing approval for the named release sequence is sufficient. Reconcile the exact -source revisions, package hashes, compatibility, deployed service, and current registry selections. Reuse the -existing qualification and CI for unchanged inputs; this repair does not require another live journey. The workflow verifies immutable package -identities and distribution state, but cannot establish that a human approved the release or that qualification ran. - -Use a clean, reviewed `main` checkout containing the promotion workflow. The promotion tag points at that reviewed -commit, which may be newer than the `claude-v` package-source tag. Do not tag the older source commit merely -because it built the package: it may not contain the promotion workflow. No new package version is needed when only -release tooling or maintainer documentation changed. +Use the existing release approval when it covers the intended package and version. Keep one operator across CLI +and plugin registry changes. Identify the exact package version and compare its registry integrity with the +successful publication or retained release evidence before changing a tag: ```sh -git fetch origin main --tags -git switch main -git merge --ff-only origin/main -node script/npm-promotion.mjs inspect -version=$(node -p 'JSON.parse(require("fs").readFileSync("release/compatibility.json")).version') -git tag -a "promote-v$version" -m "Promote qualified npm defaults for plugin $version" -git push origin "refs/tags/promote-v$version" +package='@firstdraft.com/cli' # or @firstdraft.com/claude-code +version='0.4.0' # the approved, already-published version +npm view "$package@$version" name version dist --json --prefer-online --registry=https://registry.npmjs.org/ +npm dist-tag ls "$package" --prefer-online --registry=https://registry.npmjs.org/ ``` -Before pushing, verify the tag rulesets, environment restrictions, current exact-head CI, and the tag target. A tag -name is unique and immutable. Its plugin version and the canonical CLI pin select the pair; there is no arbitrary -package, registry, or version input. The helper checks that the tag target belongs to first-parent `main` history, -the publication tag has the same compatibility bytes, the CLI publication tag matches its source pin, and both the -tagged and current-main catalogs select the qualified plugin. -The CLI's existing `v` publication tag must resolve to the canonical CLI revision; the local inspection -checks this precondition before a promotion tag is created. - -It downloads both public npm tarballs, verifies their registry integrity, matches the plugin's qualified SHA-256, -and compares all bundled CLI file bytes and modes with the standalone package. It rejects a changed `next`, a newer -`latest`, an unprotected promotion tag, or an unexpected registry URL. An already-selected version is a read-only -success. The secret-bearing job repeats verification after environment approval. - -Approve the environment job in GitHub, then inspect its receipt and independently reconcile both packages' tags and -integrity. Retain the run URL and receipt in release evidence. `next` remains unchanged. This operation does not -upgrade an installed CLI/plugin or change Drawing Board's exact source pin. - -## Partial results and recovery - -The two npm writes are sequential, not atomic. The workflow shares the publication workflow's concurrency group -within Skills; it cannot serialize a CLI-repository publication or a manual npm mutation. Keep those operations with -the same release operator. It rereads both packages before each promotion write and never automatically rolls back. -An approval waiting in the shared group blocks another publication. Cancel an abandoned run instead of leaving it -pending; reconcile any started mutation before cancellation or another release. +If `latest` already selects the intended version, no write is needed. Otherwise use the standard +[npm dist-tag command](https://docs.npmjs.com/cli/v11/commands/npm-dist-tag/): -Each invocation attempts a needed write once, with npm transport retries disabled. The pinned -[npm command](https://github.com/npm/cli/blob/v11.16.0/lib/commands/dist-tag.js) waits for the PUT or DELETE response -without verifying a subsequent read. `--prefer-online` revalidates npm's own cached tag metadata before each -command, so probe cleanup can see the preceding addition. This is separate from the helper's anonymous readbacks: -after a successful command, it makes up to six of those reads, -waiting two seconds between them only while the **complete tag map exactly matches its pre-write state**. It proceeds -only when the complete map equals the requested result. Any other tag change or read error stops immediately. -This adds at most ten seconds of waiting per write, excluding request time; it is a bounded verification window, -not a guarantee about npm propagation. Exhausting it requires read-only reconciliation, never another automatic write. - -A failed command receives one immediate readback and stops, subject to the observed-probe cleanup below. Its -`npm-promotion--` artifact and job summary record requested changes, exit status, every successful -readback in order, and `readback_status` without credentials. Failed commands also retain `command_error`: npm stderr -with terminal controls removed, the configured token and npm token-shaped strings redacted, then limited to 4,096 -characters. Older receipts have exit status only. `after` is the last observed tag map, so an earlier -sample can remain there when a later read fails; it does not establish the final outcome. Runner loss or cancellation -can also prevent receipt upload: query the registry before any further mutation. - -Before declaring promotion complete, a strict final pair check rereads each package once. `final_verification` -retains its `incomplete` or `verified` status and returned tag maps, including partial observations if a read fails. -Verified writes with an incomplete closing check require read-only reconciliation; never retry already-observed writes. +```sh +npm dist-tag add "$package@$version" latest --prefer-online --registry=https://registry.npmjs.org/ +npm dist-tag ls "$package" --prefer-online --registry=https://registry.npmjs.org/ +npm view "$package@latest" name version dist --json --prefer-online --registry=https://registry.npmjs.org/ +``` -Do not blindly rerun a failed job. A promotion rerun is read-only: it can confirm both defaults already moved, but -refuses to finish a partial promotion. Inspect the registry and receipt, repair the cause, and obtain authorization -for the concrete remaining mutation. Push a new protected `promote-v-retry-` tag -from the reviewed main commit; for example, `promote-v0.2.2-retry-1`. This requests another environment-reviewed run -without reusing the original tag. It repeats all candidate checks, skips already-selected versions, and writes only -the remaining defaults. This also handles a first attempt that changed nothing, such as an expired token. Never -move/delete/reuse either tag. After recovery, a rerun of the original job may record completion without another write. +Complete npm's maintainer authentication or second-factor prompt if requested. This authentication is for a +registry mutation, not installation or use. Do not create a long-lived CI token or temporary tags to rehearse it. -A credential-check rerun refuses writes. If a probe remains, verify its run ID and exact selected version, reconcile -both permanent tags, and remove only that observed probe under the original cleanup authorization. Dispatch the -workflow on passing current `main` with `cleanup_run_id` set to that prior run ID. After package verification, -approve its protected environment job. This mode creates no probe: it removes only the named `promotion-check-*` -tag from the two qualified packages, rejects changed versions or maps, and verifies the final maps. An absent tag -needs no write. A cleanup rerun refuses writes; inspect the receipt and registry before any further attempt. -If token cleanup fails, an authorized operator can remove that exact observed tag through interactive npm. For an -already reconciled 403, use that interactive path directly; do not repeat the same token deletion without a repair. -Do not dispatch -another check to evade an uncertain outcome. If the add command reported failure but the exact probe is observed, -the original invocation removes its own probe once before stopping; an ambiguous add or cleanup requires operator -reconciliation. A new dispatch after a reconciled token repair gets a new run ID. +After an error, timeout, or interruption, inspect the registry read-only before another write. An observed +successful change needs no retry. Record the exact version, integrity, and resulting `latest` selection. CLI and +plugin changes are separate writes; reconcile each and continue only the still-needed approved change. `next` +need not move. Dist-tags do not change the public catalog or update existing installations; any intended catalog +change follows [RELEASING.md](../RELEASING.md#4-select-the-published-version-in-the-catalog). diff --git a/evidence/2026-09-22-npm-promotion-retirement.md b/evidence/2026-09-22-npm-promotion-retirement.md new file mode 100644 index 0000000..975a93c --- /dev/null +++ b/evidence/2026-09-22-npm-promotion-retirement.md @@ -0,0 +1,22 @@ +# npm promotion retirement + +Observed 2026-09-22 after the owner authorized cleanup. + +- Confirmed no unfinished legacy promotion jobs, then disabled `Promote npm defaults` + (`firstdraft/skills`, workflow `355863836`). GitHub reported `disabled_manually`. + [PR #87](https://github.com/firstdraft/skills/pull/87) removes its source file. +- Deleted `NPM_PROMOTION_TOKEN` from the `npm-promotion` environment, verified its secret list was empty, + then deleted that environment and its deployment policies. The repository's remaining environment is `npm`; + its required reviewer and deployment branch protection remain configured. +- Revoked npm token `FirstDraftGitHubPromotion-20260913`. npm displayed `deleted 1 token` and an empty token + list; `npm token list --json` independently returned no tokens. No credential values are retained here. +- Ran `npm access set mfa=publish @firstdraft.com/cli` with the owner's security-key authentication. + It exited successfully. A fresh package Settings page selected + “Require two-factor authentication and disallow bypass 2fa tokens (recommended).” +- That page still listed trusted publisher `firstdraft/cli`, workflow `publish.yml`, environment `npm`, + with `npm publish` and `npm stage publish` permissions. The npm page explicitly states that all publishing + access options remain compatible with OIDC trusted publishers. + +Publication continues through the existing tag-triggered GitHub Actions workflows directly to `latest`. +The interactive npm authentication above was for this retirement and security-setting change; normal OIDC +publication requires no local npm login. Package versions, dist-tags, and the public catalog were unchanged. diff --git a/evidence/README.md b/evidence/README.md index 9bb64a2..d2d1963 100644 --- a/evidence/README.md +++ b/evidence/README.md @@ -35,6 +35,7 @@ The source/public split observed at the start of this documentation change is ca | Record | Observed boundary | |---|---| +| [`2026-09-22-npm-promotion-retirement.md`](2026-09-22-npm-promotion-retirement.md) | Revoked the promotion token, removed its GitHub secret and environment, disabled the legacy workflow, and restored the CLI's restrictive publishing policy while preserving GitHub trusted publishing | | [`2026-09-15-shared-plugin-0.2.4-default-promotion.md`](2026-09-15-shared-plugin-0.2.4-default-promotion.md) | Protected-tag promotion, one plugin write with bounded stale-read reconciliation, and independent final tags/archive hashes; [machine receipt](2026-09-15-shared-plugin-0.2.4-default-promotion.json) | | [`2026-09-15-shared-plugin-0.2.4-publication.md`](2026-09-15-shared-plugin-0.2.4-publication.md) | Protected publication, delayed registry visibility, verified provenance, exact registry-package adapters, and observed staging readiness before catalog/default promotion; [machine receipt](2026-09-15-shared-plugin-0.2.4-publication.json) | | [`2026-09-13-npm-token-write-verification.md`](2026-09-13-npm-token-write-verification.md) | CLI probe and plugin default writes verified; CLI default move unexercised; token DELETE returned 403 and interactive cleanup restored clean maps; [machine receipt](2026-09-13-npm-token-write-verification.json) | diff --git a/script/check b/script/check index 453f952..f17a146 100755 --- a/script/check +++ b/script/check @@ -5,5 +5,5 @@ repository=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) cd "$repository" node script/check-release-compatibility.mjs -node script/check-claude-plugin-package.mjs +node script/check-claude-plugin-package.mjs "$@" node --test 'test/**/*.test.mjs' diff --git a/script/check-catalog.mjs b/script/check-catalog.mjs new file mode 100644 index 0000000..d3d7dbe --- /dev/null +++ b/script/check-catalog.mjs @@ -0,0 +1,38 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { readFile } from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { checkSkillsReleaseCompatibility } from "./check-release-compatibility.mjs"; + +const repository = path.dirname(path.dirname(fileURLToPath(import.meta.url))); + +export function assertPublishedCatalogSelection(source, published) { + assert.equal(published.name, source.package, "registry package must match the catalog selection"); + assert.equal(published.version, source.version, "catalog must select an already-published exact version"); + assert.match(published.dist?.integrity ?? "", /^sha512-[A-Za-z0-9+/]{86}==$/, "published version must have registry integrity"); + assert.equal( + published.dist?.tarball, + `${source.registry}${source.package}/-/${source.package.split("/").at(-1)}-${source.version}.tgz`, + "published tarball must use the selected npm registry and version", + ); +} + +export async function checkCatalog(root = repository) { + await checkSkillsReleaseCompatibility(root); + const catalog = JSON.parse(await readFile(path.join(root, ".claude-plugin/marketplace.json"), "utf8")); + assert.equal(catalog.plugins.length, 1, "catalog must contain only the First Draft plugin"); + const { source } = catalog.plugins[0]; + const result = spawnSync(process.env.npm_execpath || "npm", [ + "view", `${source.package}@${source.version}`, "name", "version", "dist", "--json", "--prefer-online", + `--registry=${source.registry}`, + ], { encoding: "utf8" }); + assert.equal(result.status, 0, [result.error?.message, result.stderr].filter(Boolean).join("; ")); + assertPublishedCatalogSelection(source, JSON.parse(result.stdout)); + return `${source.package}@${source.version}`; +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) { + process.stdout.write(`Published catalog selection: ${await checkCatalog()}\n`); +} diff --git a/script/ci-scope.mjs b/script/ci-scope.mjs new file mode 100644 index 0000000..8173631 --- /dev/null +++ b/script/ci-scope.mjs @@ -0,0 +1,39 @@ +import { spawnSync } from "node:child_process"; +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { isDeepStrictEqual } from "node:util"; + +const repository = path.dirname(path.dirname(fileURLToPath(import.meta.url))); +const catalogPath = ".claude-plugin/marketplace.json"; + +export function isCatalogSelectionChange(paths, before, after) { + if (paths.length !== 1 || paths[0] !== catalogPath) return false; + const withoutSelection = (catalog) => { + const copy = structuredClone(catalog); + if (copy.plugins?.length !== 1 || copy.plugins[0].name !== "firstdraft") return null; + delete copy.plugins[0].version; + if (copy.plugins[0].source?.source !== "npm") return null; + delete copy.plugins[0].source.version; + return copy; + }; + const beforeMetadata = withoutSelection(before); + const afterMetadata = withoutSelection(after); + return beforeMetadata !== null && afterMetadata !== null && isDeepStrictEqual(beforeMetadata, afterMetadata); +} + +export function catalogOnlyChange(base, root = repository) { + const git = (...args) => spawnSync("git", args, { cwd: root, encoding: "utf8" }); + if (!base || git("cat-file", "-e", `${base}^{commit}`).status !== 0) return false; + const diff = git("diff", "--name-only", "--no-renames", "-z", base, "HEAD"); + if (diff.status !== 0) throw new Error(diff.stderr); + const paths = diff.stdout.split("\0").filter(Boolean); + if (paths.length !== 1 || paths[0] !== catalogPath) return false; + const previous = git("show", `${base}:${catalogPath}`); + if (previous.status !== 0) return false; + return isCatalogSelectionChange(paths, JSON.parse(previous.stdout), JSON.parse(readFileSync(path.join(root, catalogPath), "utf8"))); +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) { + process.stdout.write(`catalog_only=${catalogOnlyChange(process.argv[2])}\n`); +} diff --git a/script/npm-promotion.mjs b/script/npm-promotion.mjs deleted file mode 100644 index 30cb500..0000000 --- a/script/npm-promotion.mjs +++ /dev/null @@ -1,416 +0,0 @@ -import assert from "node:assert/strict"; -import { spawnSync } from "node:child_process"; -import { createHash } from "node:crypto"; -import { appendFile, mkdir, readFile, writeFile } from "node:fs/promises"; -import path from "node:path"; -import { setTimeout as delay } from "node:timers/promises"; -import { fileURLToPath } from "node:url"; -import { isDeepStrictEqual, stripVTControlCharacters } from "node:util"; - -import { cliPackageInventory } from "./check-cli-registry-package.mjs"; -import { - compareSemanticVersions, - isOrdinaryPreOneVersion, - isSemanticVersion, -} from "./check-release-compatibility.mjs"; -import { - cliPackageName, - cliPackageVersion, - cliRevision, -} from "./cli-contract/config.mjs"; - -const repository = path.dirname(path.dirname(fileURLToPath(import.meta.url))); -const registry = "https://registry.npmjs.org/"; -const pluginPackage = "@firstdraft.com/claude-code"; -const packageNames = [cliPackageName, pluginPackage]; - -export function assertDistributionState(packages) { - assert.deepEqual(packages.map(({ name }) => name), packageNames); - for (const { name, version, tags } of packages) { - assert(isOrdinaryPreOneVersion(version), `${name}: expected an ordinary 0.x version`); - assert.equal(tags.next, version, `${name}: next must select the qualified version`); - assert(isSemanticVersion(tags.latest), `${name}: latest must already identify a version`); - assert( - compareSemanticVersions(tags.latest, version) <= 0, - `${name}: refusing to move latest backward`, - ); - } -} - -export function assertCatalog(catalog, version) { - const entries = catalog.plugins.filter(({ name }) => name === "firstdraft"); - assert.equal(entries.length, 1, "expected one First Draft catalog entry"); - assert.equal(entries[0].version, version, "catalog must select the qualified plugin"); - assert.deepEqual(entries[0].source, { - source: "npm", - package: pluginPackage, - version, - registry, - }); -} - -export function assertPackageBytes(packages, pluginSha256) { - assert.deepEqual(packages.map(({ name }) => name), packageNames); - const [cli, plugin] = packages; - for (const item of packages) { - assert.equal(item.sha256, digest(item.bytes, "sha256")); - assert.equal(item.integrity, `sha512-${digest(item.bytes, "sha512", "base64")}`); - } - assert.equal(plugin.sha256, pluginSha256, "plugin tarball differs from the qualified artifact"); - const cliEntries = cliPackageInventory(cli.bytes); - const pluginEntries = cliPackageInventory(plugin.bytes, "plugin package"); - const bundledEntries = pluginEntries - .filter(({ packagePath }) => packagePath.startsWith("vendor/cli/")) - .map((entry) => ({ ...entry, packagePath: entry.packagePath.slice("vendor/cli/".length) })); - assert.deepEqual(bundledEntries, cliEntries, "registry CLI differs from the qualified plugin's bundled CLI"); - for (const [item, entries] of [[cli, cliEntries], [plugin, pluginEntries]]) { - const manifest = entries.find(({ packagePath }) => packagePath === "package.json"); - assert(manifest, `${item.name}: package.json is missing`); - const document = JSON.parse(manifest.bytes.toString("utf8")); - assert.equal(document.name, item.name); - assert.equal(document.version, item.version); - } -} - -export function assertGithubContext(env, mode, version) { - assert.equal(env.GITHUB_ACTIONS, "true", "writes run only through GitHub Actions"); - assert.equal(env.GITHUB_REPOSITORY, "firstdraft/skills"); - assert(/^[1-9]\d*$/.test(env.GITHUB_RUN_ID), "expected a GitHub run ID"); - assert(/^[1-9]\d*$/.test(env.GITHUB_RUN_ATTEMPT), "expected a GitHub run attempt"); - if (env.GITHUB_EVENT_NAME === "push") { - assert(mode === "inspect" || mode === "promote"); - assert.equal(env.GITHUB_REF_TYPE, "tag"); - assert.equal(env.GITHUB_REF_PROTECTED, "true"); - const prefix = `promote-v${version}`; - assert(env.GITHUB_REF_NAME === prefix || ( - env.GITHUB_REF_NAME?.startsWith(`${prefix}-retry-`) - && /^[1-9]\d*$/.test(env.GITHUB_REF_NAME.slice(`${prefix}-retry-`.length)) - ), "expected the qualified promotion tag or a separately approved retry tag"); - assert.equal(env.GITHUB_REF, `refs/tags/${env.GITHUB_REF_NAME}`); - } else { - assert.equal(env.GITHUB_EVENT_NAME, "workflow_dispatch"); - assert(mode === "inspect" || mode === "verify-token" || mode === "cleanup-probe"); - assert.equal(env.GITHUB_REF, "refs/heads/main"); - } -} - -async function readTagsAfterWrite({ operation, expected, readTags, wait }) { - operation.readbacks = []; - for (let attempt = 1; attempt <= 6; attempt += 1) { - let after; - try { - after = await readTags(operation.package); - } catch (error) { - operation.readback_status = "read-failed"; - throw error; - } - operation.readbacks.push(after); - operation.after = after; - if (isDeepStrictEqual(after, expected)) { - operation.readback_status = "verified"; - return after; - } - if (!isDeepStrictEqual(after, operation.before)) { - operation.readback_status = "conflicting-state"; - return after; - } - if (operation.command_status !== 0) { - operation.readback_status = "prior-state-after-command-failure"; - return after; - } - if (attempt === 6) { - operation.readback_status = "prior-state-timeout"; - assert.fail(`${operation.package}: readback did not converge after six reads; reconcile before another mutation`); - } - await wait(2000); - } -} - -export async function promoteLatest({ candidate, readTags, writeTag, attempt, report, wait = delay }) { - assertDistributionState(candidate.packages); - for (const item of candidate.packages) { - const current = await Promise.all(candidate.packages.map(async (entry) => - ({ ...entry, tags: await readTags(entry.name) }))); - assertDistributionState(current); - const before = current.find(({ name }) => name === item.name).tags; - if (before.latest === item.version) { - report.operations.push({ package: item.name, status: "already-selected", tags: before }); - continue; - } - assert.equal(attempt, "1", "reruns reconcile only; an incomplete promotion needs operator review"); - const operation = { package: item.name, before, requested: { latest: item.version } }; - report.operations.push(operation); - const result = await writeTag("add", item.name, item.version, "latest"); - operation.command_status = result.status; - if (result.error) operation.command_error = result.error; - const expected = { ...before, latest: item.version }; - const after = await readTagsAfterWrite({ operation, expected, readTags, wait }); - assert.equal(result.status, 0, "npm reported a write failure; inspect the receipt before any further mutation"); - assert.deepEqual(after, expected, "npm tag state changed unexpectedly"); - } - const final = []; - report.final_verification = { status: "incomplete", packages: final }; - for (const item of candidate.packages) { - final.push({ name: item.name, version: item.version, tags: await readTags(item.name) }); - } - assertDistributionState(final); - assert(final.every(({ version, tags }) => tags.latest === version), "promotion is incomplete"); - report.final_verification.status = "verified"; - report.status = "promoted"; -} - -export async function verifyToken({ candidate, readTags, writeTag, runId, attempt, report, wait = delay }) { - assertDistributionState(candidate.packages); - assert.equal(attempt, "1", "credential-check reruns are read-only; inspect retained probe tags"); - assert(/^[1-9]\d*$/.test(runId), "expected a GitHub run ID"); - const tag = `promotion-check-${runId}`; - report.probe_tag = tag; - for (const item of candidate.packages) { - const current = await Promise.all(candidate.packages.map(async (entry) => - ({ ...entry, tags: await readTags(entry.name) }))); - for (const { version, tags } of current) { - assert.equal(tags.latest, version, "check credentials against the already-promoted release"); - assert.equal(tags.next, version); - assert(!Object.keys(tags).some((key) => key.startsWith("promotion-check-")), - "a retained probe must be reconciled before another credential check"); - } - const before = current.find(({ name }) => name === item.name).tags; - const addition = { package: item.name, phase: "add-probe", before, requested: { [tag]: item.version } }; - report.operations.push(addition); - const added = await writeTag("add", item.name, item.version, tag); - addition.command_status = added.status; - if (added.error) addition.command_error = added.error; - const expected = { ...before, [tag]: item.version }; - const afterAdd = await readTagsAfterWrite({ operation: addition, expected, readTags, wait }); - assert.deepEqual(afterAdd, expected, "probe write needs read-only reconciliation"); - const removal = { package: item.name, phase: "remove-probe", before: afterAdd }; - report.operations.push(removal); - const removed = await writeTag("rm", item.name, item.version, tag); - removal.command_status = removed.status; - if (removed.error) removal.command_error = removed.error; - const afterRemove = await readTagsAfterWrite({ operation: removal, expected: before, readTags, wait }); - assert.deepEqual(afterRemove, before, "probe cleanup needs read-only reconciliation"); - assert.equal(added.status, 0, "npm reported a probe failure; the observed probe was cleaned up"); - assert.equal(removed.status, 0, "npm reported a cleanup failure; the probe is observed absent"); - } - report.status = "credentials-verified"; -} - -export async function cleanupProbe({ candidate, readTags, writeTag, probeRunId, runId, attempt, report, wait = delay }) { - assert.equal(attempt, "1", "cleanup reruns refuse writes; reconcile the retained receipt"); - assert(typeof probeRunId === "string" && /^[1-9]\d*$/.test(probeRunId), "expected the reconciled probe run ID"); - assert(typeof runId === "string" && /^[1-9]\d*$/.test(runId) && probeRunId !== runId, "cleanup must name a prior run"); - assertDistributionState(candidate.packages); - const tag = `promotion-check-${probeRunId}`; - report.probe_tag = tag; - const current = await Promise.all(candidate.packages.map(async (item) => - ({ ...item, tags: await readTags(item.name) }))); - for (const { name, version, tags } of current) { - assert.equal(tags.next, version, `${name}: next changed before cleanup`); - assert.equal(tags.latest, version, `${name}: latest changed before cleanup`); - assert(tags[tag] === undefined || tags[tag] === version, `${name}: probe version changed before cleanup`); - } - for (const { name, version } of current) { - const before = await readTags(name); - assert.deepEqual(before, current.find((item) => item.name === name).tags, "tags changed before cleanup"); - if (before[tag] === undefined) { - report.operations.push({ package: name, phase: "cleanup-probe", status: "already-absent", tags: before }); - continue; - } - const operation = { package: name, phase: "cleanup-probe", before }; - report.operations.push(operation); - const removed = await writeTag("rm", name, version, tag); - operation.command_status = removed.status; - if (removed.error) operation.command_error = removed.error; - const expected = { ...before }; - delete expected[tag]; - const after = await readTagsAfterWrite({ operation, expected, readTags, wait }); - assert.deepEqual(after, expected, "probe cleanup needs read-only reconciliation"); - assert.equal(removed.status, 0, "npm reported cleanup failure; the probe is observed absent"); - } - report.final_verification = { status: "incomplete", packages: [] }; - for (const { name, version, tags } of current) { - const after = await readTags(name); - report.final_verification.packages.push({ name, version, tags: after }); - const expected = { ...tags }; - delete expected[tag]; - assert.deepEqual(after, expected, "tag map changed after cleanup"); - } - report.final_verification.status = "verified"; - report.status = "probe-cleaned"; -} - -export async function inspectPromotion({ root = repository, env = process.env } = {}) { - const compatibilityBytes = await readFile(path.join(root, "release/compatibility.json"), "utf8"); - const compatibility = JSON.parse(compatibilityBytes); - assert.equal(compatibility.format, "firstdraft.release-compatibility/1"); - assert.equal(compatibility.component, "skills"); - assert.equal(compatibility.plugin_source.package, pluginPackage); - assert(isOrdinaryPreOneVersion(compatibility.version)); - assert.deepEqual(compatibility.requires.cli, [`= ${cliPackageVersion}`]); - const version = compatibility.version; - const publicationRef = `refs/tags/claude-v${version}`; - const refspecs = [ - "+refs/heads/main:refs/remotes/origin/main", - `+${publicationRef}:refs/promotion-check/publication`, - ]; - if (env.GITHUB_ACTIONS === "true") { - assertGithubContext(env, "inspect", version); - if (env.GITHUB_EVENT_NAME === "push") { - refspecs.push(`+${env.GITHUB_REF}:refs/promotion-check/request`); - } - } - git(root, ["fetch", "--no-tags", "https://github.com/firstdraft/skills", ...refspecs]); - assert.equal(git(root, ["show", "refs/promotion-check/publication:release/compatibility.json"], false), compatibilityBytes); - const source = git(root, ["rev-parse", "refs/promotion-check/publication^{commit}"]); - const main = git(root, ["rev-parse", "refs/remotes/origin/main"]); - if (env.GITHUB_ACTIONS === "true") { - const head = git(root, ["rev-parse", "HEAD"]); - assert.equal(head, git(root, ["rev-parse", `${env.GITHUB_SHA}^{commit}`])); - assert(git(root, ["rev-list", "--first-parent", main]).split("\n").includes(head), "promotion workflow must be from main history"); - if (env.GITHUB_EVENT_NAME === "push") { - assert.equal(git(root, ["rev-parse", "refs/promotion-check/request^{commit}"]), head); - } else { - assert.equal(head, main, "credential checks must use current main"); - } - } - const localCatalog = JSON.parse(await readFile(path.join(root, ".claude-plugin/marketplace.json"), "utf8")); - const currentCatalog = JSON.parse(git(root, ["show", "refs/remotes/origin/main:.claude-plugin/marketplace.json"])); - assertCatalog(localCatalog, version); - assertCatalog(currentCatalog, version); - const cliTag = `refs/tags/v${cliPackageVersion}`; - const cliRefs = git(root, ["ls-remote", "https://github.com/firstdraft/cli", cliTag, `${cliTag}^{}`]) - .split("\n").map((line) => line.split(/\s+/)); - const cliSource = cliRefs.find(([, ref]) => ref === `${cliTag}^{}`)?.[0] - ?? cliRefs.find(([, ref]) => ref === cliTag)?.[0]; - assert.equal(cliSource, cliRevision, "CLI publication tag differs from the qualified source pin"); - const packages = await Promise.all([ - fetchPackage(cliPackageName, cliPackageVersion), - fetchPackage(pluginPackage, version), - ]); - assertDistributionState(packages); - assertPackageBytes(packages, compatibility.plugin_source.tarball_sha256); - return { - plugin_version: version, - skills_source: source, - cli_source: cliSource, - catalog_commit: main, - packages: packages.map(({ bytes, ...item }) => item), - }; -} - -async function fetchPackage(name, version) { - const metadata = await fetchJson(`${registry}${name}`); - assert.equal(metadata.name, name); - const selected = metadata.versions[version]; - assert(selected, `${name}@${version} is not published`); - assert.equal(selected.name, name); - assert.equal(selected.version, version); - const filename = `${name.split("/")[1]}-${version}.tgz`; - assert.equal(selected.dist.tarball, `${registry}${name}/-/${filename}`); - const response = await fetchPublic(selected.dist.tarball); - const bytes = Buffer.from(await response.arrayBuffer()); - return { - name, version, tags: metadata["dist-tags"], bytes, - sha256: digest(bytes, "sha256"), integrity: selected.dist.integrity, - }; -} - -async function readTags(name) { - assert(packageNames.includes(name)); - const tags = await fetchJson(`${registry}-/package/${encodeURIComponent(name)}/dist-tags`); - delete tags._etag; - return tags; -} - -async function fetchJson(url) { - return (await fetchPublic(url)).json(); -} - -async function fetchPublic(url) { - const response = await fetch(url, { - redirect: "error", - signal: AbortSignal.timeout(30_000), - headers: { "Cache-Control": "no-cache" }, - }); - assert(response.ok, `public metadata request failed: ${response.status} ${url}`); - return response; -} - -function digest(bytes, algorithm, encoding = "hex") { - return createHash(algorithm).update(bytes).digest(encoding); -} - -function git(root, args, trim = true) { - const result = spawnSync("git", args, { cwd: root, encoding: "utf8" }); - assert.equal(result.status, 0, result.stderr || "git verification failed"); - return trim ? result.stdout.trim() : result.stdout; -} - -export function npmDistTagArguments(operation, name, version, tag) { - assert(packageNames.includes(name)); - assert(operation === "add" || operation === "rm"); - assert(isOrdinaryPreOneVersion(version)); - assert(tag === "latest" || /^promotion-check-[1-9]\d*$/.test(tag)); - assert(operation !== "rm" || tag !== "latest", "only temporary probe tags may be removed"); - return ["dist-tag", operation, operation === "add" ? `${name}@${version}` : name, tag, - `--registry=${registry}`, "--prefer-online", "--fetch-retries=0", "--fetch-timeout=30000"]; -} - -function writeTag(operation, name, version, tag) { - const args = npmDistTagArguments(operation, name, version, tag); - assert(process.env.NODE_AUTH_TOKEN, "npm promotion token is missing"); - const result = spawnSync("npm", args, { - encoding: "utf8", timeout: 60_000, stdio: ["ignore", "pipe", "pipe"], - }); - return { - status: result.status, - ...(result.status === 0 ? {} : { - error: redactNpmError(result.stderr || result.error?.message || "npm returned no error text", process.env.NODE_AUTH_TOKEN), - }), - }; -} - -export function redactNpmError(message, token) { - assert(typeof token === "string" && token.length > 0, "cannot redact without the configured token"); - return stripVTControlCharacters(message).replaceAll(token, "[REDACTED]") - .replace(/npm_[A-Za-z0-9]+/g, "[REDACTED]").trim().slice(0, 4096); -} - -async function main() { - const mode = process.argv[2] ?? "inspect"; - assert(process.argv.length <= 3 && ["inspect", "promote", "verify-token", "cleanup-probe"].includes(mode)); - const report = { - observed_at: new Date().toISOString(), mode, status: "incomplete", operations: [], - request: { ref: process.env.GITHUB_REF, run_id: process.env.GITHUB_RUN_ID, attempt: process.env.GITHUB_RUN_ATTEMPT }, - }; - try { - const candidate = await inspectPromotion(); - report.candidate = candidate; - if (mode === "inspect") { - report.status = "verified"; - } else { - assertGithubContext(process.env, mode, candidate.plugin_version); - const options = { candidate, readTags, writeTag, report, attempt: process.env.GITHUB_RUN_ATTEMPT }; - if (mode === "promote") await promoteLatest(options); - else if (mode === "cleanup-probe") await cleanupProbe({ ...options, - probeRunId: process.env.NPM_PROMOTION_CLEANUP_RUN_ID, runId: process.env.GITHUB_RUN_ID }); - else await verifyToken({ ...options, runId: process.env.GITHUB_RUN_ID }); - } - } finally { - const json = `${JSON.stringify(report, null, 2)}\n`; - process.stdout.write(json); - if (process.env.RUNNER_TEMP) { - const directory = path.join(process.env.RUNNER_TEMP, "npm-promotion"); - await mkdir(directory, { recursive: true }); - await writeFile(path.join(directory, "receipt.json"), json); - } - if (process.env.GITHUB_STEP_SUMMARY) { - await appendFile(process.env.GITHUB_STEP_SUMMARY, `\n### npm promotion\n\n\`\`\`json\n${json}\`\`\`\n`); - } - } -} - -if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { - await main(); -} diff --git a/skills/create-full-stack-app/references/diagnostics-and-recovery.md b/skills/create-full-stack-app/references/diagnostics-and-recovery.md index 8a68887..67c3ef0 100644 --- a/skills/create-full-stack-app/references/diagnostics-and-recovery.md +++ b/skills/create-full-stack-app/references/diagnostics-and-recovery.md @@ -8,10 +8,9 @@ JSON object. An unrecognized prefixed line, a progress line after the envelope, interleaved output fail closed. Branch on the object's stable `error` and structured fields rather than the human-readable `detail` or broad process exit status. -The reviewed source-candidate CLI is revision -`660c02e46cdf36ec76dd556de8c96ef67ed3b035`, with JavaScript-source runtime digest -`ddd9b8ee4d83135a668b7a97e2522ba23b9478339662c1f6115e5273851abf81`. Its source package is -`@firstdraft.com/cli@0.4.0`. Check the command surface rather than assuming the version alone +The source candidate uses `@firstdraft.com/cli@0.4.0`. Its exact reviewed revision and runtime digest are owned by +[the CLI contract configuration](https://github.com/firstdraft/skills/blob/claude-v0.4.0/script/cli-contract/config.mjs) +at this plugin's protected release tag. Check the command surface rather than assuming the version alone establishes compatibility. These source checks do not prove plugin/catalog publication, service authentication, staging compatibility, or a complete user journey. diff --git a/skills/create-full-stack-app/references/foundation-plan-020.md b/skills/create-full-stack-app/references/foundation-plan-020.md index 19c54b5..d51e5d5 100644 --- a/skills/create-full-stack-app/references/foundation-plan-020.md +++ b/skills/create-full-stack-app/references/foundation-plan-020.md @@ -115,10 +115,10 @@ The bundled schema was copied byte-for-byte from `5576ec5e10d108f0a2d0f9fa336249324642f092e4444f6c11e4ab738f3fa58b`. This is exact contract provenance, not release or execution evidence. -The source candidate and pinned contract check use reviewed CLI revision -`660c02e46cdf36ec76dd556de8c96ef67ed3b035`, with JavaScript-source runtime digest -`ddd9b8ee4d83135a668b7a97e2522ba23b9478339662c1f6115e5273851abf81`, as contract provenance rather than release -or execution evidence. It exposes `generate uuid`, `generate application-key`, `plan init`, `plan push`, +The source candidate and pinned contract check use the exact reviewed CLI revision and runtime digest in +[the CLI contract configuration](https://github.com/firstdraft/skills/blob/claude-v0.4.0/script/cli-contract/config.mjs) +at this plugin's protected release tag, as contract provenance rather than release or execution evidence. The CLI +exposes `generate uuid`, `generate application-key`, `plan init`, `plan push`, `plan status`, local `plan compile` (equivalent to `--output .`), explicit `plan compile --github`, `plan compile --output`, `compilation status`, and `compilation download`. It has no public `plan subject-id` or `plan publish`. The coordinated checkout declares the diff --git a/test/catalog-ci.test.mjs b/test/catalog-ci.test.mjs new file mode 100644 index 0000000..02044f6 --- /dev/null +++ b/test/catalog-ci.test.mjs @@ -0,0 +1,77 @@ +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import test from "node:test"; + +import { catalogOnlyChange, isCatalogSelectionChange } from "../script/ci-scope.mjs"; +import { assertPublishedCatalogSelection } from "../script/check-catalog.mjs"; + +const source = { + source: "npm", package: "@firstdraft.com/claude-code", version: "0.2.5", registry: "https://registry.npmjs.org/", +}; +const catalog = { name: "firstdraft-skills", plugins: [{ name: "firstdraft", version: source.version, source }] }; +const promoted = structuredClone(catalog); +promoted.plugins[0].version = promoted.plugins[0].source.version = "0.4.0"; +const catalogPath = ".claude-plugin/marketplace.json"; + +test("only catalog version selection takes the reduced CI path", () => { + assert.equal(isCatalogSelectionChange([catalogPath], catalog, promoted), true); + for (const paths of [[], ["README.md"], [catalogPath, "script/check"], ["renamed.json", catalogPath]]) { + assert.equal(isCatalogSelectionChange(paths, catalog, promoted), false); + } + for (const mutate of [ + (value) => { value.name = "another-marketplace"; }, + (value) => { value.plugins.push({ name: "another-plugin" }); }, + (value) => { value.plugins[0].source.registry = "https://other.example/"; }, + (value) => { value.plugins[0].source.source = "github"; }, + (value) => { value.plugins[0].description = "Changed discovery metadata"; }, + ]) { + const changed = structuredClone(promoted); + mutate(changed); + assert.equal(isCatalogSelectionChange([catalogPath], catalog, changed), false); + } +}); + +test("git scope falls back to full CI without a base or after another file changes", (t) => { + const root = mkdtempSync(path.join(tmpdir(), "firstdraft-catalog-ci-")); + t.after(() => rmSync(root, { recursive: true, force: true })); + const git = (...args) => execFileSync("git", args, { cwd: root, encoding: "utf8" }).trim(); + git("init", "--quiet"); + git("config", "user.name", "Test"); + git("config", "user.email", "test@example.com"); + mkdirSync(path.join(root, ".claude-plugin")); + writeFileSync(path.join(root, catalogPath), JSON.stringify(catalog)); + git("add", "."); + git("commit", "--quiet", "-m", "Initial catalog"); + const base = git("rev-parse", "HEAD"); + writeFileSync(path.join(root, catalogPath), JSON.stringify(promoted)); + git("commit", "--quiet", "-am", "Select published version"); + assert.equal(catalogOnlyChange(base, root), true); + assert.equal(catalogOnlyChange("0".repeat(40), root), false); + assert.equal(catalogOnlyChange(undefined, root), false); + writeFileSync(path.join(root, "new-input"), "changed\n"); + git("add", "."); + git("commit", "--quiet", "-m", "Change an unknown input"); + assert.equal(catalogOnlyChange(base, root), false); +}); + +test("catalog publication check rejects missing or mismatched registry identity", () => { + const published = { + name: source.package, version: source.version, + dist: { + integrity: `sha512-${Buffer.alloc(64, 1).toString("base64")}`, + tarball: `${source.registry}${source.package}/-/claude-code-${source.version}.tgz`, + }, + }; + assert.doesNotThrow(() => assertPublishedCatalogSelection(source, published)); + for (const changed of [ + { ...published, name: "@other/plugin" }, + { ...published, version: "0.4.0" }, + { ...published, dist: undefined }, + { ...published, dist: { ...published.dist, tarball: "https://other.example/plugin.tgz" } }, + ]) { + assert.throws(() => assertPublishedCatalogSelection(source, changed)); + } +}); diff --git a/test/npm-promotion-cli-cache.test.mjs b/test/npm-promotion-cli-cache.test.mjs deleted file mode 100644 index cef013d..0000000 --- a/test/npm-promotion-cli-cache.test.mjs +++ /dev/null @@ -1,65 +0,0 @@ -import assert from "node:assert/strict"; -import { execFile } from "node:child_process"; -import { mkdtemp, rm, writeFile } from "node:fs/promises"; -import { createServer } from "node:http"; -import { tmpdir } from "node:os"; -import path from "node:path"; -import test from "node:test"; -import { promisify } from "node:util"; - -import { npmDistTagArguments } from "../script/npm-promotion.mjs"; - -const run = promisify(execFile); - -test("npm probe cleanup refreshes metadata cached before the addition", async (t) => { - const directory = await mkdtemp(path.join(tmpdir(), "npm-promotion-cache-")); - t.after(() => rm(directory, { recursive: true, force: true })); - const tags = { latest: "0.2.2", next: "0.2.2" }; - const tag = "promotion-check-12345"; - const collection = "/-/package/@firstdraft.com%2fcli/dist-tags"; - const requests = []; - const server = createServer(async (request, response) => { - requests.push([request.method, request.url]); - response.setHeader("Content-Type", "application/json"); - if (request.method === "GET" && request.url === collection) { - response.setHeader("Cache-Control", "public, max-age=300"); - response.end(JSON.stringify(tags)); - } else if (request.method === "PUT" && request.url === `${collection}/${tag}`) { - let body = ""; - for await (const chunk of request) body += chunk; - tags[tag] = JSON.parse(body); - response.end("{}"); - } else if (request.method === "DELETE" && request.url === `${collection}/${tag}`) { - delete tags[tag]; - response.end("{}"); - } else { - response.writeHead(404); - response.end("{}"); - } - }); - await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); - t.after(() => new Promise((resolve) => server.close(resolve))); - const registry = `http://127.0.0.1:${server.address().port}/`; - const userconfig = path.join(directory, "user.npmrc"); - const globalconfig = path.join(directory, "global.npmrc"); - await Promise.all([writeFile(userconfig, ""), writeFile(globalconfig, "")]); - const env = Object.fromEntries(Object.entries(process.env) - .filter(([key]) => !/^(npm_config_|npm_token$|node_auth_token$)/i.test(key))); - Object.assign(env, { - NPM_CONFIG_USERCONFIG: userconfig, - NPM_CONFIG_GLOBALCONFIG: globalconfig, - NPM_CONFIG_CACHE: path.join(directory, "cache"), - NPM_CONFIG_UPDATE_NOTIFIER: "false", - }); - for (const operation of ["add", "rm"]) { - const args = npmDistTagArguments(operation, "@firstdraft.com/cli", "0.2.2", tag) - .map((argument) => argument === "--registry=https://registry.npmjs.org/" - ? `--registry=${registry}` : argument); - await run("npm", args, { cwd: directory, env }); - } - assert.deepEqual(requests, [ - ["GET", collection], ["PUT", `${collection}/${tag}`], - ["GET", collection], ["DELETE", `${collection}/${tag}`], - ]); - assert.deepEqual(tags, { latest: "0.2.2", next: "0.2.2" }); -}); diff --git a/test/npm-promotion.test.mjs b/test/npm-promotion.test.mjs deleted file mode 100644 index 10629c5..0000000 --- a/test/npm-promotion.test.mjs +++ /dev/null @@ -1,582 +0,0 @@ -import assert from "node:assert/strict"; -import { createHash } from "node:crypto"; -import test from "node:test"; - -import { - assertCatalog, assertDistributionState, assertGithubContext, - assertPackageBytes, cleanupProbe, npmDistTagArguments, promoteLatest, redactNpmError, verifyToken, -} from "../script/npm-promotion.mjs"; -import { file, tarball } from "./helpers/tarball.mjs"; - -const names = ["@firstdraft.com/cli", "@firstdraft.com/claude-code"]; -const version = "0.2.2"; - -function harness(latest = "0.2.1") { - const packages = names.map((name) => ({ name, version, tags: { next: version, latest, legacy: "0.1.0" } })); - const states = new Map(packages.map(({ name, tags }) => [name, structuredClone(tags)])); - const writes = []; - const waits = []; - return { - candidate: { packages }, report: { status: "incomplete", operations: [] }, - attempt: "1", runId: "12345", states, writes, waits, - wait: async (milliseconds) => { waits.push(milliseconds); }, - readTags: async (name) => structuredClone(states.get(name)), - writeTag: async (operation, name, target, tag) => { - writes.push({ operation, name, target, tag }); - if (operation === "add") states.get(name)[tag] = target; - else delete states.get(name)[tag]; - return { status: 0 }; - }, - }; -} - -function lagSuccessfulWrites(h, staleReads = 2) { - const pending = new Map(); - const read = h.readTags; - const write = h.writeTag; - h.readTags = async (name) => pending.get(name)?.shift() ?? read(name); - h.writeTag = async (...args) => { - const before = await read(args[1]); - const result = await write(...args); - pending.set(args[1], Array.from({ length: staleReads }, () => structuredClone(before))); - return result; - }; -} - -test("promotion accepts only the qualified, monotonic package pair", () => { - assertDistributionState(harness().candidate.packages); - for (const mutate of [ - (p) => p.reverse(), - (p) => p.pop(), - (p) => { p[0].name = "@someone/cli"; }, - (p) => { p[0].tags.next = "0.2.3"; }, - (p) => { p[1].tags.latest = "0.3.0"; }, - (p) => { delete p[1].tags.latest; }, - (p) => { p[0].version = "0.2.2-alpha.1"; }, - ]) { - const p = harness().candidate.packages; - mutate(p); - assert.throws(() => assertDistributionState(p)); - } -}); - -test("promotion requires the exact public npm catalog selection", () => { - const catalog = { plugins: [{ name: "firstdraft", version, source: { - source: "npm", package: names[1], version, registry: "https://registry.npmjs.org/", - } }] }; - assertCatalog(catalog, version); - for (const mutate of [ - (c) => c.plugins.push(c.plugins[0]), - (c) => { c.plugins[0].version = "0.2.3"; }, - (c) => { c.plugins[0].source.version = "latest"; }, - (c) => { c.plugins[0].source.registry = "https://example.com/"; }, - ]) { - const c = structuredClone(catalog); - mutate(c); - assert.throws(() => assertCatalog(c, version)); - } -}); - -test("GitHub writes require the protected version tag or a main credential check", () => { - const env = { - GITHUB_ACTIONS: "true", GITHUB_REPOSITORY: "firstdraft/skills", - GITHUB_RUN_ID: "123", GITHUB_RUN_ATTEMPT: "1", GITHUB_EVENT_NAME: "push", - GITHUB_REF_TYPE: "tag", GITHUB_REF_PROTECTED: "true", - GITHUB_REF: `refs/tags/promote-v${version}`, GITHUB_REF_NAME: `promote-v${version}`, - }; - assertGithubContext(env, "promote", version); - assertGithubContext({ ...env, GITHUB_REF: `refs/tags/promote-v${version}-retry-1`, - GITHUB_REF_NAME: `promote-v${version}-retry-1` }, "promote", version); - for (const override of [ - { GITHUB_ACTIONS: "false" }, { GITHUB_REPOSITORY: "fork/skills" }, - { GITHUB_REF_PROTECTED: "false" }, { GITHUB_REF_TYPE: "branch" }, - { GITHUB_REF: "refs/tags/promote-v0.2.1" }, { GITHUB_EVENT_NAME: "pull_request" }, - { GITHUB_RUN_ATTEMPT: "0" }, { GITHUB_RUN_ID: "../123" }, - { GITHUB_REF_NAME: `promote-v${version}-retry-0` }, - { GITHUB_REF_NAME: `promote-v${version}-retry-1-extra` }, - ]) assert.throws(() => assertGithubContext({ ...env, ...override }, "promote", version)); - const dispatch = { ...env, GITHUB_EVENT_NAME: "workflow_dispatch", GITHUB_REF: "refs/heads/main" }; - assertGithubContext(dispatch, "verify-token", version); - assertGithubContext(dispatch, "cleanup-probe", version); - assert.throws(() => assertGithubContext(dispatch, "promote", version)); - assert.throws(() => assertGithubContext(env, "verify-token", version)); - assert.throws(() => assertGithubContext(env, "cleanup-probe", version)); - assert.throws(() => assertGithubContext({ ...dispatch, GITHUB_REF: "refs/heads/feature" }, "verify-token", version)); -}); - -test("npm argument construction restricts registry, packages, versions, operations and tags", () => { - assert.deepEqual(npmDistTagArguments("add", names[0], version, "latest"), [ - "dist-tag", "add", `${names[0]}@${version}`, "latest", - "--registry=https://registry.npmjs.org/", "--prefer-online", "--fetch-retries=0", "--fetch-timeout=30000", - ]); - assert.deepEqual(npmDistTagArguments("rm", names[1], version, "promotion-check-123"), [ - "dist-tag", "rm", names[1], "promotion-check-123", - "--registry=https://registry.npmjs.org/", "--prefer-online", "--fetch-retries=0", "--fetch-timeout=30000", - ]); - for (const args of [ - ["publish", names[0], version, "latest"], - ["add", "@someone/cli", version, "latest"], - ["add", names[0], "latest", "latest"], - ["add", names[0], version, "next"], - ["rm", names[0], version, "latest"], - ["rm", names[0], version, "promotion-check-../123"], - ]) assert.throws(() => npmDistTagArguments(...args)); -}); - -function archives(mutate = () => {}) { - const cli = [file("package/package.json", JSON.stringify({ name: names[0], version })), - file("package/bin/firstdraft.js", "#!/usr/bin/env node\n", 0o755)]; - const bundled = cli.map((entry) => ({ ...entry, name: entry.name.replace("package/", "package/vendor/cli/") })); - mutate(bundled); - const bytes = [tarball(cli), tarball([ - file("package/package.json", JSON.stringify({ name: names[1], version })), ...bundled, - ])]; - return bytes.map((bytes, index) => ({ name: names[index], version, bytes, - sha256: createHash("sha256").update(bytes).digest("hex"), - integrity: `sha512-${createHash("sha512").update(bytes).digest("base64")}`, - })); -} - -test("qualified plugin digest anchors both package manifests and complete CLI contents", () => { - const packages = archives(); - assertPackageBytes(packages, packages[1].sha256); - assert.throws(() => assertPackageBytes(packages, "wrong"), /qualified artifact/); - for (const field of ["sha256", "integrity", "version"]) { - const bad = structuredClone(packages); - bad[0].bytes = packages[0].bytes; - bad[1].bytes = packages[1].bytes; - bad[0][field] = "wrong"; - assert.throws(() => assertPackageBytes(bad, packages[1].sha256)); - } - for (const mutate of [ - (entries) => entries.pop(), - (entries) => entries.push(file("package/vendor/cli/extra", "extra")), - (entries) => { entries[1].mode = 0o644; }, - (entries) => { entries[1].bytes = Buffer.from("changed"); }, - ]) { - const bad = archives(mutate); - assert.throws(() => assertPackageBytes(bad, bad[1].sha256), /bundled CLI/); - } -}); - -test("promotion moves CLI then plugin, preserving all other tags", async () => { - const h = harness(); - await promoteLatest(h); - assert.deepEqual(h.writes, names.map((name) => ({ operation: "add", name, target: version, tag: "latest" }))); - for (const tags of h.states.values()) assert.deepEqual(tags, { next: version, latest: version, legacy: "0.1.0" }); - assert.deepEqual(h.report.final_verification, { - status: "verified", packages: names.map((name) => ({ name, version, tags: h.states.get(name) })), - }); - assert.equal(h.report.status, "promoted"); -}); - -test("verified writes retain a stale closing observation without waiting or rewriting", async () => { - const h = harness(); - const read = h.readTags; - const closingReads = []; - h.readTags = async (name) => { - if (h.report.operations.length === 2 - && h.report.operations.every(({ readback_status }) => readback_status === "verified")) { - closingReads.push(name); - if (name === names[0]) return h.candidate.packages[0].tags; - } - return read(name); - }; - await assert.rejects(promoteLatest(h), /promotion is incomplete/); - assert.deepEqual(closingReads, names); - assert.equal(h.writes.length, 2); - assert.deepEqual(h.waits, []); - assert(h.report.operations.every(({ readback_status }) => readback_status === "verified")); - assert.deepEqual(h.report.final_verification, { - status: "incomplete", packages: [ - { name: names[0], version, tags: h.candidate.packages[0].tags }, - { name: names[1], version, tags: h.states.get(names[1]) }, - ], - }); - assert.equal(h.report.status, "incomplete"); -}); - -test("a failed closing read retains the preceding package observation", async () => { - const h = harness(); - const read = h.readTags; - const closingReads = []; - h.readTags = async (name) => { - if (h.report.operations.length === 2 - && h.report.operations.every(({ readback_status }) => readback_status === "verified")) { - closingReads.push(name); - if (name === names[1]) throw new Error("offline during final verification"); - } - return read(name); - }; - await assert.rejects(promoteLatest(h), /offline during final verification/); - assert.deepEqual(closingReads, names); - assert.equal(h.writes.length, 2); - assert.deepEqual(h.waits, []); - assert(h.report.operations.every(({ readback_status }) => readback_status === "verified")); - assert.deepEqual(h.report.final_verification, { - status: "incomplete", packages: [{ name: names[0], version, tags: h.states.get(names[0]) }], - }); - assert.equal(h.report.status, "incomplete"); -}); - -test("promotion waits for exact prior tag maps to converge without repeating writes", async () => { - const h = harness(); - lagSuccessfulWrites(h); - await promoteLatest(h); - assert.equal(h.writes.length, 2); - assert.deepEqual(h.waits, [2000, 2000, 2000, 2000]); - for (const operation of h.report.operations) { - assert.deepEqual(operation.readbacks.slice(0, 3), [ - operation.before, operation.before, { ...operation.before, latest: version }, - ]); - assert.equal(operation.readback_status, "verified"); - } - assert.equal(h.report.status, "promoted"); -}); - -test("promotion stops after six stale readbacks without another mutation", async () => { - const h = harness(); - lagSuccessfulWrites(h, 20); - await assert.rejects(promoteLatest(h), /readback did not converge/); - assert.equal(h.writes.length, 1); - assert.deepEqual(h.waits, [2000, 2000, 2000, 2000, 2000]); - const operation = h.report.operations[0]; - assert.equal(operation.readbacks.length, 6); - assert.deepEqual(operation.after, operation.before); - assert.equal(operation.readback_status, "prior-state-timeout"); - assert.equal(h.report.status, "incomplete"); -}); - -test("a conflicting read after a stale read stops immediately", async () => { - for (const change of [{ latest: "0.2.3" }, { next: "0.2.3" }, { legacy: "0.1.1" }, { extra: version }]) { - const h = harness(); - const before = structuredClone(h.states.get(names[0])); - const read = h.readTags; - let afterWriteReads = 0; - h.readTags = async (name) => { - if (!h.writes.length) return read(name); - afterWriteReads += 1; - return afterWriteReads === 1 ? before : { ...before, latest: version, ...change }; - }; - await assert.rejects(promoteLatest(h), /npm tag state changed unexpectedly/); - assert.equal(afterWriteReads, 2); - assert.deepEqual(h.waits, [2000]); - assert.equal(h.writes.length, 1); - assert.equal(h.report.operations[0].readback_status, "conflicting-state"); - } -}); - -test("a read failure after a stale sample retains that sample and stops", async () => { - const h = harness(); - const read = h.readTags; - let afterWriteReads = 0; - h.readTags = async (name) => { - if (!h.writes.length) return read(name); - if (++afterWriteReads === 1) return h.candidate.packages[0].tags; - throw new Error("offline"); - }; - await assert.rejects(promoteLatest(h), /offline/); - assert.equal(h.writes.length, 1); - assert.deepEqual(h.waits, [2000]); - assert.deepEqual(h.report.operations[0].readbacks, [h.candidate.packages[0].tags]); - assert.equal(h.report.operations[0].readback_status, "read-failed"); -}); - -test("already selected versions and completed reruns make no writes", async () => { - const h = harness(version); - h.attempt = "2"; - await promoteLatest(h); - assert.equal(h.writes.length, 0); - assert(h.report.operations.every(({ status }) => status === "already-selected")); -}); - -test("an incomplete rerun refuses writes, including a partial prior promotion", async () => { - for (const firstSelected of [false, true]) { - const h = harness(); - h.attempt = "2"; - if (firstSelected) h.states.get(names[0]).latest = version; - await assert.rejects(promoteLatest(h), /reruns reconcile only/); - assert.equal(h.writes.length, 0); - } -}); - -test("a new approved request can finish a reconciled partial promotion", async () => { - const h = harness(); - h.states.get(names[0]).latest = version; - await promoteLatest(h); - assert.deepEqual(h.writes, [{ operation: "add", name: names[1], target: version, tag: "latest" }]); - assert.equal(h.report.status, "promoted"); -}); - -test("drift in either package stops before the first mutation", async () => { - const h = harness(); - h.states.get(names[1]).next = "0.2.3"; - await assert.rejects(promoteLatest(h), /next must select/); - assert.equal(h.writes.length, 0); -}); - -test("npm errors and unexpected state stop without retrying or mutating the next package", async () => { - for (const outcome of ["error-before-write", "error-after-write", "tag-drift", "read-failure"]) { - const h = harness(); - const write = h.writeTag; - h.writeTag = async (...args) => { - if (outcome === "error-before-write") { h.writes.push(args); return { status: 1 }; } - await write(...args); - if (outcome === "tag-drift") h.states.get(names[0]).legacy = "0.1.1"; - if (outcome === "read-failure") h.readTags = async () => { throw new Error("offline"); }; - return { status: outcome === "error-after-write" ? 1 : 0 }; - }; - // Keep the passed reader responsive to an injected outage after the write. - const originalRead = h.readTags; - await assert.rejects(promoteLatest({ ...h, readTags: (name) => - outcome === "read-failure" && h.writes.length ? h.readTags(name) : originalRead(name) })); - assert.equal(h.writes.length, 1); - assert.equal(h.states.get(names[1]).latest, "0.2.1"); - assert.deepEqual(h.report.operations[0].requested, { latest: version }); - assert.deepEqual(h.waits, []); - assert.equal(h.report.status, "incomplete"); - } -}); - -test("credential check exercises both packages and removes only its own temporary tags", async () => { - const h = harness(version); - await verifyToken(h); - assert.equal(h.writes.length, 4); - assert.deepEqual(h.writes.map(({ operation }) => operation), ["add", "rm", "add", "rm"]); - assert(h.writes.every(({ tag }) => tag === "promotion-check-12345")); - for (const item of h.candidate.packages) assert.deepEqual(h.states.get(item.name), item.tags); - assert.equal(h.report.status, "credentials-verified"); -}); - -test("credential checks wait for successful probe add and removal readbacks", async () => { - const h = harness(version); - lagSuccessfulWrites(h); - await verifyToken(h); - assert.deepEqual(h.writes.map(({ operation }) => operation), ["add", "rm", "add", "rm"]); - assert.equal(h.waits.length, 8); - for (const operation of h.report.operations) { - assert.equal(operation.readbacks.length, 3); - assert.equal(operation.readback_status, "verified"); - } - assert.equal(h.report.status, "credentials-verified"); -}); - -test("a probe add visibility timeout never removes an unobserved probe", async () => { - const h = harness(version); - lagSuccessfulWrites(h, 20); - await assert.rejects(verifyToken(h), /readback did not converge/); - assert.equal(h.writes.length, 1); - assert.equal(h.states.get(names[0])["promotion-check-12345"], version); - assert.equal(h.report.operations[0].readbacks.length, 6); - assert.equal(h.report.operations[0].readback_status, "prior-state-timeout"); -}); - -test("a probe removal visibility timeout never repeats deletion", async () => { - const h = harness(version); - const read = h.readTags; - h.readTags = async (name) => h.writes.length < 2 ? read(name) - : { ...h.candidate.packages[0].tags, "promotion-check-12345": version }; - await assert.rejects(verifyToken(h), /readback did not converge/); - assert.deepEqual(h.writes.map(({ operation }) => operation), ["add", "rm"]); - assert.equal(h.states.get(names[0])["promotion-check-12345"], undefined); - assert.equal(h.report.operations[1].readbacks.length, 6); - assert.equal(h.report.operations[1].readback_status, "prior-state-timeout"); -}); - -test("credential checks refuse reruns, existing probe tags, and unpromoted releases", async () => { - for (const mutate of [ - (h) => { h.attempt = "2"; }, - (h) => { h.states.get(names[0])["promotion-check-12345"] = version; }, - (h) => { h.states.get(names[0]).latest = "0.2.1"; }, - ]) { - const h = harness(version); - mutate(h); - await assert.rejects(verifyToken(h)); - assert.equal(h.writes.length, 0); - } -}); - -test("retained probes on either package block a new check before any write", async () => { - for (const name of names) { - for (const tag of ["promotion-check-12345", "promotion-check-12344"]) { - const h = harness(version); - h.states.get(name)[tag] = version; - await assert.rejects(verifyToken(h), /a retained probe must be reconciled/); - assert.deepEqual(h.writes, []); - assert.deepEqual(h.report.operations, []); - } - } -}); - -test("an observed probe is cleaned after an add error, then the check stops", async () => { - const h = harness(version); - const write = h.writeTag; - h.writeTag = async (...args) => { await write(...args); return { status: args[0] === "add" ? 1 : 0 }; }; - await assert.rejects(verifyToken(h), /probe failure/); - assert.equal(h.writes.length, 2); - assert.equal(h.states.get(names[0])["promotion-check-12345"], undefined); - assert.deepEqual(h.waits, []); - assert.equal(h.report.status, "incomplete"); -}); - -test("a failed probe add stops after one prior-state read without waiting or removing", async () => { - const h = harness(version); - const read = h.readTags; - let afterWriteReads = 0; - h.writeTag = async (...args) => { h.writes.push(args); return { status: 1 }; }; - h.readTags = async (name) => { - if (h.writes.length && ++afterWriteReads > 1) { - return { ...h.candidate.packages[0].tags, "promotion-check-12345": version }; - } - return read(name); - }; - await assert.rejects(verifyToken(h), /probe write needs read-only reconciliation/); - assert.equal(h.writes.length, 1); - assert.equal(afterWriteReads, 1); - assert.deepEqual(h.waits, []); - assert.equal(h.report.operations[0].readback_status, "prior-state-after-command-failure"); -}); - -test("conflicting probe readbacks stop without further writes", async () => { - for (const writeCount of [1, 2]) { - const h = harness(version); - const read = h.readTags; - h.readTags = async (name) => ({ ...await read(name), - ...(h.writes.length === writeCount ? { legacy: "0.1.1" } : {}), - }); - await assert.rejects(verifyToken(h), /needs read-only reconciliation/); - assert.equal(h.writes.length, writeCount); - assert.deepEqual(h.waits, []); - assert.equal(h.report.operations.at(-1).readback_status, "conflicting-state"); - } -}); - -test("ambiguous probe cleanup retains the receipt and never repeats deletion", async () => { - const h = harness(version); - const write = h.writeTag; - h.writeTag = async (...args) => { - if (args[0] === "rm") { h.writes.push(args); return { status: 1 }; } - return write(...args); - }; - await assert.rejects(verifyToken(h), /cleanup needs read-only reconciliation/); - assert.equal(h.writes.length, 2); - assert.equal(h.states.get(names[0])["promotion-check-12345"], version); - assert.equal(h.report.operations[1].command_status, 1); - assert.deepEqual(h.waits, []); -}); - -test("a failed probe removal observed absent stops without repeating deletion", async () => { - const h = harness(version); - const write = h.writeTag; - h.writeTag = async (...args) => { await write(...args); return { status: args[0] === "rm" ? 1 : 0 }; }; - await assert.rejects(verifyToken(h), /cleanup failure; the probe is observed absent/); - assert.equal(h.writes.length, 2); - assert.equal(h.states.get(names[0])["promotion-check-12345"], undefined); - assert.equal(h.report.operations[1].command_status, 1); - assert.equal(h.report.operations[1].readback_status, "verified"); - assert.deepEqual(h.waits, []); - assert.equal(h.report.status, "incomplete"); -}); - -test("cleanup removes only the reconciled prior probe and preserves all other tags", async () => { - const h = harness(version); - h.probeRunId = "12344"; - for (const tags of h.states.values()) Object.assign(tags, { - "promotion-check-12344": version, "promotion-check-12343": version, - }); - await cleanupProbe(h); - assert.deepEqual(h.writes, names.map((name) => ({ operation: "rm", name, target: version, tag: "promotion-check-12344" }))); - assert.equal(h.report.status, "probe-cleaned"); - assert.equal(h.report.final_verification.status, "verified"); - for (const tags of h.states.values()) assert.deepEqual(tags, { - next: version, latest: version, legacy: "0.1.0", "promotion-check-12343": version, - }); -}); - -test("cleanup already absent on both packages makes no write", async () => { - const h = harness(version); - h.probeRunId = "12344"; - await cleanupProbe(h); - assert.deepEqual(h.writes, []); - assert.equal(h.report.status, "probe-cleaned"); - assert(h.report.operations.every(({ status }) => status === "already-absent")); -}); - -test("cleanup rejects invalid context or a changed version before any write", async () => { - for (const mutate of [ - (h) => { h.attempt = "2"; }, - (h) => { h.probeRunId = h.runId; }, - (h) => { h.probeRunId = "../12344"; }, - (h) => { h.probeRunId = ["12344"]; }, - (h) => { h.states.get(names[1]).latest = "0.2.1"; }, - (h) => { h.states.get(names[1]).next = "0.2.1"; }, - (h) => { h.states.get(names[1])["promotion-check-12344"] = "0.2.1"; }, - ]) { - const h = harness(version); - h.probeRunId = "12344"; - h.states.get(names[0])["promotion-check-12344"] = version; - mutate(h); - await assert.rejects(cleanupProbe(h)); - assert.deepEqual(h.writes, []); - } -}); - -test("failed cleanup preserves diagnostic and observation without repeating a write", async () => { - const h = harness(version); - h.probeRunId = "12344"; - h.states.get(names[0])["promotion-check-12344"] = version; - h.writeTag = async (...args) => { - h.writes.push(args); - return { status: 1, error: "npm error code E403" }; - }; - await assert.rejects(cleanupProbe(h), /read-only reconciliation/); - assert.equal(h.writes.length, 1); - assert.deepEqual(h.waits, []); - assert.equal(h.report.operations[0].command_error, "npm error code E403"); - assert.equal(h.report.operations[0].after["promotion-check-12344"], version); - assert.equal(h.report.status, "incomplete"); -}); - -test("cleanup waits only on unchanged successful-write readbacks", async () => { - const h = harness(version); - h.probeRunId = "12344"; - h.states.get(names[0])["promotion-check-12344"] = version; - lagSuccessfulWrites(h); - await cleanupProbe(h); - assert.equal(h.writes.length, 1); - assert.deepEqual(h.waits, [2000, 2000]); - assert.equal(h.report.status, "probe-cleaned"); -}); - -test("cleanup closing check retains a changed final map without another mutation", async () => { - const h = harness(version); - h.probeRunId = "12344"; - const read = h.readTags; - h.readTags = async (name) => { - if (h.report.final_verification) return { ...(await read(name)), unexpected: version }; - return read(name); - }; - await assert.rejects(cleanupProbe(h), /changed after cleanup/); - assert.deepEqual(h.writes, []); - assert.equal(h.report.final_verification.status, "incomplete"); - assert.equal(h.report.final_verification.packages[0].tags.unexpected, version); -}); - -test("npm error text redacts credentials before truncation and preserves useful errors", () => { - const token = "configured-secret"; - const text = `\u001b[31mnpm error code E403\u001b[0m\nBearer ${token}\nnpm_otherToken123`; - assert.equal(redactNpmError(text, token), "npm error code E403\nBearer [REDACTED]\n[REDACTED]"); - const long = redactNpmError(`${"x".repeat(4090)}${token}`, token); - assert.equal(long.length, 4096); - assert(!long.includes("configured")); - assert.throws(() => redactNpmError("error", ""), /cannot redact/); -}); - -test("promotion and credential failures retain the npm diagnostic", async () => { - for (const [method, latest] of [[promoteLatest, "0.2.1"], [verifyToken, version]]) { - const h = harness(latest); - h.writeTag = async () => ({ status: 1, error: "npm error code EOTP" }); - await assert.rejects(method(h)); - assert.equal(h.report.operations[0].command_error, "npm error code EOTP"); - } -}); diff --git a/test/release-compatibility.test.mjs b/test/release-compatibility.test.mjs index c784d54..6fa8c4c 100644 --- a/test/release-compatibility.test.mjs +++ b/test/release-compatibility.test.mjs @@ -25,14 +25,25 @@ test("release compatibility matches the installable plugin manifest", async () = assert(declaredPluginVersion, "the Skill must identify its plugin compatibility version"); assert.equal(declaredPluginVersion[1], compatibility.version); + const cliConfigurationUrl = + `https://github.com/firstdraft/skills/blob/claude-v${compatibility.version}/script/cli-contract/config.mjs`; + for (const name of ["diagnostics-and-recovery.md", "foundation-plan-020.md"]) { + const reference = await readText(`skills/create-full-stack-app/references/${name}`); + assert.equal( + reference.match(/\[the CLI contract configuration\]\(([^)]+)\)/)?.[1], + cliConfigurationUrl, + `${name}: bundled CLI provenance must use the plugin's release tag`, + ); + assert(reference.includes(`@firstdraft.com/cli@${cliPackageVersion}`)); + } + assert.deepEqual(compatibility, { format: "firstdraft.release-compatibility/1", component: "skills", version: "0.4.0", plugin_source: { package: "@firstdraft.com/claude-code", - tarball_sha256: - "a09786608db0452fcaff6c8f42cfd28148ced16834f94a03493fbff0edd1e223", + tarball_sha256: compatibility.plugin_source.tarball_sha256, }, requires: { api_contract: [">= 0.4.0", "< 0.5.0"], @@ -122,7 +133,7 @@ test("current release docs route through structured identities", async () => { candidateSmokeEvidence, /controlled Service revision is a descendant of the pinned current-truth Service revision[\s\S]*?cc72dad5b26b887f3f21496b568b80678ceac47f[\s\S]*?does not repin the packaged[\s\S]*?current-authority source/, ); - assert.equal(publicPlugin.version, "0.2.5", "do not promote an unpublished candidate"); + assert.equal(publicPlugin.version, publicPlugin.source.version); assert.match(releasing, /release\/compatibility\.json.*owns the candidate/s); assert.match(releasing, /One user approval may cover the complete coordinated release/); assert.match(releasing, /without asking again at\s+every step/); diff --git a/test/repository.test.mjs b/test/repository.test.mjs index 327bab6..2ae3640 100644 --- a/test/repository.test.mjs +++ b/test/repository.test.mjs @@ -26,6 +26,8 @@ import { import { analyzerRelease as foundationPlanAnalyzerRelease, compilationTarget as foundationPlanTarget, + cliRevision as cliContractBaseline, + cliRuntimeSha256 as cliContractRuntimeDigest, compilerRelease as foundationPlanCompilerRelease, foundationPlanFormat, rootOutputRecovery, @@ -115,10 +117,6 @@ const compilationEvidenceCliBaseline = "121272cd592055354d09a4fe90e55c3ca002770c"; const compilationEvidenceCliRuntimeDigest = "205e664df0ed9c7e63651a1c2c01e749a04d8879fe7f62cc4c1e13b66dce738d"; -const cliContractBaseline = - "660c02e46cdf36ec76dd556de8c96ef67ed3b035"; -const cliContractRuntimeDigest = - "ddd9b8ee4d83135a668b7a97e2522ba23b9478339662c1f6115e5273851abf81"; const previousPublicCliContractBaseline = "d38ef3e54a6476b3a91f22a17fe7bd47aa6d6d68"; const previousPublicCliContractRuntimeDigest = @@ -392,7 +390,6 @@ test("revision pins remain exhaustive across coordination surfaces", async () => priorAndroidEvidenceBaseline, currentFoundationIosCoreRevision, currentFoundationAndroidCoreRevision, - cliContractBaseline, catalogPromotionBaseline, ]); const skillSource = await readFile( @@ -430,7 +427,7 @@ test("revision pins remain exhaustive across coordination surfaces", async () => const workflow = ( await readFile(path.join(repository, ".github", "workflows", "ci.yml"), "utf8") ).replace(/^.*uses:\s+\S+@[0-9a-f]{40}.*$/gm, ""); - assertRevisionTokens(workflow, [cliContractBaseline]); + assertRevisionTokens(workflow, []); const contractConfig = await readFile( path.join(repository, "script", "cli-contract", "config.mjs"), "utf8", @@ -456,7 +453,6 @@ test("revision pins remain exhaustive across coordination surfaces", async () => currentCompilerServiceBaseline, discoverySmokeServiceBaseline, compilationEvidenceCliBaseline, - cliContractBaseline, previousPublicCliContractBaseline, previousCliContractBaseline, historicalCliContractBaseline, @@ -926,11 +922,10 @@ test("Claude Code packaging selects canonical authoring source exactly once", as assert.equal(marketplace.name, claudeMarketplaceName); assert.equal(marketplace.plugins.length, 1); assert.equal(marketplace.plugins[0].name, claudePluginName); - assert.equal(marketplace.plugins[0].version, "0.2.5"); assert.deepEqual(marketplace.plugins[0].source, { source: "npm", package: "@firstdraft.com/claude-code", - version: "0.2.5", + version: marketplace.plugins[0].version, registry: "https://registry.npmjs.org/", }); assert.equal(packageTemplate.version, "0.4.0"); @@ -1087,16 +1082,9 @@ test("CI checks the exact modular CLI contract", async () => { `repository: firstdraft/cli\\s+ref: main\\s+fetch-depth: 0`, ), ); - assert.equal( - [...publishWorkflow.matchAll(/[0-9a-f]{40}/g)].filter( - ([revision]) => revision === cliContractBaseline, - ).length, - 2, - ); assert.doesNotMatch( publishWorkflow - .replace(/^.*uses:\s+\S+@[0-9a-f]{40}.*$/gm, "") - .replaceAll(cliContractBaseline, ""), + .replace(/^.*uses:\s+\S+@[0-9a-f]{40}.*$/gm, ""), /\b[0-9a-f]{40}\b/, ); assert.doesNotMatch( @@ -1130,7 +1118,7 @@ test("CI checks the exact modular CLI contract", async () => { ); assert.match( workflow, - /name: Rehearse release ordering\s+if: matrix\.node == '24\.18\.0'[\s\S]*?\+refs\/tags\/claude-v\*:refs\/release-check\/tags\/claude-v\*[\s\S]*?node script\/check-plugin-release-order\.mjs --prospective/, + /name: Rehearse release ordering\s+if: steps\.scope\.outputs\.catalog_only != 'true' && matrix\.node == '24\.18\.0'[\s\S]*?\+refs\/tags\/claude-v\*:refs\/release-check\/tags\/claude-v\*[\s\S]*?node script\/check-plugin-release-order\.mjs --prospective/, ); assert.deepEqual( workflow.match( @@ -1264,24 +1252,21 @@ test("CI checks the exact modular CLI contract", async () => { ]) { assert.doesNotMatch(repositoryCheck, new RegExp(networkedCheck)); } - assert.match( - workflow, - new RegExp( - `merge-base --is-ancestor ${cliContractBaseline} HEAD`, - ), - ); - assert.match( - workflow, - new RegExp(`checkout --detach ${cliContractBaseline}`), - ); + for (const source of [workflow, publishWorkflow]) { + assert.match(source, /import \{ cliRevision \} from "\.\/script\/cli-contract\/config\.mjs"/); + assert.match(source, /merge-base --is-ancestor "\$cli_revision" HEAD/); + assert.match(source, /checkout --detach "\$cli_revision"/); + } assert.match( workflow, /node script\/check-cli-contract\.mjs tmp\/firstdraft-cli/, ); assert.match( workflow, - /node script\/check-claude-plugin-package\.mjs --cli-root tmp\/firstdraft-cli/, + /sh script\/check --cli-root tmp\/firstdraft-cli/, ); + assert.doesNotMatch(workflow, /node script\/check-claude-plugin-package/); + assert.match(repositoryCheck, /node script\/check-claude-plugin-package\.mjs "\$@"/); assert(contractConfig.includes(cliContractBaseline)); assert(contractConfig.includes(cliContractRuntimeDigest)); assert.match(contractConfig, /src\/commands\/compilation\.js/); @@ -2329,8 +2314,7 @@ test("complete examples and eval Plans validate against the bundled exact schema ); assert(referenceSource.includes(foundationPlanSchemaDigest)); assert(referenceSource.includes(currentFoundationPlanSchemaBaseline)); - assert(referenceSource.includes(cliContractBaseline)); - assert(referenceSource.includes(cliContractRuntimeDigest)); + assert.match(referenceSource, /CLI contract configuration.*script\/cli-contract\/config\.mjs/); assert.match( referenceSource, /bundled schema was copied byte-for-byte from\s+`docs\/architecture\/design\/foundation-plan\.schema\.json` at Service revision[\s\S]*?exact contract provenance, not\s+release or execution evidence/, @@ -4388,7 +4372,7 @@ test("recovery evals stage and preserve existing Plan state", async () => { recoverySection[1], /\[stable error family\]\(references\/diagnostics-and-recovery\.md#stable-error-families\)/, ); - assert(recoveryReference.includes(cliContractBaseline)); + assert.match(recoveryReference, /CLI contract configuration.*script\/cli-contract\/config\.mjs/); const stableErrors = recoveryReference.match( /## Stable error families([\s\S]*?)## Ambiguous mutations/, ); From 4a27fad72cdee6bd85c3cf15c6e94ad655f0a2a2 Mon Sep 17 00:00:00 2001 From: Raghu Betina Date: Tue, 22 Sep 2026 19:36:11 -0500 Subject: [PATCH 2/2] Focus Skills checks on current behavior Stop coupling ordinary checks to historical receipts, mutable prose, and source-token inventories. Keep current metadata, fixture semantics, package provenance, executable integration, and publication safeguards. Remove the retired installer recorder and its observation-only helpers. Keep the explicit isolated environment used by packaged Claude checks. Leave deep Publication projections to CLI tests that now cover creation and polling outcomes. Retain a consumer polling failure and recovery journeys against the unchanged public CLI pin. Reconcile the candidate digest after combining the approved release cleanup with current Plan guidance. These infrastructure cuts do not change the assembled plugin bytes. --- README.md | 11 +- release/compatibility.json | 2 +- script/check-claude-plugin-install.mjs | 576 ----- script/claude-plugin-observation.mjs | 218 -- .../cli-contract/publication-validation.mjs | 188 -- script/plugin-isolation.mjs | 206 -- test/claude-plugin-observation.test.mjs | 240 -- test/plugin-isolation.test.mjs | 199 +- test/release-compatibility.test.mjs | 655 +---- test/repository.test.mjs | 2115 +---------------- 10 files changed, 95 insertions(+), 4315 deletions(-) delete mode 100644 script/check-claude-plugin-install.mjs delete mode 100644 script/claude-plugin-observation.mjs delete mode 100644 test/claude-plugin-observation.test.mjs diff --git a/README.md b/README.md index 19615a6..9e05c5b 100644 --- a/README.md +++ b/README.md @@ -116,8 +116,7 @@ of Compilation, builds, or CI. A registry page example does not change the Rails ## Development -Use Node.js 22 or newer and a real, non-shallow Git checkout. Checks inspect the repository index, historical -evidence objects, and the complete Skill tree. +Use Node.js 22 or newer and a Git checkout. Checks inspect the repository index and complete Skill tree. ~~~sh npm ci --ignore-scripts @@ -133,8 +132,9 @@ The check covers: - deterministic plugin packaging with a stub CLI; and - release compatibility. -CI separately checks the exact pinned CLI contract and candidate package digest. The release runbook owns the -commands for reproducing that check against a local exact CLI checkout. +CI checks consumer commands against the exact pinned CLI and verifies the candidate package digest. CLI-owned tests +cover the complete Publication protocol matrix; Skills retains representative recovery cases, compatible fixtures, +and packaged-executable integration. The release runbook owns the commands for reproducing these checks. Preview the plugin directly from a checkout: @@ -152,8 +152,7 @@ node script/check-packaged-claude-plugin-install.mjs --claude /absolute/path/to/ These install checks use temporary client state, discover every Skill through the real client, compare all Skill files with the candidate, and exercise the bundled CLI without a global `firstdraft`. They need no agent login or -First Draft service. The old `check-claude-plugin-install.mjs` is a retired historical recording path, not the -assembled-package check. Behavioral cases remain shared across clients; [the eval guide](evals/README.md) describes +First Draft service. Behavioral cases remain shared across clients; [the eval guide](evals/README.md) describes the separate agent-session checks. If the installed GitHub CLI supports Skill preview: diff --git a/release/compatibility.json b/release/compatibility.json index 8471eaf..90363b9 100644 --- a/release/compatibility.json +++ b/release/compatibility.json @@ -4,7 +4,7 @@ "version": "0.4.0", "plugin_source": { "package": "@firstdraft.com/claude-code", - "tarball_sha256": "a09786608db0452fcaff6c8f42cfd28148ced16834f94a03493fbff0edd1e223" + "tarball_sha256": "4a6a4dfb0aa23e803bc62d58050c538ee63f025c8c971c3439453a8e0397fbce" }, "requires": { "api_contract": [ diff --git a/script/check-claude-plugin-install.mjs b/script/check-claude-plugin-install.mjs deleted file mode 100644 index db86ab9..0000000 --- a/script/check-claude-plugin-install.mjs +++ /dev/null @@ -1,576 +0,0 @@ -import assert from "node:assert/strict"; -import { - chmodSync, - existsSync, - mkdirSync, - mkdtempSync, - readFileSync, - readdirSync, - rmSync, - statSync, - writeFileSync, -} from "node:fs"; -import { homedir, tmpdir } from "node:os"; -import path from "node:path"; -import { fileURLToPath } from "node:url"; - -import { - canonicalClaudePluginSkillFiles, - classifyInventoryEntry, - forbiddenClaudePluginPathSegments, -} from "./claude-plugin-boundaries.mjs"; -import { - parseClaudeCodeVersion, - parseComponentInventory, - resolveNativeExecutable, - runPluginCommand, -} from "./claude-plugin-command.mjs"; -import { - observedManifestValidation, - observedFileBytes, - observedFileInventory, - observedFileTreeSha256, - renderStatePresenceNames, - reviewedPackagingObservation, - serializePluginObservation, -} from "./claude-plugin-observation.mjs"; -import { - assertDefaultClaudeStateLocations, - changedStateEntries, - isolatedPluginEnvironment, - pathEntryExists, - pluginStateTargets, - resolvedStateTargetDiagnostics, - snapshotStateTargets, - stateTargetPresence, -} from "./plugin-isolation.mjs"; - -const repository = path.dirname(path.dirname(fileURLToPath(import.meta.url))); -const installSmokeMarketplace = JSON.parse( - readFileSync( - path.join(repository, ".claude-plugin", "marketplace.json"), - "utf8", - ), -); -assertLocalInstallSmokeSupported(installSmokeMarketplace); -assertDefaultClaudeStateLocations(process.env); - -const portableSkill = path.join( - repository, - "skills", - "create-full-stack-app", -); -const marketplaceName = "firstdraft-skills"; -const pluginName = "firstdraft"; -const forbiddenSegments = new Set(forbiddenClaudePluginPathSegments); -const requiredRegistryTargets = [ - "installedPlugins", - "knownMarketplaces", - "pluginCatalog", -]; -const packageManagers = [ - "bun", - "bunx", - "corepack", - "npm", - "npx", - "pnpm", - "pnpx", - "yarn", - "yarnpkg", -]; -const committedObservationPath = path.join( - repository, - "evidence", - "claude-code-plugin-install-observation.json", -); -const observationPath = requestedObservationPath(process.argv.slice(2)); - -const claude = resolveNativeExecutable(process.env.CLAUDE_BIN ?? "claude"); -const realHome = homedir(); -const realConfigDirectory = path.join(realHome, ".claude"); -const realPluginsDirectory = path.join(realConfigDirectory, "plugins"); -const realStateTargets = pluginStateTargets({ - configDirectory: realConfigDirectory, - pluginsDirectory: realPluginsDirectory, - marketplaceName, - pluginName, -}); -const realStateBefore = realClaudeStateSnapshot(); -const realStatePresence = stateTargetPresence(realStateBefore); -assert( - requiredRegistryTargets.some((name) => - realStatePresence.present.includes(name), - ), - "real Claude registry monitor found none of installedPlugins, " + - "knownMarketplaces, or pluginCatalog; update pluginStateTargets before " + - "trusting the isolation claim", -); -const smokeRoot = mkdtempSync( - path.join(tmpdir(), "firstdraft-claude-plugin-install-"), -); -const packageManagerSentinel = path.join( - smokeRoot, - "unexpected-package-manager", -); -const failures = []; -let successOutput; -let observation; -let realStateEscapeReported = false; - -try { - const homeDirectory = path.join(smokeRoot, "home"); - const configDirectory = path.join(smokeRoot, "config"); - const pluginsDirectory = path.join(smokeRoot, "plugins"); - const runtimeDirectory = path.join(smokeRoot, "runtime"); - const temporaryDirectory = path.join(smokeRoot, "tmp"); - const xdgCacheDirectory = path.join(smokeRoot, "xdg-cache"); - const xdgConfigDirectory = path.join(smokeRoot, "xdg-config"); - const xdgDataDirectory = path.join(smokeRoot, "xdg-data"); - const xdgRuntimeDirectory = path.join(smokeRoot, "xdg-runtime"); - const xdgStateDirectory = path.join(smokeRoot, "xdg-state"); - const guardsDirectory = path.join(smokeRoot, "package-manager-guards"); - const workingDirectory = path.join(smokeRoot, "work"); - for (const directory of [ - homeDirectory, - configDirectory, - pluginsDirectory, - runtimeDirectory, - temporaryDirectory, - xdgCacheDirectory, - xdgConfigDirectory, - xdgDataDirectory, - xdgRuntimeDirectory, - xdgStateDirectory, - guardsDirectory, - workingDirectory, - ]) { - mkdirSync(directory, { mode: 0o700, recursive: true }); - chmodSync(directory, 0o700); - } - - for (const packageManager of packageManagers) { - const guard = path.join(guardsDirectory, packageManager); - writeFileSync( - guard, - "#!/bin/sh\n" + - `printf '%s\\n' \"$0 $*\" >> ${shellQuote(packageManagerSentinel)}\n` + - "exit 97\n", - { mode: 0o700 }, - ); - chmodSync(guard, 0o700); - } - - const environment = isolatedPluginEnvironment({ - guardsDirectory, - homeDirectory, - configDirectory, - pluginsDirectory, - runtimeDirectory, - temporaryDirectory, - xdgCacheDirectory, - xdgConfigDirectory, - xdgDataDirectory, - xdgRuntimeDirectory, - xdgStateDirectory, - }); - const commandOptions = { - cwd: workingDirectory, - environment, - }; - const claudeCodeVersion = parseClaudeCodeVersion( - runPluginCommand(claude, ["--version"], commandOptions).stdout, - ); - const marketplaceValidationArguments = [ - "plugin", - "validate", - "--strict", - repository, - ]; - const marketplaceValidationResult = runPluginCommand( - claude, - marketplaceValidationArguments, - commandOptions, - ); - const marketplaceValidation = observedManifestValidation({ - arguments_: marketplaceValidationArguments, - repository, - result: marketplaceValidationResult, - }); - const previewValidationArguments = [ - "plugin", - "validate", - "--strict", - path.join(repository, ".claude-plugin", "plugin.json"), - ]; - const previewValidationResult = runPluginCommand( - claude, - previewValidationArguments, - commandOptions, - ); - const previewValidation = observedManifestValidation({ - arguments_: previewValidationArguments, - repository, - result: previewValidationResult, - }); - - runPluginCommand( - claude, - ["plugin", "marketplace", "add", repository, "--scope", "user"], - commandOptions, - ); - const isolatedMarketplaceRegistry = path.join( - pluginsDirectory, - "known_marketplaces.json", - ); - assert( - existsSync(isolatedMarketplaceRegistry), - "isolated Claude marketplace registry was not created", - ); - const isolatedMarketplaces = JSON.parse( - readFileSync(isolatedMarketplaceRegistry, "utf8"), - ); - assert( - Object.hasOwn(isolatedMarketplaces, marketplaceName), - "isolated Claude marketplace registry omits firstdraft-skills", - ); - const isolatedMarketplaceTree = path.join( - pluginsDirectory, - "marketplaces", - marketplaceName, - ); - assert.equal( - pathEntryExists(isolatedMarketplaceTree), - false, - "isolated directory marketplace unexpectedly created a persistent tree", - ); - assertRealStateUnchanged("after isolated marketplace add"); - runPluginCommand( - claude, - ["plugin", "install", `${pluginName}@${marketplaceName}`, "--scope", "user"], - commandOptions, - ); - assertRealStateUnchanged("after isolated plugin install"); - const details = runPluginCommand( - claude, - ["plugin", "details", `${pluginName}@${marketplaceName}`], - commandOptions, - ).stdout; - const marketplace = JSON.parse( - readFileSync( - path.join(repository, ".claude-plugin", "marketplace.json"), - "utf8", - ), - ); - const marketplacePlugin = marketplace.plugins.find( - ({ name }) => name === pluginName, - ); - assert(marketplacePlugin, "marketplace plugin entry is missing"); - const inventory = { - ...parseComponentInventory(details), - commandsDeclared: Object.hasOwn(marketplacePlugin, "commands"), - }; - assert.deepEqual(inventory, { - agents: 0, - commandsDeclared: false, - hooks: 0, - lspServers: 0, - mcpServers: 0, - skillsAndCommands: 1, - }); - - assert.equal( - existsSync(packageManagerSentinel), - false, - `Claude Code invoked a package manager: ${ - existsSync(packageManagerSentinel) - ? readFileSync(packageManagerSentinel, "utf8").trim() - : "unknown" - }`, - ); - const isolatedPluginData = path.join( - pluginsDirectory, - "data", - `${pluginName}-${marketplaceName}`, - ); - assert.equal( - pathEntryExists(isolatedPluginData), - false, - "isolated install unexpectedly created persistent plugin data", - ); - - const pluginVersionsDirectory = path.join( - pluginsDirectory, - "cache", - marketplaceName, - pluginName, - ); - assert( - existsSync(pluginVersionsDirectory), - "isolated Claude plugin cache path was not created", - ); - const versions = readdirSync(pluginVersionsDirectory, { - withFileTypes: true, - }) - .filter((entry) => entry.isDirectory()) - .map((entry) => entry.name) - .sort(); - assert.equal(versions.length, 1, `expected one cached version: ${versions}`); - - const installedPlugin = path.join(pluginVersionsDirectory, versions[0]); - const expectedFiles = filesUnder(portableSkill); - const installedFiles = filesUnder(installedPlugin); - assert.deepEqual(expectedFiles, canonicalClaudePluginSkillFiles); - assert.deepEqual(installedFiles, expectedFiles); - - for (const relativePath of installedFiles) { - assert.equal( - relativePath - .split(path.sep) - .some((segment) => forbiddenSegments.has(segment)), - false, - `unexpected installed path: ${relativePath}`, - ); - assertNoAutodiscoverableComponent(relativePath); - assert.deepEqual( - readFileSync(path.join(installedPlugin, relativePath)), - readFileSync(path.join(portableSkill, relativePath)), - `installed bytes differ: ${relativePath}`, - ); - } - const installedFileInventory = observedFileInventory( - installedPlugin, - installedFiles, - ); - const canonicalFileInventory = observedFileInventory( - portableSkill, - expectedFiles, - ); - assert.deepEqual(installedFileInventory, canonicalFileInventory); - const installedBytes = observedFileBytes(installedFileInventory); - const treeSha256 = observedFileTreeSha256(installedFileInventory); - observation = { - schemaVersion: 3, - observedOn: new Date().toISOString().slice(0, 10), - claudeCode: { - version: claudeCodeVersion, - componentInventory: { - agents: inventory.agents, - hooks: inventory.hooks, - lspServers: inventory.lspServers, - mcpServers: inventory.mcpServers, - skillsAndCommands: inventory.skillsAndCommands, - }, - }, - manifestValidation: { - marketplace: marketplaceValidation, - previewPlugin: previewValidation, - }, - installedPlugin: { - marketplace: marketplaceName, - name: pluginName, - commandsDeclared: inventory.commandsDeclared, - fileCount: installedFileInventory.length, - totalBytes: installedBytes, - treeSha256, - files: installedFileInventory, - }, - realStateMonitor: { - present: realStatePresence.present, - absent: realStatePresence.absent, - requiredRegistryAnyOf: requiredRegistryTargets, - excluded: ["~/.claude.json"], - }, - }; - - successOutput = - "Claude Code strict validation: marketplace=passed, preview=passed; " + - `isolated install: ${installedFiles.length} canonical Skill files, ` + - `${installedBytes} bytes; live inventory Skills=1, Agents=0, Hooks=0, ` + - "MCP servers=0, LSP servers=0; derived Commands=absent from manifest " + - "declaration and exact installed files because CLI combines Skills/Commands; " + - "no PATH-level package manager invocation; real-state monitor " + - `present=${renderStatePresenceNames(realStatePresence.present)}, ` + - `absent=${renderStatePresenceNames(realStatePresence.absent)}, ` + - "excluded=~/.claude.json\n"; -} catch (error) { - if (existsSync(packageManagerSentinel)) { - let invocations = "sentinel present but unreadable"; - try { - invocations = readFileSync(packageManagerSentinel, "utf8").trim(); - } catch (sentinelError) { - failures.push( - new Error("failed to read the package-manager sentinel", { - cause: sentinelError, - }), - ); - } - failures.push( - new Error( - `Claude Code invoked a PATH-level package manager:\n${invocations}`, - { cause: error }, - ), - ); - } else { - failures.push(error); - } -} finally { - try { - rmSync(smokeRoot, { recursive: true, force: true }); - } catch (error) { - failures.push( - new Error("failed to remove isolated Claude state", { cause: error }), - ); - } -} - -if (existsSync(smokeRoot)) { - failures.push(new Error("isolated Claude state was not removed")); -} -try { - if (!realStateEscapeReported) { - assertRealStateUnchanged("after isolated smoke cleanup"); - } -} catch (error) { - failures.push( - new Error("failed to verify real Claude state after isolated smoke", { - cause: error, - }), - ); -} -if (failures.length > 0) { - throw new AggregateError(failures, "Claude Code isolated install smoke failed"); -} -assert(observation, "isolated install did not produce an observation"); -observation.checks = { - childWorkingDirectory: "isolated", - packageManagerInvocation: "absent", - realStateUnchanged: true, - temporaryStateRemoved: true, -}; -if (observationPath) { - writeFileSync(observationPath, serializePluginObservation(observation)); -} else { - assertMatchesCommittedObservation(observation); -} -process.stdout.write(successOutput); - -function filesUnder(directory, root = directory) { - const entries = readdirSync(directory, { withFileTypes: true }).sort( - compareDirectoryEntries, - ); - const files = []; - - for (const entry of entries) { - const item = path.join(directory, entry.name); - switch (classifyInventoryEntry(entry, item)) { - case "directory": - files.push(...filesUnder(item, root)); - break; - case "file": - assert(statSync(item).isFile()); - files.push(path.relative(root, item)); - break; - } - } - - return files; -} - -function assertNoAutodiscoverableComponent(relativePath) { - const segments = relativePath.split(path.sep); - assert.equal( - segments.includes("commands"), - false, - `installed source contains a Commands component: ${relativePath}`, - ); - assert.equal( - segments.includes("hooks"), - false, - `installed source contains a Hooks component: ${relativePath}`, - ); - assert.notEqual( - path.basename(relativePath), - ".mcp.json", - `installed source contains an MCP component: ${relativePath}`, - ); - if (segments.includes("agents")) { - assert.notEqual( - path.extname(relativePath), - ".md", - `installed source contains an Agent component: ${relativePath}`, - ); - } -} - -function changedRealStateEntries(before) { - return changedStateEntries(before, realStateTargets); -} - -function assertRealStateUnchanged(stage) { - const changedRealState = changedRealStateEntries(realStateBefore); - if (changedRealState.length === 0) return; - - realStateEscapeReported = true; - throw new Error( - `real Claude configuration or plugin cache changed ${stage}: ` + - `${resolvedStateTargetDiagnostics(changedRealState, realStateTargets).join(", ")}\n` + - "Inspect the named paths, then remove any escaped user-scoped state with:\n" + - " claude plugin uninstall firstdraft@firstdraft-skills --scope user\n" + - " claude plugin marketplace remove firstdraft-skills --scope user", - ); -} - -function realClaudeStateSnapshot() { - return snapshotStateTargets(realStateTargets); -} - -function assertMatchesCommittedObservation(current) { - const committed = JSON.parse(readFileSync(committedObservationPath, "utf8")); - assert.deepEqual( - reviewedPackagingObservation(current), - reviewedPackagingObservation(committed), - "live Claude Code plugin observation differs from committed evidence; " + - "the historical recording path is retired, so use the vendored-package " + - "qualification in RELEASING.md for new evidence", - ); -} - -function compareDirectoryEntries(left, right) { - if (left.name < right.name) return -1; - if (left.name > right.name) return 1; - return 0; -} - -function shellQuote(value) { - return `'${value.replaceAll("'", `'"'"'`)}'`; -} - -function requestedObservationPath(arguments_) { - if (arguments_.length === 0) return undefined; - assert.deepEqual( - arguments_.slice(0, 1), - ["--observation-output"], - "only --observation-output is supported", - ); - assert.equal( - arguments_.length, - 2, - "--observation-output requires exactly one path", - ); - return path.resolve(arguments_[1]); -} - -function assertLocalInstallSmokeSupported(marketplace) { - const marketplacePlugin = marketplace.plugins?.find( - ({ name }) => name === "firstdraft", - ); - assert(marketplacePlugin, "marketplace plugin entry is missing"); - assert.equal( - typeof marketplacePlugin.source, - "string", - "the historical local Claude Code install smoke does not support the npm " + - "plugin source; use the staged-package preflight and then qualify the " + - "published package through the public marketplace", - ); -} diff --git a/script/claude-plugin-observation.mjs b/script/claude-plugin-observation.mjs deleted file mode 100644 index c10541b..0000000 --- a/script/claude-plugin-observation.mjs +++ /dev/null @@ -1,218 +0,0 @@ -import assert from "node:assert/strict"; -import { createHash } from "node:crypto"; -import { readFileSync, statSync } from "node:fs"; -import path from "node:path"; -import { stripVTControlCharacters } from "node:util"; - -export function observedManifestValidation({ - arguments_, - repository, - result, -}) { - assert.deepEqual( - arguments_.slice(0, 3), - ["plugin", "validate", "--strict"], - "manifest validation evidence requires the exact strict CLI invocation", - ); - assert.equal( - arguments_.length, - 4, - "manifest validation evidence requires exactly one validation path", - ); - assert.equal( - result.status, - 0, - "manifest validation evidence requires a successful exit status", - ); - const capturedOutput = normalizeObservationText( - [result.stdout, result.stderr].filter(Boolean).join("\n"), - repository, - ); - const successLines = [ - ...capturedOutput.matchAll( - /^[ \t]*(?:✔|✓)?[ \t]*Validation passed[ \t]*$/gim, - ), - ]; - assert.equal( - successLines.length, - 1, - "strict validator output must contain exactly one `Validation passed` " + - `line; captured output:\n${capturedOutput || "(empty)"}`, - ); - - const invokedPath = path.resolve(arguments_[3]); - const relativePath = path.relative(repository, invokedPath); - assert( - relativePath === "" || - (!relativePath.startsWith(`..${path.sep}`) && - relativePath !== ".." && - !path.isAbsolute(relativePath)), - "manifest validation path must remain within the repository", - ); - const normalizedInvokedPath = normalizeObservationArgument( - invokedPath, - repository, - ); - const expectedValidationTarget = { - kind: relativePath === "" ? "marketplace" : "plugin", - path: - relativePath === "" - ? "/.claude-plugin/marketplace.json" - : normalizedInvokedPath, - }; - const capturedValidationTargets = [ - ...capturedOutput.matchAll( - /^[ \t]*Validating[ \t]+(marketplace|plugin)[ \t]+manifest:[ \t]*(.+?)[ \t]*$/gim, - ), - ].map((match) => ({ kind: match[1].toLowerCase(), path: match[2] })); - assert.deepEqual( - capturedValidationTargets, - [expectedValidationTarget], - "captured validator target must match the normalized strict invocation", - ); - - return { - capturedOutput, - normalizedArgv: [ - "", - ...arguments_.map((argument) => - normalizeObservationArgument(argument, repository), - ), - ], - passed: true, - path: - relativePath === "" ? "." : relativePath.split(path.sep).join("/"), - strict: true, - }; -} - -export function renderManifestValidationEvidence(label, validation) { - assert.equal(typeof label, "string", "validation evidence label must be text"); - assert(label.length > 0, "validation evidence label must not be empty"); - assert( - Array.isArray(validation.normalizedArgv), - "validation evidence requires normalized argv", - ); - assert.equal( - typeof validation.capturedOutput, - "string", - "validation evidence requires captured normalized output", - ); - return [ - `Repository-relative target: \`${validation.path}\` (${label})`, - "", - "Normalized child argv:", - "", - "```json", - JSON.stringify(validation.normalizedArgv), - "```", - "", - "Captured normalized output:", - "", - "```text", - validation.capturedOutput, - "```", - ].join("\n"); -} - -export function renderStatePresenceNames(names) { - assert(Array.isArray(names), "state-presence names must be an array"); - assert( - names.every((name) => typeof name === "string" && name.length > 0), - "state-presence names must contain only nonempty strings", - ); - return names.length > 0 ? names.join(",") : "(none)"; -} - -export function assertNoObservationAbsolutePathLeaks(observation) { - const posixAbsolutePath = - /(?:^|[\s"'`([{=,:])\/(?!\/)[^\s"'`)\]}>,;]+|file:\/\/\/[^\s"'`)\]}>,;]+/; - const windowsAbsolutePath = - /(?:^|[\s"'`([{=,:])(?:[A-Za-z]:[\\/]|\\\\[^\\/\s]+[\\/][^\\/\s]+)/; - - for (const { location, text } of observationTextEntries(observation)) { - const leakedPath = - text.match(posixAbsolutePath) ?? text.match(windowsAbsolutePath); - assert.equal( - leakedPath, - null, - `observation ${location} contains an absolute filesystem path: ${leakedPath?.[0]}`, - ); - } -} - -export function reviewedPackagingObservation(observation) { - assert(observation && typeof observation === "object"); - return { - schemaVersion: observation.schemaVersion, - claudeCode: observation.claudeCode, - manifestValidation: observation.manifestValidation, - installedPlugin: observation.installedPlugin, - }; -} - -export function observedFileInventory(directory, relativePaths) { - return relativePaths.map((relativePath) => { - const absolutePath = path.join(directory, relativePath); - const contents = readFileSync(absolutePath); - const details = statSync(absolutePath); - if (!details.isFile()) { - throw new Error(`observed inventory entry is not a file: ${relativePath}`); - } - return { - path: relativePath, - bytes: contents.length, - sha256: createHash("sha256").update(contents).digest("hex"), - }; - }); -} - -export function observedFileTreeSha256(files) { - return createHash("sha256") - .update(JSON.stringify(files)) - .digest("hex"); -} - -export function observedFileBytes(files) { - return files.reduce((total, file) => total + file.bytes, 0); -} - -export function serializePluginObservation(observation) { - assertNoObservationAbsolutePathLeaks(observation); - return `${JSON.stringify(observation, null, 2)}\n`; -} - -function normalizeObservationArgument(argument, repository) { - assert.equal(typeof argument, "string", "validator arguments must be text"); - if (argument === repository) return ""; - if (argument.startsWith(`${repository}${path.sep}`)) { - const relativePath = path.relative(repository, argument); - return `/${relativePath.split(path.sep).join("/")}`; - } - return argument; -} - -function normalizeObservationText(source, repository) { - return stripVTControlCharacters(source) - .replaceAll("\r\n", "\n") - .replaceAll(repository, "") - .trimEnd(); -} - -function* observationTextEntries(value, location = "$") { - if (typeof value === "string") { - yield { location, text: value }; - return; - } - if (Array.isArray(value)) { - for (const [index, item] of value.entries()) { - yield* observationTextEntries(item, `${location}[${index}]`); - } - return; - } - if (value && typeof value === "object") { - for (const [key, item] of Object.entries(value)) { - yield* observationTextEntries(item, `${location}.${key}`); - } - } -} diff --git a/script/cli-contract/publication-validation.mjs b/script/cli-contract/publication-validation.mjs index bf103b7..75c7327 100644 --- a/script/cli-contract/publication-validation.mjs +++ b/script/cli-contract/publication-validation.mjs @@ -3,7 +3,6 @@ import { readFileSync } from "node:fs"; import { projectId, - publicationId, safeGithubReasonCodes, storedApiUrl, } from "./config.mjs"; @@ -26,166 +25,6 @@ import { export async function verifyPublicationValidation(context) { const planSource = readFileSync(context.moviePlanPath); const digest = sha256(planSource); - const invalidCases = [ - { - label: "different-project", - changes: { projectChanges: { id: publicationId } }, - }, - { - label: "different-project-head", - changes: { - projectChanges: { head_source_sha256: "8".repeat(64) }, - }, - }, - { - label: "different-compilation-head", - changes: { - compilationChanges: { head_source_sha256: "8".repeat(64) }, - }, - }, - { - label: "different-compilation-generation", - changes: { compilationChanges: { graph_version: 2 } }, - }, - { - label: "public-repository", - changes: { repositoryChanges: { private: false } }, - }, - { - label: "organization-owner", - changes: { - repositoryChanges: { - owner: { id: 123456, login: "octocat", type: "Organization" }, - }, - }, - }, - { - label: "unknown-progress-phase", - status: "provisioning_repository", - changes: { progressChanges: { phase: "github_guessing" } }, - }, - { - label: "unknown-progress-reason", - status: "provisioning_repository", - changes: { - progressChanges: { - phase: "github_preflight", - retry_count: 1, - reason_code: "github.private_exception", - }, - }, - }, - { - label: "progress-retry-count-out-of-range", - status: "provisioning_repository", - changes: { - progressChanges: { - phase: "github_preflight", - retry_count: 8, - reason_code: "github.api_unavailable", - }, - }, - }, - { - label: "progress-retry-without-reason", - status: "provisioning_repository", - changes: { - progressChanges: { - phase: "github_preflight", - retry_count: 1, - }, - }, - }, - { - label: "progress-reason-without-retry", - status: "provisioning_repository", - changes: { - progressChanges: { - phase: "github_preflight", - reason_code: "github.api_unavailable", - }, - }, - }, - { - label: "progress-retry-outside-preflight", - status: "provisioning_repository", - changes: { - progressChanges: { - phase: "preparing_repository", - retry_count: 1, - reason_code: "github.api_unavailable", - }, - }, - }, - { - label: "progress-invalid-retry-time", - status: "provisioning_repository", - changes: { - progressChanges: { - phase: "github_preflight", - retry_at: "later", - retry_count: 1, - reason_code: "github.api_unavailable", - }, - }, - }, - { - label: "progress-retry-time-without-count", - status: "provisioning_repository", - changes: { - progressChanges: { - phase: "github_preflight", - retry_at: "2026-08-07T16:15:00.000000Z", - }, - }, - }, - { - label: "progress-noncanonical-retry-time", - status: "provisioning_repository", - changes: { - progressChanges: { - phase: "github_preflight", - retry_at: "2026-08-07T16:15:00Z", - retry_count: 1, - reason_code: "github.api_unavailable", - }, - }, - }, - { - label: "failed-publication-with-running-compilation", - status: "failed", - changes: { compilationChanges: { status: "running" } }, - }, - { - label: "cancelled-publication-with-queued-compilation", - status: "cancelled", - changes: { compilationChanges: { status: "queued" } }, - }, - ]; - - for (const { label, status = "succeeded", changes } of invalidCases) { - const invalid = publicationLifecycleProjection( - digest, - status, - changes, - ); - await assertInvalidPublication( - context, - planSource, - `publication-${label}`, - invalid, - ); - } - - const additive = publicationLifecycleProjection(digest, "succeeded"); - additive.canary = "canary-private-publication-extension"; - await assertInvalidPublication( - context, - planSource, - "publication-additive-response", - additive, - ["canary-private-publication-extension"], - ); const additiveProgress = publicationLifecycleProjection(digest, "succeeded"); additiveProgress.publication.progress.canary = "canary-private-publication-progress-extension"; @@ -196,33 +35,6 @@ export async function verifyPublicationValidation(context) { additiveProgress, ["canary-private-publication-progress-extension"], ); - const missingProgress = publicationLifecycleProjection(digest, "succeeded"); - delete missingProgress.publication.progress; - await assertInvalidPublication( - context, - planSource, - "publication-missing-progress", - missingProgress, - ); - const nullProgress = publicationLifecycleProjection(digest, "succeeded"); - nullProgress.publication.progress = null; - await assertInvalidPublication( - context, - planSource, - "publication-null-progress", - nullProgress, - ); - const incompleteProgress = publicationLifecycleProjection( - digest, - "succeeded", - ); - delete incompleteProgress.publication.progress.retry_at; - await assertInvalidPublication( - context, - planSource, - "publication-incomplete-progress", - incompleteProgress, - ); await verifyObservationRegression(context, planSource, digest); await verifyGenerationReplacement(context, planSource, digest); diff --git a/script/plugin-isolation.mjs b/script/plugin-isolation.mjs index 14eb7ce..90dc4e7 100644 --- a/script/plugin-isolation.mjs +++ b/script/plugin-isolation.mjs @@ -1,38 +1,3 @@ -import assert from "node:assert/strict"; -import { createHash } from "node:crypto"; -import { - closeSync, - constants, - fstatSync, - lstatSync, - openSync, - readFileSync, - readdirSync, -} from "node:fs"; -import path from "node:path"; - -const parentStateLocationVariables = Object.freeze([ - "CLAUDE_CONFIG_DIR", - "CLAUDE_CODE_PLUGIN_CACHE_DIR", -]); - -export function assertDefaultClaudeStateLocations(environment) { - assert( - environment && typeof environment === "object", - "parent environment must be an object", - ); - const overrides = parentStateLocationVariables.filter((name) => - Object.hasOwn(environment, name), - ); - assert.equal( - overrides.length, - 0, - "Claude plugin isolation smoke requires CLAUDE_CONFIG_DIR and " + - "CLAUDE_CODE_PLUGIN_CACHE_DIR to be unset; parent overrides present: " + - overrides.join(", "), - ); -} - export function isolatedPluginEnvironment({ guardsDirectory, homeDirectory, @@ -64,174 +29,3 @@ export function isolatedPluginEnvironment({ XDG_STATE_HOME: xdgStateDirectory, }; } - -export function pluginStateTargets({ - configDirectory, - pluginsDirectory, - marketplaceName, - pluginName, -}) { - return { - credentials: metadataOnly( - path.join(configDirectory, ".credentials.json"), - ), - installedPlugins: contentAware( - path.join(pluginsDirectory, "installed_plugins.json"), - ), - knownMarketplaces: contentAware( - path.join(pluginsDirectory, "known_marketplaces.json"), - ), - pluginCatalog: contentAware( - path.join(pluginsDirectory, "plugin-catalog-cache.json"), - ), - settings: metadataOnly(path.join(configDirectory, "settings.json")), - settingsLocal: metadataOnly( - path.join(configDirectory, "settings.local.json"), - ), - targetCache: contentAware( - path.join(pluginsDirectory, "cache", marketplaceName), - ), - targetData: contentAware( - path.join(pluginsDirectory, "data", `${pluginName}-${marketplaceName}`), - ), - targetMarketplace: contentAware( - path.join(pluginsDirectory, "marketplaces", marketplaceName), - ), - }; -} - -export function snapshotStateTargets(targets) { - return Object.fromEntries( - Object.entries(targets).map( - ([name, { target, includeFileContents }]) => [ - name, - snapshotStateTarget(target, { includeFileContents }), - ], - ), - ); -} - -export function changedStateEntries(before, targets) { - const after = snapshotStateTargets(targets); - return [...new Set([...Object.keys(before), ...Object.keys(after)])] - .sort() - .filter((name) => before[name]?.digest !== after[name]?.digest); -} - -export function stateTargetPresence(snapshot) { - const presence = { present: [], absent: [] }; - for (const name of Object.keys(snapshot).sort()) { - presence[snapshot[name].present ? "present" : "absent"].push(name); - } - return presence; -} - -export function resolvedStateTargetDiagnostics(names, targets) { - assert(Array.isArray(names), "changed state names must be an array"); - return names.map((name) => { - const target = targets[name]?.target; - assert.equal( - typeof target, - "string", - `changed state target is missing: ${name}`, - ); - return `${name}=${path.resolve(target)}`; - }); -} - -export function pathEntryExists(target) { - try { - lstatSync(target); - return true; - } catch (error) { - if (error.code === "ENOENT") return false; - throw error; - } -} - -function metadataOnly(target) { - return { target, includeFileContents: false }; -} - -function contentAware(target) { - return { target, includeFileContents: true }; -} - -function snapshotStateTarget(target, { includeFileContents }) { - const entry = filesystemEntry(target, { includeFileContents }); - return { - digest: createHash("sha256") - .update(JSON.stringify(entry)) - .digest("hex"), - present: entry[0] !== "missing", - }; -} - -function filesystemEntry(target, { includeFileContents }) { - let details; - try { - details = lstatSync(target, { bigint: true }); - } catch (error) { - if (error.code === "ENOENT") return ["missing"]; - throw error; - } - - const metadata = { - birthtimeNs: details.birthtimeNs.toString(), - ctimeNs: details.ctimeNs.toString(), - device: details.dev.toString(), - group: details.gid.toString(), - inode: details.ino.toString(), - mode: details.mode.toString(), - mtimeNs: details.mtimeNs.toString(), - links: details.nlink.toString(), - size: details.size.toString(), - user: details.uid.toString(), - }; - - if (details.isSymbolicLink()) { - throw new Error(`monitored state contains a symlink: ${target}`); - } - if (details.isFile()) { - return [ - "file", - metadata, - includeFileContents - ? regularFileDigest(target, details) - : null, - ]; - } - if (!details.isDirectory()) return ["other", metadata]; - - const entries = readdirSync(target) - .sort() - .map((name) => [ - name, - filesystemEntry(path.join(target, name), { includeFileContents }), - ]); - return ["directory", metadata, entries]; -} - -function regularFileDigest(target, expectedDetails) { - if (constants.O_NOFOLLOW === undefined) { - throw new Error("filesystem snapshots require O_NOFOLLOW"); - } - const descriptor = openSync( - target, - constants.O_RDONLY | constants.O_NOFOLLOW, - ); - try { - const openedDetails = fstatSync(descriptor, { bigint: true }); - if ( - openedDetails.dev !== expectedDetails.dev || - openedDetails.ino !== expectedDetails.ino - ) { - throw new Error(`filesystem entry changed while snapshotting: ${target}`); - } - return createHash("sha256") - .update(readFileSync(descriptor)) - .digest("hex"); - } finally { - closeSync(descriptor); - } -} diff --git a/test/claude-plugin-observation.test.mjs b/test/claude-plugin-observation.test.mjs deleted file mode 100644 index 70f2c3c..0000000 --- a/test/claude-plugin-observation.test.mjs +++ /dev/null @@ -1,240 +0,0 @@ -import assert from "node:assert/strict"; -import path from "node:path"; -import test from "node:test"; - -import { - assertNoObservationAbsolutePathLeaks, - observedManifestValidation, - renderManifestValidationEvidence, - renderStatePresenceNames, - reviewedPackagingObservation, -} from "../script/claude-plugin-observation.mjs"; - -test("manifest validation evidence comes from the exact successful invocation", () => { - const repository = path.resolve("/checkout"); - assert.deepEqual( - observedManifestValidation({ - arguments_: ["plugin", "validate", "--strict", repository], - repository, - result: { - status: 0, - stderr: "", - stdout: - `Validating marketplace manifest: ${repository}/.claude-plugin/marketplace.json\n` + - "\u001b[32m✔ Validation passed\u001b[0m\n", - }, - }), - { - capturedOutput: - "Validating marketplace manifest: /.claude-plugin/marketplace.json\n" + - "✔ Validation passed", - normalizedArgv: [ - "", - "plugin", - "validate", - "--strict", - "", - ], - passed: true, - path: ".", - strict: true, - }, - ); - assert.deepEqual( - observedManifestValidation({ - arguments_: [ - "plugin", - "validate", - "--strict", - path.join(repository, ".claude-plugin", "plugin.json"), - ], - repository, - result: { - status: 0, - stderr: "✔ Validation passed\n", - stdout: - `Validating plugin manifest: ${repository}/.claude-plugin/plugin.json\n`, - }, - }), - { - capturedOutput: - "Validating plugin manifest: /.claude-plugin/plugin.json\n\n" + - "✔ Validation passed", - normalizedArgv: [ - "", - "plugin", - "validate", - "--strict", - "/.claude-plugin/plugin.json", - ], - passed: true, - path: ".claude-plugin/plugin.json", - strict: true, - }, - ); - assert.throws( - () => - observedManifestValidation({ - arguments_: ["plugin", "validate", "--strict", repository], - repository, - result: { status: 0, stderr: "", stdout: "Validation complete\n" }, - }), - /captured output:\nValidation complete/, - ); - assert.throws( - () => - observedManifestValidation({ - arguments_: ["plugin", "validate", "--strict", repository], - repository, - result: { - status: 0, - stderr: "", - stdout: "✔ Validation passed\n✔ Validation passed\n", - }, - }), - /exactly one `Validation passed` line/, - ); - assert.throws( - () => - observedManifestValidation({ - arguments_: ["plugin", "validate", repository], - repository, - result: { status: 0, stderr: "", stdout: "✔ Validation passed\n" }, - }), - /exact strict CLI invocation/, - ); - assert.throws( - () => - observedManifestValidation({ - arguments_: [ - "plugin", - "validate", - "--strict", - path.join(repository, ".claude-plugin", "plugin.json"), - ], - repository, - result: { - status: 0, - stderr: "", - stdout: - `Validating marketplace manifest: ${repository}/.claude-plugin/marketplace.json\n` + - "✔ Validation passed\n", - }, - }), - /captured validator target must match the normalized strict invocation/, - ); -}); - -test("observation evidence rejects embedded host absolute paths", () => { - assert.doesNotThrow(() => - assertNoObservationAbsolutePathLeaks({ - checkout: "/.claude-plugin/plugin.json", - excluded: "~/.claude.json", - homepage: "https://github.com/firstdraft/skills", - mediaType: "application/json", - repository: "firstdraft/skills", - ratio: "1/2", - evidenceMarkdown: - "# Evidence\n\n- Excluded: `~/.claude.json`\n\n" + - "[reference](https://code.claude.com/docs/en/plugins)", - }), - ); - for (const leakedPath of [ - "failure at /Users/alice/source/file", - "target=/private/var/tmp/result", - "failure at /data/build/output", - "target=/nix/store/tool", - "checkout: /custom/checkout/file", - "result=file:///arbitrary/host/output", - String.raw`failure at C:\Users\alice\source\file`, - String.raw`target=\\server\share\result`, - ]) { - assert.throws( - () => assertNoObservationAbsolutePathLeaks({ detail: leakedPath }), - /contains an absolute filesystem path/, - ); - } -}); - -test("manifest validation evidence rendering uses observed normalized fields", () => { - const rendered = renderManifestValidationEvidence("marketplace", { - capturedOutput: "Validating marketplace manifest: /manifest.json\n✔ Validation passed", - normalizedArgv: [ - "", - "plugin", - "validate", - "--strict", - "", - ], - path: ".", - }); - assert.equal( - rendered, - [ - "Repository-relative target: `.` (marketplace)", - "", - "Normalized child argv:", - "", - "```json", - '["","plugin","validate","--strict",""]', - "```", - "", - "Captured normalized output:", - "", - "```text", - "Validating marketplace manifest: /manifest.json", - "✔ Validation passed", - "```", - ].join("\n"), - ); -}); - -test("state-presence rendering has one explicit empty representation", () => { - assert.equal(renderStatePresenceNames([]), "(none)"); - assert.equal( - renderStatePresenceNames(["installedPlugins", "knownMarketplaces"]), - "installedPlugins,knownMarketplaces", - ); - assert.throws( - () => renderStatePresenceNames([""]), - /only nonempty strings/, - ); -}); - -test("committed comparison excludes run-local observation state", () => { - const packaging = { - schemaVersion: 2, - claudeCode: { version: "2.1.220" }, - manifestValidation: { marketplace: { passed: true } }, - installedPlugin: { name: "firstdraft", totalBytes: 100 }, - }; - const first = { - ...packaging, - observedOn: "2026-08-01", - realStateMonitor: { - absent: ["targetCache"], - present: ["installedPlugins"], - }, - checks: { realStateUnchanged: true }, - }; - const second = { - ...packaging, - observedOn: "2026-08-02", - realStateMonitor: { - absent: [], - present: ["knownMarketplaces", "targetCache"], - }, - checks: { realStateUnchanged: true, temporaryStateRemoved: true }, - }; - assert.deepEqual( - reviewedPackagingObservation(first), - reviewedPackagingObservation(second), - ); - assert.notDeepEqual( - reviewedPackagingObservation(first), - reviewedPackagingObservation({ - ...second, - installedPlugin: { ...second.installedPlugin, totalBytes: 101 }, - }), - ); -}); diff --git a/test/plugin-isolation.test.mjs b/test/plugin-isolation.test.mjs index 76b9f5a..b517a4d 100644 --- a/test/plugin-isolation.test.mjs +++ b/test/plugin-isolation.test.mjs @@ -1,57 +1,8 @@ import assert from "node:assert/strict"; import { spawnSync } from "node:child_process"; -import { - chmod, - mkdir, - mkdtemp, - readlink, - rm, - stat, - symlink, - writeFile, -} from "node:fs/promises"; -import { tmpdir } from "node:os"; -import path from "node:path"; import test from "node:test"; -import { - assertDefaultClaudeStateLocations, - changedStateEntries, - isolatedPluginEnvironment, - pathEntryExists, - pluginStateTargets, - resolvedStateTargetDiagnostics, - snapshotStateTargets, - stateTargetPresence, -} from "../script/plugin-isolation.mjs"; - -test("default real-state labels reject parent location overrides", () => { - assert.doesNotThrow(() => - assertDefaultClaudeStateLocations({ UNRELATED: "retained" }), - ); - - for (const environment of [ - { CLAUDE_CONFIG_DIR: "" }, - { CLAUDE_CONFIG_DIR: "sensitive-config-value" }, - { CLAUDE_CODE_PLUGIN_CACHE_DIR: "sensitive-cache-value" }, - { - CLAUDE_CONFIG_DIR: "sensitive-config-value", - CLAUDE_CODE_PLUGIN_CACHE_DIR: "sensitive-cache-value", - }, - ]) { - assert.throws( - () => assertDefaultClaudeStateLocations(environment), - (error) => { - assert.match( - error.message, - /requires CLAUDE_CONFIG_DIR and CLAUDE_CODE_PLUGIN_CACHE_DIR to be unset/, - ); - assert.doesNotMatch(error.message, /sensitive-(?:config|cache)-value/); - return true; - }, - ); - } -}); +import { isolatedPluginEnvironment } from "../script/plugin-isolation.mjs"; test("isolated plugin commands receive only the explicit environment", () => { const sentinelEnvironment = { @@ -146,151 +97,3 @@ test("isolated plugin commands receive only the explicit environment", () => { } } }); - -test("changed-state diagnostics include resolved absolute monitored paths", () => { - assert.deepEqual( - resolvedStateTargetDiagnostics( - ["targetCache", "knownMarketplaces"], - { - knownMarketplaces: { target: "relative/known_marketplaces.json" }, - targetCache: { target: "relative/cache" }, - }, - ), - [ - `targetCache=${path.resolve("relative/cache")}`, - `knownMarketplaces=${path.resolve("relative/known_marketplaces.json")}`, - ], - ); -}); - -test("path entry presence does not mistake dangling links for absence", async (t) => { - const temporaryDirectory = await mkdtemp( - path.join(tmpdir(), "firstdraft-plugin-path-entry-"), - ); - t.after(() => rm(temporaryDirectory, { force: true, recursive: true })); - const missing = path.join(temporaryDirectory, "missing"); - const dangling = path.join(temporaryDirectory, "dangling"); - const regular = path.join(temporaryDirectory, "regular"); - - assert.equal(pathEntryExists(missing), false); - await symlink(missing, dangling); - await writeFile(regular, "present"); - assert.equal(pathEntryExists(dangling), true); - assert.equal(pathEntryExists(regular), true); -}); - -test("target state snapshots detect nested changes and reject links", async (t) => { - const temporaryDirectory = await mkdtemp( - path.join(tmpdir(), "firstdraft-plugin-state-snapshot-"), - ); - t.after(() => rm(temporaryDirectory, { force: true, recursive: true })); - const configDirectory = path.join(temporaryDirectory, "config"); - const pluginsDirectory = path.join(temporaryDirectory, "plugins"); - const targets = pluginStateTargets({ - configDirectory, - pluginsDirectory, - marketplaceName: "firstdraft-skills", - pluginName: "firstdraft", - }); - const absentSnapshot = snapshotStateTargets(targets); - assert.deepEqual(stateTargetPresence(absentSnapshot), { - absent: [ - "credentials", - "installedPlugins", - "knownMarketplaces", - "pluginCatalog", - "settings", - "settingsLocal", - "targetCache", - "targetData", - "targetMarketplace", - ], - present: [], - }); - assert.equal(absentSnapshot.targetCache.present, false); - assert.match(absentSnapshot.targetCache.digest, /^[0-9a-f]{64}$/); - assert.equal(targets.targetCache.includeFileContents, true); - assert.equal(targets.targetData.includeFileContents, true); - assert.equal(targets.targetMarketplace.includeFileContents, true); - assert.equal(targets.installedPlugins.includeFileContents, true); - assert.equal(targets.knownMarketplaces.includeFileContents, true); - assert.equal(targets.pluginCatalog.includeFileContents, true); - for (const name of [ - "credentials", - "settings", - "settingsLocal", - ]) { - assert.equal(targets[name].includeFileContents, false); - } - - const nestedFile = path.join( - targets.targetCache.target, - "firstdraft", - "revision", - "nested.txt", - ); - await mkdir(path.dirname(nestedFile), { recursive: true }); - await writeFile(nestedFile, "alpha"); - const beforeNestedMutation = snapshotStateTargets(targets); - assert.deepEqual(stateTargetPresence(beforeNestedMutation), { - absent: [ - "credentials", - "installedPlugins", - "knownMarketplaces", - "pluginCatalog", - "settings", - "settingsLocal", - "targetData", - "targetMarketplace", - ], - present: ["targetCache"], - }); - await writeFile(nestedFile, "omega"); - assert.deepEqual(changedStateEntries(beforeNestedMutation, targets), [ - "targetCache", - ]); - const beforeMetadataMutation = snapshotStateTargets(targets); - const currentMode = (await stat(nestedFile)).mode & 0o777; - await chmod(nestedFile, currentMode === 0o600 ? 0o644 : 0o600); - assert.deepEqual(changedStateEntries(beforeMetadataMutation, targets), [ - "targetCache", - ]); - - const outsideDirectory = path.join(temporaryDirectory, "outside"); - const outsideFile = path.join(outsideDirectory, "outside.txt"); - await mkdir(outsideDirectory); - await writeFile(outsideFile, "first"); - const beforeLinkedTargetMutation = snapshotStateTargets(targets); - await symlink( - outsideDirectory, - path.join(targets.targetCache.target, "outside-link"), - ); - assert.throws( - () => changedStateEntries(beforeLinkedTargetMutation, targets), - /monitored state contains a symlink: .*outside-link/, - ); - - const otherOutsideDirectory = path.join(temporaryDirectory, "other-outside"); - await mkdir(otherOutsideDirectory); - const outsideLink = path.join(targets.targetCache.target, "outside-link"); - await rm(outsideLink); - await symlink(otherOutsideDirectory, outsideLink); - assert.equal(await readlink(outsideLink), otherOutsideDirectory); - assert.throws( - () => snapshotStateTargets(targets), - /monitored state contains a symlink: .*outside-link/, - ); - await rm(outsideLink); - await writeFile(outsideFile, "other"); - - const danglingTarget = path.join(temporaryDirectory, "missing-target"); - const danglingLink = path.join(temporaryDirectory, "dangling-link"); - await symlink(danglingTarget, danglingLink); - const danglingTargets = { - targetCache: { target: danglingLink, includeFileContents: true }, - }; - assert.throws( - () => snapshotStateTargets(danglingTargets), - /monitored state contains a symlink: .*dangling-link/, - ); -}); diff --git a/test/release-compatibility.test.mjs b/test/release-compatibility.test.mjs index 6fa8c4c..79b7b32 100644 --- a/test/release-compatibility.test.mjs +++ b/test/release-compatibility.test.mjs @@ -30,7 +30,9 @@ test("release compatibility matches the installable plugin manifest", async () = for (const name of ["diagnostics-and-recovery.md", "foundation-plan-020.md"]) { const reference = await readText(`skills/create-full-stack-app/references/${name}`); assert.equal( - reference.match(/\[the CLI contract configuration\]\(([^)]+)\)/)?.[1], + [...reference.matchAll(/\[[^\]]+\]\(([^)]+)\)/g)] + .map(([, destination]) => destination) + .find((destination) => destination.endsWith("/script/cli-contract/config.mjs")), cliConfigurationUrl, `${name}: bundled CLI provenance must use the plugin's release tag`, ); @@ -53,171 +55,6 @@ test("release compatibility matches the installable plugin manifest", async () = }); }); -test("current release docs route through structured identities", async () => { - const [ - compatibility, - marketplace, - readme, - releasing, - releaseHistory, - candidateSmokeEvidence, - syntheticAppearanceAnalysis, - ] = - await Promise.all([ - readJson("release/compatibility.json"), - readJson(".claude-plugin/marketplace.json"), - readText("README.md"), - readText("RELEASING.md"), - readText("evidence/release-history.md"), - readText("evidence/2026-08-30-claude-plugin-0.2.1-two-turn-smokes.md"), - readJson( - "evals/create-full-stack-app/fixtures/appearance-current-analysis.json", - ), - ]); - const publicPlugin = marketplace.plugins.find( - ({ name }) => name === "firstdraft", - ); - - assert.match(readme, /\(release\/compatibility\.json\)/); - assert.match(readme, /\(RELEASING\.md\)/); - assert( - releasing.includes( - `@firstdraft.com/claude-code@${compatibility.version}`, - ), - ); - assert.match(releasing, /Publish directly to npm `latest`/); - assert.match(releasing, /Reuse successful hosted CI for the exact release commit/); - assert.match(releasing, /attached `analysis\.gap_set_sha256`.*live GapSet digests include Project identity/s); - assert.match(releasing, /Never copy a fixture or a\s+prior Project's digest/); - const publicationGapSetDigest = - "19a65129ae87823366a9d83c99d82bcd9bd7af901312a7aed79253b33f662c85"; - const directGapSetDigest = - "23705bf4134a77c762d74ef819096a8861b48687dc5782cee47fbee11d6ce5e0"; - const observedSmokeGapSetDigests = [ - ...candidateSmokeEvidence.matchAll( - /\| Attached analysis \|[^\n]*?GapSet SHA-256 `([0-9a-f]{64})` \|/g, - ), - ].map((match) => match[1]); - for (const identity of [ - "b59565c83965f8f8436b16ac62660e89c9edd539", - "20967d6b84cd957b8052984da9bc1098ef1725d1", - "6ba0efb4fcb2dbf06d412ea8847593593fa832dc9cbcb419857a74c42e6cf74f", - "799a184cb2453ceadf5575f7b46ba975e084f192", - "06cf7e51148e69b6ca732cfdf9b86e939f1c3cdc", - "52cdb2900607023ad9a10456af35231369bd27c3bf32786297fe3d3eea017a3f", - publicationGapSetDigest, - directGapSetDigest, - "ddec63e329d10fc55b8308273478773c0658e0178a766e37d1e84c71c359bf62", - "17bc2c5e62935210fcdccb897108fd0148ee520df78d959b7bf87e27ac43d2e9", - "a35ba28b4a432309ebd42f03a1c50fdd24ca98310b1024565ad90642f82d0ee9", - ]) { - assert(candidateSmokeEvidence.includes(identity)); - } - assert.deepEqual(observedSmokeGapSetDigests, [ - publicationGapSetDigest, - directGapSetDigest, - ]); - assert.notEqual(observedSmokeGapSetDigests[0], observedSmokeGapSetDigests[1]); - for (const digest of observedSmokeGapSetDigests) { - assert.notEqual(digest, syntheticAppearanceAnalysis.analysis.gap_set_sha256); - } - assert.match( - candidateSmokeEvidence, - /count before approval `0`[\s\S]*?`plan compile` count after approval `1`[\s\S]*?no real GitHub side effect[\s\S]*?count before approval `0`[\s\S]*?`plan compile --output \.\/application` count after approval `1`[\s\S]*?198 files[\s\S]*?no `\.git`[\s\S]*?Publication count `0`/, - ); - assert.match( - candidateSmokeEvidence, - /does not publish the plugin or prove registry\/public[\s\S]*?installation[\s\S]*?protected tag[\s\S]*?npm dist-tag[\s\S]*?public catalog[\s\S]*?deploy the Service[\s\S]*?real GitHub Publication[\s\S]*?template-and-Codespace journey/, - ); - assert.match( - candidateSmokeEvidence, - /controlled Service revision is a descendant of the pinned current-truth Service revision[\s\S]*?cc72dad5b26b887f3f21496b568b80678ceac47f[\s\S]*?does not repin the packaged[\s\S]*?current-authority source/, - ); - assert.equal(publicPlugin.version, publicPlugin.source.version); - assert.match(releasing, /release\/compatibility\.json.*owns the candidate/s); - assert.match(releasing, /One user approval may cover the complete coordinated release/); - assert.match(releasing, /without asking again at\s+every step/); - assert.match(releasing, /ambiguous tag push or npm publication.*read-only before attempting another mutation/s); - assert.match(releasing, /same-singleton Publication replay.*never applies to an ambiguous Plan push or direct Compilation start/s); - assert.match(releasing, /Documentation-only or workflow-only changes.*not another product\s+journey/s); - - assert.doesNotMatch(releasing, /^## (?:Current 0\.1\.1|Completed)/m); - assert.doesNotMatch(releasing, /firstdraft-package-first\.XXXXXX/); - assert.match( - releaseHistory, - /archived on 2026-08-13[\s\S]*?snapshot is point-in-time evidence, not current authority/, - ); - assert.doesNotMatch(releaseHistory, /current source-candidate version/); -}); - -test("release smokes use the local path only when needed", async () => { - const [releasing, evalIndex] = await Promise.all([ - readText("RELEASING.md"), - readText("evals/README.md"), - ]); - const smoke = markdownSection(releasing, "2. Smoke the local path only when useful").replace(/\s+/g, " "); - assert.match(smoke, /Compilation runs on the service; the output and runtime stay local/); - assert.match(smoke, /firstdraft plan compile.*--output \./); - assert.match(smoke, /Verify materialization.*boot Rails.*open one primary page/); - assert.match(smoke, /no GitHub Publication, Codespace, multi-session interview, dual-client install, native build, or Revyl session is required/); - assert.match(smoke, /existing approval of the candidate and gaps is sufficient/i); - assert.match(evalIndex, /focused behavioral regressions.*not a mandatory release sequence/s); -}); - -test("dated release-state observation retains its recorded facts", async () => { - const observation = await readJson( - "evidence/2026-08-13-release-state.json", - ); - - assert.deepEqual(observation, { - format: "firstdraft.skills-release-state-observation/1", - observed_on: "2026-08-13", - source_candidate: { - version: "0.1.2", - integration_commit_at_observation: - "f4855c5bf0d44800690a64dc9d874106e5d9e2ab", - tarball_sha256: - "e89a14b7a28ec5b6384038cec106f31c7496f076344726b02b3a674b344755f5", - published_to_npm: false, - protected_tag: null, - selected_by_catalog: false, - }, - public_plugin: { - package: "@firstdraft.com/claude-code", - version: "0.1.1", - source_commit: "263326a47a502b56af7780093988c6b860b2d5d2", - protected_tag: "claude-v0.1.1", - catalog_promotion_commit: - "ff2f0863f85e1f95194c8e3fbe9986b56efb0ad1", - npm_next: "0.1.1", - npm_latest: "0.1.1", - }, - public_cli: { - package: "@firstdraft.com/cli", - version: "0.1.0", - source_commit: "d37d8b6775a0b97ce10bd651485bd308fed1dda2", - protected_tag: "v0.1.0", - npm_next: "0.1.0", - npm_latest: "0.1.0", - }, - compatibility: { - api_contract: [">= 0.2.0", "< 0.3.0"], - foundation_plan_formats: [ - "firstdraft.foundation-plan.sketch/0.19", - ], - }, - outstanding: [ - "Exact-byte fresh-agent semantic-approval qualification for candidate 0.1.2", - "Protected tag and npm publication of candidate 0.1.2 under next", - "Public catalog promotion and post-merge two-command installation for candidate 0.1.2", - "Separate npm latest promotion for candidate 0.1.2", - "Existing-install update or auto-refresh behavior", - "Authenticated template-and-Codespace journey", - "Full v14 qualification", - ], - }); -}); - test("release compatibility rejects shape and manifest drift", async () => { const documents = await releaseDocuments(); const withExtraKey = structuredClone(documents); @@ -351,492 +188,6 @@ test("semantic-version precedence orders ordinary and historical versions", () = assert.equal(compareSemanticVersions("0.1.0+one", "0.1.0+two"), 0); }); -test("archived release chronology retains exact observed facts", async () => { - const [ - releasing, - agents, - readme, - directPackageEvidence, - pluginReleaseEvidence, - directPackageReleaseEvidence, - pluginPatchReleaseEvidence, - directPackagePatchEvidence, - publicPatchInstallEvidence, - stablePromotionEvidence, - discoverySmokeEvidence, - ] = await Promise.all([ - readText("evidence/release-history.md"), - readText("AGENTS.md"), - readText("evidence/repository-history.md"), - readText("evidence/2026-08-07-direct-package-alpha3-check.md"), - readText("evidence/2026-08-09-claude-plugin-0.1.0-release.md"), - readText("evidence/2026-08-09-direct-package-0.1.0-check.md"), - readText("evidence/2026-08-12-claude-plugin-0.1.1-release.md"), - readText("evidence/2026-08-12-direct-package-0.1.1-check.md"), - readText( - "evidence/2026-08-12-public-claude-code-plugin-0.1.1-install.md", - ), - readText("evidence/2026-08-12-stable-npm-promotion.md"), - readText("evidence/2026-08-10-staging-movie-catalog-discovery-smoke.md"), - ]); - assert.match( - releasing, - /Plugin 0\.1\.1 corrected[\s\S]*?At the 2026-08-12 stable-tag promotion recorded below[\s\S]*?selected by both npm `next` and `latest`[\s\S]*?public catalog promotion[\s\S]*?ff2f0863f85e1f95194c8e3fbe9986b56efb0ad1[\s\S]*?naming that immutable version[\s\S]*?marketplace merge itself published[\s\S]*?no package bytes and moved no npm dist-tag[\s\S]*?later stable-tag promotion was a separate explicitly approved[\s\S]*?registry mutation/, - ); - assert.match( - releasing, - /Alpha\.4\s+and alpha\.5 were assembled as source candidates and abandoned before catalog promotion/, - ); - assert.match( - releasing, - /2026-08-06 alpha\.3 public-install observation proves the prior alpha\.3 package and bundled alpha\.2 CLI only; no[\s\S]*?public install of plugin 0\.1\.0 was observed/, - ); - assert.match( - releasing, - /marketplace-promotion change was merged[\s\S]*?e0212cad0a89a8b0e38678e371389085f6ddc254/, - ); - assert.match( - releasing, - /ordinary[\s\S]*?releases exist under protected tags[\s\S]*?claude-v0\.1\.0[\s\S]*?claude-v0\.1\.1[\s\S]*?v0\.1\.0[\s\S]*?npm `next`[\s\S]*?`latest` both selected plugin 0\.1\.1 at that 2026-08-12 observation point[\s\S]*?npm `next` and `latest` both selected[\s\S]*?CLI 0\.1\.0 then[\s\S]*?historical public catalog at[\s\S]*?e0212cad0a89a8b0e38678e371389085f6ddc254[\s\S]*?selected plugin 0\.1\.0[\s\S]*?public catalog[\s\S]*?ff2f0863f85e1f95194c8e3fbe9986b56efb0ad1[\s\S]*?selecting[\s\S]*?plugin 0\.1\.1/, - ); - assertTextOrder(releasing, [ - "## Completed 0.1.1 patch flow", - "1. Clean, non-shallow checkouts resolved exact Skills", - "2. The complete repository and CLI-contract checks passed", - "3. Skills source integrated at `263326a47a502b56af7780093988c6b860b2d5d2`", - "4. A human explicitly authorized", - "5. The protected tag and npm publication completed under `next`", - "6. The separate marketplace change moved exact package selection from 0.1.0 to published 0.1.1", - "ff2f0863f85e1f95194c8e3fbe9986b56efb0ad1", - "it was not a circular pre-merge gate", - "7. After the exact 0.1.1 package, catalog, and public-install checks described above", - "plugin 0.1.1 and CLI 0.1.0 under both `next` and `latest`", - "No package bytes were republished", - "preceding package, catalog, public-install, and CLI checks were the required release-specific qualification", - "dist-tag move and reconciliation were the mutation and completion check, not the gate", - "release-specific qualification did not claim or require full v14", - "For later releases, never make a post-merge public-install observation a pre-merge gate for enabling the catalog promotion that must precede it", - "the public install is a separate observation of the exact merged catalog", - "## Completed 0.1.0 candidate flow", - ]); - assert.match( - readme, - /ordinary releases exist under protected tags[\s\S]*?claude-v0\.1\.1[\s\S]*?At the 2026-08-12 stable-tag observation point[\s\S]*?npm `next` and `latest` both[\s\S]*?selected plugin 0\.1\.1[\s\S]*?npm `next` and `latest` both selected CLI 0\.1\.0[\s\S]*?historical public catalog at[\s\S]*?e0212cad0a89a8b0e38678e371389085f6ddc254[\s\S]*?selected plugin 0\.1\.0[\s\S]*?public catalog[\s\S]*?ff2f0863f85e1f95194c8e3fbe9986b56efb0ad1[\s\S]*?selecting exact published plugin 0\.1\.1[\s\S]*?Published[\s\S]*?plugin 0\.1\.1 corrects the four Publication\/Publish negatives in canonical source[\s\S]*?retains[\s\S]*?ambiguous "packed reviewed CLI" attribution[\s\S]*?change immutable package bytes[\s\S]*?new[\s\S]*?SemVer[\s\S]*?recorded deterministic digest[\s\S]*?separate qualification/, - ); - assert.match( - readme, - /After the required release-specific qualification,[\s\S]*?moving `latest` is a separate explicitly[\s\S]*?approved mutation[\s\S]*?stable[\s\S]*?plugin release is not complete until the public catalog selects the[\s\S]*?exact qualified package[\s\S]*?both `next` and `latest` selecting that same version/, - ); - assert.match( - pluginReleaseEvidence, - /source commit `b3e53a240aaf79a776538e9b1410689d8a4e79ee`[\s\S]*?tag object is `ddbc7456647a62bf2dc13b2b897cadbf4e486344`[\s\S]*?publication run[\s\S]*?`31321014564`[\s\S]*?tarball SHA-256 `02fad6cd2207f3d2ab7598f0aa67825520ebc5b807294e0c241774ee3ac6a89d`[\s\S]*?triggering ref is a protected tag[\s\S]*?one\s+verified registry signature and one verified attestation[\s\S]*?`next` dist-tag named `0\.1\.0`[\s\S]*?`latest` remained `0\.1\.0-alpha\.3`[\s\S]*?public Claude marketplace[\s\S]*?`0\.1\.0-alpha\.3`/, - ); - assert.match( - pluginReleaseEvidence, - /This establishes only the package, tag, workflow, provenance-presence, digest, and dist-tag identities[\s\S]*?did\s+not install the package through Claude Code[\s\S]*?call staging[\s\S]*?qualify Movie Catalog[\s\S]*?verify the two-command public installation path/, - ); - assert.match( - directPackageReleaseEvidence, - /Node\s+v24\.18\.0[\s\S]*?npm 11\.16\.0[\s\S]*?Claude Code 2\.1\.224[\s\S]*?one added package/, - ); - assert.match( - directPackageReleaseEvidence, - /@firstdraft\.com\/claude-code@0\.1\.0[\s\S]*?firstdraft@0\.1\.0[\s\S]*?skills\/create-full-stack-app\/SKILL\.md[\s\S]*?claude plugin validate --strict [\s\S]*?session-scoped `firstdraft@inline`[\s\S]*?bundled `firstdraft --version` returned exact `0\.1\.0` with empty stderr/, - ); - assert.match( - directPackageReleaseEvidence, - /one verified registry signature and one verified attestation[\s\S]*?did\s+not authenticate or call a model[\s\S]*?configure or call First Draft[\s\S]*?exercise staging[\s\S]*?GitHub Publication[\s\S]*?change the public marketplace catalog[\s\S]*?two-command marketplace installation path/, - ); - for (const exactIdentity of [ - "263326a47a502b56af7780093988c6b860b2d5d2", - "91b16537373ee567a2741783ce692cd9b2daadd3", - "31631531058", - "sha512-imSYruwBnSgCTttXzBjHIpPQaBV7lo9T1JlthBDrngzYUM/rDw8n68lpTOFrdBxnrn2ZTzlwYk9kHc6YpbE8xw==", - "520772f0b1acba6ae015198ba8fd36f38bbf3f85", - "800e9ebd63843c7c680810979c35ade37de31d5e203e89a75a09f80d3399d656", - ]) { - assert(pluginPatchReleaseEvidence.includes(exactIdentity)); - } - assert.match( - pluginPatchReleaseEvidence, - /registry package contains 33 files[\s\S]*?one verified registry[\s\S]*?signature and one verified attestation[\s\S]*?npm `next` names `0\.1\.1`[\s\S]*?`latest` remains `0\.1\.0-alpha\.3`[\s\S]*?pre-promotion observation[\s\S]*?selected plugin 0\.1\.0/, - ); - assert.match( - pluginPatchReleaseEvidence, - /establishes only the package, tag, successful workflow, digest, provenance-presence, file count, and dist-tag[\s\S]*?did not install through the public marketplace commands[\s\S]*?configure[\s\S]*?First Draft[\s\S]*?GitHub Publication[\s\S]*?move[\s\S]*?`latest`[\s\S]*?merge the catalog promotion/, - ); - assert.match( - directPackagePatchEvidence, - /@firstdraft\.com\/claude-code@0\.1\.1[\s\S]*?Node[\s\S]*?v24\.18\.0[\s\S]*?npm 11\.16\.0[\s\S]*?Claude Code 2\.1\.228[\s\S]*?firstdraft@0\.1\.1[\s\S]*?skills\/create-full-stack-app\/SKILL\.md[\s\S]*?claude plugin validate --strict [\s\S]*?firstdraft@inline[\s\S]*?no `userConfig` prompt[\s\S]*?exact version `0\.1\.0`[\s\S]*?top-level `--help`[\s\S]*?both `sh` and `zsh`[\s\S]*?repository-owned `bin\/firstdraft` wrapper/, - ); - assert.match( - directPackagePatchEvidence, - /set no First Draft credentials and made no First Draft service call[\s\S]*?did not authenticate or call a model[\s\S]*?GitHub Publication[\s\S]*?change the\s+public marketplace catalog[\s\S]*?two-command public marketplace installation path/, - ); - assert.match( - publicPatchInstallEvidence, - /Claude Code 2\.1\.228[\s\S]*?claude plugin marketplace add firstdraft\/skills[\s\S]*?claude plugin install firstdraft@firstdraft-skills[\s\S]*?Node v24\.18\.0[\s\S]*?npm 11\.16\.0[\s\S]*?Git 2\.54\.0[\s\S]*?fresh isolated home, Claude configuration,[\s\S]*?plugin cache, XDG, and npm state[\s\S]*?exited 1[\s\S]*?`loggedIn=false`[\s\S]*?`authMethod=none`[\s\S]*?HTTPS GitHub source[\s\S]*?After both commands[\s\S]*?marketplace clone's HEAD was exact catalog-promotion revision[\s\S]*?ff2f0863f85e1f95194c8e3fbe9986b56efb0ad1[\s\S]*?working tree remained clean/, - ); - assert.match( - publicPatchInstallEvidence, - /catalog-selection JSON named npm source `@firstdraft\.com\/claude-code@0\.1\.1`[\s\S]*?enabled user-scope plugin `firstdraft@firstdraft-skills` at exact version `0\.1\.1`[\s\S]*?installed package manifest named `@firstdraft\.com\/claude-code` at exact version `0\.1\.1`[\s\S]*?strict validation passed for both the fetched marketplace and installed plugin[\s\S]*?skills\/create-full-stack-app\/SKILL\.md[\s\S]*?\.\/skills\/create-full-stack-app[\s\S]*?enabled session-scope plugin `firstdraft@inline` at exact version `0\.1\.1`[\s\S]*?`installPath` realpath equaled the installed plugin root's realpath[\s\S]*?no `userConfig`[\s\S]*?exact `0\.1\.0` to stdout[\s\S]*?nothing to stderr/, - ); - assert.match( - publicPatchInstallEvidence, - /no First Draft credential or state files[\s\S]*?did not authenticate or[\s\S]*?call a model[\s\S]*?configure a First Draft token[\s\S]*?call a First Draft service[\s\S]*?author or push a Plan[\s\S]*?AnalysisRun[\s\S]*?Compilation[\s\S]*?Publication[\s\S]*?mutate GitHub[\s\S]*?fork a template[\s\S]*?create a Codespace[\s\S]*?closes only the fresh public two-command marketplace installation path for exact plugin 0\.1\.1[\s\S]*?does not establish[\s\S]*?existing-install update or auto-refresh behavior[\s\S]*?authenticated installed-Skill journey[\s\S]*?full v14 qualification/, - ); - assert.match( - stablePromotionEvidence, - /pre-mutation check completed at `2026-08-12T21:23:03Z`[\s\S]*?two dist-tag queries below[\s\S]*?plugin `next` at 0\.1\.1 and `latest` at historical alpha\.3[\s\S]*?CLI `next` at 0\.1\.0 and `latest` at historical alpha\.2[\s\S]*?plugin package, catalog, and public-install checks were complete[\s\S]*?CLI 0\.1\.0 had its dated release evidence[\s\S]*?exact bundled CLI exercised by the plugin checks[\s\S]*?release-specific checks and explicit user[\s\S]*?approval,[\s\S]*?changed only npm dist-tags[\s\S]*?Read-only reconciliation[\s\S]*?`2026-08-12T21:24:42Z`[\s\S]*?npm view @firstdraft\.com\/claude-code dist-tags --json[\s\S]*?npm view @firstdraft\.com\/claude-code version[\s\S]*?npm view @firstdraft\.com\/cli dist-tags --json[\s\S]*?npm view @firstdraft\.com\/cli version[\s\S]*?metadata queries, which were not fresh package installations[\s\S]*?@firstdraft\.com\/claude-code[\s\S]*?`next` and `latest` both resolving to `0\.1\.1`[\s\S]*?sha512-imSYruwBnSgCTttXzBjHIpPQaBV7lo9T1JlthBDrngzYUM\/rDw8n68lpTOFrdBxnrn2ZTzlwYk9kHc6YpbE8xw==[\s\S]*?520772f0b1acba6ae015198ba8fd36f38bbf3f85[\s\S]*?@firstdraft\.com\/cli[\s\S]*?`next` and `latest` both resolving to `0\.1\.0`[\s\S]*?versionless npm metadata resolution selecting plugin 0\.1\.1 and CLI 0\.1\.0[\s\S]*?8d74ddfe968804e6d2d7b4b5b8ed5c37d2697d18[\s\S]*?ff2f0863f85e1f95194c8e3fbe9986b56efb0ad1[\s\S]*?catalog version and exact[\s\S]*?npm source version were both `0\.1\.1`/, - ); - assert.match( - stablePromotionEvidence, - /No package version or tarball was published, replaced, removed, or deprecated[\s\S]*?No protected tag, catalog source,[\s\S]*?First Draft service, deployment, repository, or Codespace changed[\s\S]*?historical alpha\.3 and alpha\.2[\s\S]*?time-bounded observations remain unchanged[\s\S]*?closes the release-specific stable package-default and catalog identity only[\s\S]*?does not establish[\s\S]*?existing-install update or[\s\S]*?auto-refresh behavior[\s\S]*?authenticated installed-Skill journey[\s\S]*?full v14 qualification/, - ); - assert.doesNotMatch( - publicPatchInstallEvidence, - /(?:discover|enumerat)(?:ed|ion)?[^\n]*Skill/i, - ); - assert.match( - readme, - /0\.1\.1 public-install observation[\s\S]*?install enabled user-scope plugin 0\.1\.1[\s\S]*?strictly validate the[\s\S]*?marketplace and plugin[\s\S]*?verify the canonical Skill file[\s\S]*?manifest declaration[\s\S]*?run bundled CLI 0\.1\.0 from fresh unauthenticated state/, - ); - assert.doesNotMatch( - readme, - /(?:discover|enumerat)\w*[^\n]*canonical Skill/i, - ); - assert.match( - releasing, - /public-install observation[\s\S]*?fetched clone clean at exact catalog-promotion revision[\s\S]*?ff2f0863f85e1f95194c8e3fbe9986b56efb0ad1[\s\S]*?catalog selection and installed package manifest naming exact npm[\s\S]*?@firstdraft\.com\/claude-code@0\.1\.1[\s\S]*?canonical Skill file and declaration[\s\S]*?inline `installPath`[\s\S]*?realpath equality/, - ); - for (const source of [ - pluginPatchReleaseEvidence, - directPackagePatchEvidence, - publicPatchInstallEvidence, - stablePromotionEvidence, - ]) { - assert(!source.includes(repository)); - assert.doesNotMatch(source, /(?:\/Users\/|\/home\/|[A-Za-z]:\\)/); - assert.doesNotMatch(source, /\/(?:private\/)?tmp\//); - assert.doesNotMatch(source, /\.firstdraft\/state\.json/); - assert.doesNotMatch( - source, - /(?:authorization|bearer|api[_-]?key|access[_-]?token|refresh[_-]?token|client[_-]?secret|BEGIN [A-Z ]+PRIVATE KEY)/i, - ); - } - assert.match( - discoverySmokeEvidence, - /Status: passed the bounded discovery-promotion gate[\s\S]*?does not claim a completed v14 qualification/, - ); - assert.match( - discoverySmokeEvidence, - /4007fc5ef0734e2fc3e3e59714919025bd73d621[\s\S]*?b3e53a240aaf79a776538e9b1410689d8a4e79ee[\s\S]*?02fad6cd2207f3d2ab7598f0aa67825520ebc5b807294e0c241774ee3ac6a89d[\s\S]*?d37d8b6775a0b97ce10bd651485bd308fed1dda2/, - ); - assert.match( - discoverySmokeEvidence, - /Claude Code 2\.1\.222[\s\S]*?1,572-byte Foundation Plan[\s\S]*?831f5d960416c7c3f01f0a75b417f5d4330abf68062527b52ce8528f0b7ef37a[\s\S]*?exactly one AnalysisRun[\s\S]*?one Compilation[\s\S]*?one Publication[\s\S]*?job-d9smqin10e5c73a6m72g[\s\S]*?github\.name_conflict[\s\S]*?created the next repository[\s\S]*?published the artifact/, - ); - assert.match( - discoverySmokeEvidence, - /No GitHub PAT was created or used[\s\S]*?independent clone, ref, commit, tree, blob, mode, size, or byte-for-byte artifact comparison[\s\S]*?generated-repository credential-category scan[\s\S]*?singleton replay[\s\S]*?full v14 qualification/, - ); - assert.match( - discoverySmokeEvidence, - /did not begin with a template fork or Codespace[\s\S]*?did not install plugin 0\.1\.0 through the public[\s\S]*?marketplace catalog[\s\S]*?immediate\s+post-promotion check/, - ); - assert(!discoverySmokeEvidence.includes(repository)); - assert(!discoverySmokeEvidence.includes("demostudent27")); - assert.doesNotMatch( - discoverySmokeEvidence, - /(?:\/Users\/|\/home\/|[A-Za-z]:\\)/, - ); - assert.doesNotMatch( - discoverySmokeEvidence, - /\.firstdraft\/state\.json/, - ); - assert.doesNotMatch( - discoverySmokeEvidence, - /(?:authorization|bearer|api[_-]?key|access[_-]?token|refresh[_-]?token|client[_-]?secret|BEGIN [A-Z ]+PRIVATE KEY)/i, - ); - assert.match( - readme, - /That live smoke remains a 0\.1\.0 gate[\s\S]*?Plugin 0\.1\.1 retains CLI 0\.1\.0 and service API 0\.2 compatibility[\s\S]*?requires no[\s\S]*?service mutation[\s\S]*?requires no new pre-merge live smoke[\s\S]*?exact 0\.1\.1[\s\S]*?direct-package check[\s\S]*?exact promotion-head Node 24\.18\.0 CI[\s\S]*?Neither pre-merge gate proves the post-merge public[\s\S]*?installation[\s\S]*?separate isolated observation now establishes that path for exact plugin 0\.1\.1 only/, - ); - assert.match( - releasing, - /staging Movie Catalog discovery smoke[\s\S]*?plugin 0\.1\.0[\s\S]*?human explicitly selected[\s\S]*?bounded PAT-less smoke as the pre-catalog[\s\S]*?does\s+not constitute full v14 qualification/, - ); - assert.match(agents, /A marketplace merge changes the live catalog/); - assert.match(agents, /normal passing PR checks/); - assert.match(agents, /Do not bypass the protected environment or CI/); - assert(releasing.includes("claude-v$package_version")); - assert.match( - releasing, - /Once an npm version, protected release tag, or catalog version exists,[\s\S]*?maps\s+forever to exactly one package tarball/, - ); - assert.match( - releasing, - /pre-1\.0 candidates use ordinary `0\.MINOR\.PATCH` versions[\s\S]*?minor component for a breaking compatibility-line change[\s\S]*?patch component for an otherwise\s+backward-compatible change/, - ); - assert.match( - releasing, - /Do not add compatibility aliases[\s\S]*?npm `next`[\s\S]*?package-publication and qualification channel[\s\S]*?has no[\s\S]*?SemVer meaning[\s\S]*?After the required release-specific[\s\S]*?qualification passes,[\s\S]*?moving `latest` is a separate explicitly approved mutation[\s\S]*?non-catalog package,[\s\S]*?define[\s\S]*?qualification before the move[\s\S]*?catalog selection is not its gate[\s\S]*?Do not call a stable[\s\S]*?plugin release[\s\S]*?complete until the exact package is selected by the public catalog[\s\S]*?both `next`[\s\S]*?and `latest` selecting that same version/, - ); - assert.match( - releasing, - /publication workflow may[\s\S]*?advance `next`, but it must leave `latest` at its pre-publication identity[\s\S]*?After the exact package passes its[\s\S]*?required release-specific qualification and the public catalog selects it,[\s\S]*?later separately approved action may[\s\S]*?move `latest`[\s\S]*?manual registry mutation is outside the reviewer-gated tag publication workflow[\s\S]*?explicit[\s\S]*?approval and single-operator serialization[\s\S]*?Reconcile the package, both dist-tags,[\s\S]*?exact catalog source read-only before calling the stable plugin release complete[\s\S]*?never authorizes new package bytes, a service deployment, or a[\s\S]*?catalog edit/, - ); - assert.match( - releasing, - /retained preparation evidence does not establish that a formal 0\.1\.0 maintenance-window notice, start, or end[\s\S]*?Publishing, service deployment, catalog promotion, and any future lane-scoped window each require new,[\s\S]*?explicit authorization/, - ); - assert.match( - releasing, - /service API contract `>= 0\.2\.0` and `< 0\.3\.0`[\s\S]*?activation and promotion steps have occurred; they are not pending instructions[\s\S]*?one coordinated rollout whose service-activation\s+phase occupies a maintenance[\s\S]*?window[\s\S]*?package bytes may be published first[\s\S]*?unsupported for First Draft operations[\s\S]*?against the earlier public API contract/, - ); - assert.match( - releasing, - /service API contract 0\.2 responses are incompatible with the alpha\.2 CLI bundled in\s+public plugin alpha\.3[\s\S]*?Only an existing alpha\.3 installation that resolves its declared default endpoint to shared\s+staging\/API 0\.2 is incompatible[\s\S]*?option-default injection[\s\S]*?auto-refresh or update behavior[\s\S]*?remain unproved/, - ); - assert.match( - releasing, - /For a[\s\S]*?future breaking transition, this source does not\s+approve that interruption[\s\S]*?human must explicitly approve the[\s\S]*?package-first rollout and the later brief maintenance window/, - ); - assert.match( - releasing, - /do not\s+deploy it while plugin 0\.1\.0 remains only an unpublished candidate/, - ); - assert.match( - releasing, - /Do not assume catalog promotion updated an existing installation[\s\S]*?retained record proves neither Claude Code\s+auto-refresh\/update behavior nor whether any installed alpha\.3 copy resolved its declared option default to shared\s+staging\/API 0\.2/, - ); - assert.match( - releasing, - /dated release observation[\s\S]*?CLI 0\.1\.0 was under `next` while[\s\S]*?`latest` remained alpha\.2 at its publication-time boundary/, - ); - assert.match( - releasing, - /before staging[\s\S]*?activated API 0\.2,[\s\S]*?exact-`next` install was incompatible and unsupported for Plan, Compile, or[\s\S]*?any other First Draft API operation/, - ); - const directPackageCheck = releasing.match( - /historical instruction was: Run this strict no-service check after npm reconciliation[\s\S]*?```sh\n([\s\S]*?)\n ```/, - )?.[1]; - assert(directPackageCheck, "missing direct-next no-service package check"); - assert.match( - directPackageCheck, - /if ! check_root="\$\(mktemp -d \/tmp\/firstdraft-package-first\.XXXXXX\)"[\s\S]*?FAILED: could not allocate package-first evidence root[\s\S]*?test -z "\$check_root"[\s\S]*?cd -P "\$check_root"[\s\S]*?package_spec='@firstdraft\.com\/claude-code@0\.1\.0'[\s\S]*?expect_plugin_version='0\.1\.0'[\s\S]*?expect_cli_version='0\.1\.0'/, - ); - assert.match( - directPackageCheck, - /export HOME=[\s\S]*?export TMPDIR=[\s\S]*?export CLAUDE_CONFIG_DIR=[\s\S]*?export CLAUDE_CODE_PLUGIN_CACHE_DIR=[\s\S]*?export XDG_CONFIG_HOME=[\s\S]*?export XDG_RUNTIME_DIR=[\s\S]*?export DISABLE_AUTOUPDATER=1[\s\S]*?export CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=1[\s\S]*?export CLAUDE_CODE_DISABLE_OFFICIAL_MARKETPLACE_AUTOINSTALL=1[\s\S]*?unset FIRSTDRAFT_API_TOKEN FIRSTDRAFT_API_URL FIRSTDRAFT_BASE_URL/, - ); - assert.match( - directPackageCheck, - /: > "\$check_root\/user-npmrc"[\s\S]*?: > "\$check_root\/global-npmrc"[\s\S]*?npm install[\s\S]*?--prefix[\s\S]*?--cache[\s\S]*?--userconfig "\$check_root\/user-npmrc"[\s\S]*?--globalconfig "\$check_root\/global-npmrc"[\s\S]*?--registry=https:\/\/registry\.npmjs\.org\/[\s\S]*?--@firstdraft\.com:registry=https:\/\/registry\.npmjs\.org\/[\s\S]*?--no-audit[\s\S]*?--no-fund[\s\S]*?--ignore-scripts[\s\S]*?"\$package_spec"/, - ); - assert.match( - directPackageCheck, - /packageManifest\.name === "@firstdraft\.com\/claude-code"[\s\S]*?pluginManifest\.name === "firstdraft"[\s\S]*?pluginManifest\.skills\[0\] === "\.\/skills\/create-full-stack-app"[\s\S]*?SKILL\.md/, - ); - assert.match( - directPackageCheck, - /claude plugin validate --strict "\$plugin_root"[\s\S]*?claude --plugin-dir "\$plugin_root" plugin list --json/, - ); - assert.match( - directPackageCheck, - /id === "firstdraft@inline"[\s\S]*?plugin\.scope === "session"[\s\S]*?plugin\.enabled === true[\s\S]*?fs\.realpathSync\(plugin\.installPath\) === fs\.realpathSync\(process\.env\.PLUGIN_ROOT\)/, - ); - assert.match( - directPackageCheck, - /"\$plugin_root\/bin\/firstdraft" --version[\s\S]*?cli-version\.txt[\s\S]*?tr -d '\\r\\n'[\s\S]*?= "\$expect_cli_version"/, - ); - assert.match( - directPackageCheck, - /\)\s+status=\$\?[\s\S]*?if test "\$status" -eq 0[\s\S]*?PASS: package-first evidence retained at[\s\S]*?FAILED: inspect[\s\S]*?do not open the maintenance window[\s\S]*?exit "\$status"/, - ); - assert.equal( - [...directPackageCheck.matchAll(/"\$plugin_root\/bin\/firstdraft"/g)].length, - 1, - ); - assert.doesNotMatch( - directPackageCheck, - /claude\s+plugin\s+(?:details|install|marketplace)/i, - ); - assert.match( - releasing, - /Do not add `claude plugin details` to this no-service variant[\s\S]*?Claude `count_tokens`[\s\S]*?temporary root until its evidence is accepted[\s\S]*?record its cleanup separately/, - ); - assert.match( - releasing, - /dated direct-package observation[\s\S]*?same hardened procedure succeeding against public alpha\.3[\s\S]*?bundled CLI alpha\.2/, - ); - assert.match( - directPackageEvidence, - /Claude Code 2\.1\.224[\s\S]*?Node v24\.3\.0[\s\S]*?npm 11\.4\.2[\s\S]*?public registry/, - ); - assert.match( - directPackageEvidence, - /`loggedIn=false`[\s\S]*?`authMethod=none`[\s\S]*?`claude plugin validate --strict ` completing successfully[\s\S]*?ID `firstdraft@inline`[\s\S]*?session scope[\s\S]*?enabled state[\s\S]*?canonically equal/, - ); - assert.match( - directPackageEvidence, - /package name and version `@firstdraft\.com\/claude-code@0\.1\.0-alpha\.3`[\s\S]*?bundled `firstdraft --version` returning exact `0\.1\.0-alpha\.2`/, - ); - assert.match( - directPackageEvidence, - /fail-closed rehearsal[\s\S]*?deliberately missing local package[\s\S]*?exited nonzero[\s\S]*?`FAILED`[\s\S]*?without a success line[\s\S]*?stopped after npm/, - ); - assert.match( - directPackageEvidence, - /does not establish unpublished\s+plugin 0\.1\.0[\s\S]*?move an npm dist-tag[\s\S]*?register or change a marketplace[\s\S]*?call a model[\s\S]*?call staging[\s\S]*?mutate a First Draft service/, - ); - for (const source of [directPackageEvidence, directPackageReleaseEvidence]) { - assert.doesNotMatch( - source, - /\/(?:private\/)?tmp\/firstdraft-package-first(?:-[0-9]{8})?\.[A-Za-z0-9]+/, - ); - } - assert.doesNotMatch( - releasing, - /plugin cannot be published before service activation/i, - ); - assertTextOrder(releasing, [ - "The externally observed package publication and reconciliation, API 0.2 activation, bounded discovery smoke, and catalog-promotion mutations below occurred from 2026-08-07 through 2026-08-10", - "must not be replayed", - "1. The operator reconciled the already-published CLI 0.1.0 registry package", - "2. The operator pushed the protected Skills publication tag", - "3. In isolated Claude state, the operator performed the direct-`next` package check", - "4. The exact API 0.2 service revision was deployed first to the staging web role", - "then to the staging worker role", - "selected 0.1.0 Movie Catalog discovery smoke", - "separate direct-`next` no-service check from step 3", - "separately billed Standard Render Compilation One-Off", - "persistent Standard Solid Queue worker ran Publication coordination and all three recorded Publication attempts", - "A GitHub PAT, independent clone or byte verification, generated-repository credential scan, and singleton replay", - "5. After both roles and the selected 0.1.0 discovery smoke were verified, the marketplace-promotion change was merged", - "At that historical revision", - "The later public catalog promotion merged at exact catalog-promotion revision", - "selecting exact published plugin 0.1.1", - "### Completed post-merge 0.1.1 public-install observation", - "6. After the public catalog promotion merged at exact catalog-promotion revision", - "claude plugin marketplace add firstdraft/skills", - "claude plugin install firstdraft@firstdraft-skills", - "catalog-selection JSON named npm package `@firstdraft.com/claude-code@0.1.1`", - "installed package manifest named that exact package and version", - "canonical `skills/create-full-stack-app/SKILL.md` file", - "`installPath` realpath equal to the installed plugin root", - "This closes only the fresh two-command installation path for exact selected plugin 0.1.1", - "### Outstanding authenticated product journey", - "7. The authenticated template-and-Codespace product journey remains outstanding", - "obtain fresh explicit authorization for exactly one serialized qualification invocation", - "Merely reading this step authorizes no marketplace registration, plugin installation, template repository, Codespace, token onboarding, Compile, Publication, destination-repository mutation, retry, or cleanup", - "Pin the exact merged Skills SHA in the updated Drawing Board", - "use that template to create a repository and a fresh Codespace", - "run `claude`", - "make a plain-English application request", - "expected fresh private GitHub repository", - "The full v14 qualification remains a separate, stricter boundary", - "The selected live discovery smoke remains a 0.1.0 gate", - "same CLI 0.1.0 and service API 0.2 compatibility line", - "does not require a new pre-merge live smoke", - "0.1.1 public-package direct-install check", - "exact promotion head's Node 24.18.0 CI", - "Neither pre-merge gate proves the post-merge", - "separate dated observation now proves that path for exact plugin 0.1.1 only", - ]); - assert.match( - releasing, - /workflow published the[\s\S]*?qualified `@firstdraft\.com\/claude-code@0\.1\.0` tarball with npm provenance under `next`[\s\S]*?pre-promotion point, npm `latest` and[\s\S]*?the public marketplace catalog still named alpha\.3/, - ); - assert.match( - readme, - /staging web and worker both reported[\s\S]*?4007fc5ef0734e2fc3e3e59714919025bd73d621[\s\S]*?selected gate for promoting new catalog installs[\s\S]*?Public plugin alpha\.3 bundles CLI alpha\.2 and declares shared staging as its default endpoint[\s\S]*?only an[\s\S]*?installation that resolves that declared default to shared staging\/API 0\.2 is incompatible[\s\S]*?option-default injection[\s\S]*?auto-refresh or update behavior[\s\S]*?remain unproved[\s\S]*?fresh public[\s\S]*?marketplace install of exact plugin 0\.1\.1 is now observed[\s\S]*?existing-install update behavior and authenticated[\s\S]*?template-and-Codespace discovery remain outstanding/, - ); - assert.match( - readme, - /On 2026-08-06, those versionless commands resolved the then-current catalog and installed plugin alpha\.3 with bundled[\s\S]*?CLI alpha\.2[\s\S]*?On 2026-08-12, a distinct observation ran the same versionless commands against exact catalog-promotion[\s\S]*?ff2f0863f85e1f95194c8e3fbe9986b56efb0ad1[\s\S]*?installed exact plugin 0\.1\.1 with bundled CLI 0\.1\.0[\s\S]*?fresh unauthenticated state[\s\S]*?proves neither an existing-install update nor an authenticated First[\s\S]*?Draft journey/, - ); - assert.doesNotMatch( - readme, - /exact plugin 0\.1\.0 with bundled CLI 0\.1\.0 in a fresh unauthenticated check/, - ); - assert.match( - releasing, - /Only an existing alpha\.3 installation that resolves its declared default endpoint to shared[\s\S]*?staging\/API 0\.2 is incompatible[\s\S]*?For an installation verified as affected,[\s\S]*?separately verified Claude Code update procedure[\s\S]*?affected-user follow-up/, - ); - assert.match( - releasing, - /exact 0\.1\.0 promotion\s+head's Node 24\.18\.0 CI[\s\S]*?release-order rehearsal,[\s\S]*?passed without an administrative[\s\S]*?bypass[\s\S]*?durable rule remains:[\s\S]*?require the exact promotion-head job and never use an administrative bypass/, - ); - assert.match( - releasing, - /At that historical revision,[\s\S]*?resolved a catalog that selected exact plugin 0\.1\.0[\s\S]*?later[\s\S]*?public catalog promotion merged at exact catalog-promotion revision[\s\S]*?ff2f0863f85e1f95194c8e3fbe9986b56efb0ad1[\s\S]*?selecting exact published plugin 0\.1\.1/, - ); - assert.match( - releasing, - /Completed post-merge 0\.1\.1 public-install observation[\s\S]*?fetched marketplace clone remained clean at that exact HEAD[\s\S]*?catalog-selection JSON named[\s\S]*?@firstdraft\.com\/claude-code@0\.1\.1[\s\S]*?installed package manifest named that exact package and[\s\S]*?version[\s\S]*?canonical `skills\/create-full-stack-app\/SKILL\.md` file and declared it[\s\S]*?`installPath` realpath equal to the[\s\S]*?installed plugin root[\s\S]*?bundled CLI reported exact version 0\.1\.0 with empty stderr[\s\S]*?Outstanding authenticated product journey/, - ); - assert.match( - releasing, - /selected live discovery smoke remains a 0\.1\.0 gate[\s\S]*?same CLI 0\.1\.0 and service API 0\.2[\s\S]*?requires no service mutation[\s\S]*?does not require a new pre-merge live[\s\S]*?smoke[\s\S]*?0\.1\.1 public-package direct-install check[\s\S]*?exact[\s\S]*?Node 24\.18\.0 CI[\s\S]*?Neither pre-merge gate proves the post-merge[\s\S]*?two-command public installation path[\s\S]*?separate dated observation now proves that path for exact plugin 0\.1\.1[\s\S]*?does not replace any authenticated product-journey or full-v14 gate/, - ); - assert.match( - releasing, - /For the next candidate,[\s\S]*?then-current supported Claude Code CLI[\s\S]*?local validation, packed install, registry publication, or catalog source change never proves the two-command[\s\S]*?Only a separately recorded post-merge installation through the merged public catalog[\s\S]*?proves that path for the exact selected version/, - ); - assert.match( - releasing, - /selected 0\.1\.0 Movie Catalog discovery smoke[\s\S]*?Claude-authored exact Plan[\s\S]*?valid analysis[\s\S]*?successful\s+Compilation[\s\S]*?successful OAuth\/App-backed publication[\s\S]*?separately billed Standard Render Compilation One-Off[\s\S]*?persistent Standard[\s\S]*?Solid Queue worker ran Publication coordination and all three recorded Publication attempts[\s\S]*?A GitHub PAT,[\s\S]*?independent clone or byte verification[\s\S]*?singleton replay[\s\S]*?not part of this user-selected discovery gate[\s\S]*?must\s+not claim a full v14 qualification/, - ); - assert.match( - releasing, - /pre-promotion rollback recipe retained for the historical API-activation phase[\s\S]*?restore the then-exact API[\s\S]*?0\.1 rollback revision[\s\S]*?leave the catalog at alpha\.3[\s\S]*?dated recipe is historical and[\s\S]*?non-actionable[\s\S]*?must not be applied to any later qualification[\s\S]*?or to current staging/, - ); - assert.match( - releasing, - /For any later qualification after this rollout[\s\S]*?explicit approval for its own lane-scoped window[\s\S]*?notice, start, affected-user disposition, rollback, and completion criteria[\s\S]*?stop all other and new operator-controlled Compile and Publication invocations in the qualification lane[\s\S]*?Permit only[\s\S]*?single separately authorized qualification invocation[\s\S]*?serialized from start through retained outcome[\s\S]*?Public-plugin traffic to shared staging may continue as unattributed capacity activity[\s\S]*?current service runbook's drift and stop rules[\s\S]*?exact web, worker, queue, catalog, and installation state[\s\S]*?read-only[\s\S]*?fresh explicit authorization naming the exact rollback revisions and catalog action[\s\S]*?completion evidence before declaring the window closed[\s\S]*?Do not presume that API 0\.1 or plugin[\s\S]*?alpha\.3 is the current rollback target[\s\S]*?later authorized authenticated path in step 7 fails[\s\S]*?do not infer that the[\s\S]*?failure requires a catalog mutation[\s\S]*?catalog repoint[\s\S]*?one reviewable source[\s\S]*?change[\s\S]*?\.claude-plugin\/marketplace\.json[\s\S]*?test\/repository\.test\.mjs[\s\S]*?test\/release-compatibility\.test\.mjs[\s\S]*?README\.md[\s\S]*?RELEASING\.md[\s\S]*?active version and endpoint resolution[\s\S]*?verified-affected installation[\s\S]*?explicitly accept its continuing outage[\s\S]*?Selecting a prior immutable catalog package is not reusing that SemVer for different bytes/, - ); - assert.match( - releasing, - /authorized recovery includes a catalog repoint[\s\S]*?recovery approval must separately name the exact immutable package[\s\S]*?npm `next` and `latest`[\s\S]*?apply that approved dist-tag disposition[\s\S]*?reconcile both tags with the exact catalog source read-only[\s\S]*?Do not assume a catalog or dist-tag change updated[\s\S]*?existing installations[\s\S]*?catalog, dist-tags, service roles, and supported clients[\s\S]*?agree/, - ); - assert.doesNotMatch( - releasing, - /operator started the announced maintenance window|notified affected users, and stopped new Compile/, - ); - assert.match( - releasing, - /Publishing changed bytes for an existing npm, protected-tag, or catalog\s+release identity is forbidden[\s\S]*?new SemVer[\s\S]*?Repointing the catalog to a prior immutable package for\s+rollback is allowed[\s\S]*?unpublished and unpromoted candidate may instead be revised/, - ); - - assert.match(agents, /Never reuse a published npm version, protected release tag, or marketplace version for different package bytes/); - assert.match(agents, /Before 1\.0 use a minor bump for a breaking\s+compatibility change, otherwise a patch/); - assert.match(agents, /read-only reconciliation of ambiguous external effects/); - assert.match(agents, /require release authorization/); - assert.match(agents, /one approved coordinated sequence covers its named steps without repeated prompts/); - assert.match(agents, /Publish new packages directly to npm `latest`/); - -}); - -function assertTextOrder(source, fragments) { - const normalizedSource = source.replace(/\s+/g, " "); - let previousIndex = -1; - for (const fragment of fragments) { - const normalizedFragment = fragment.replace(/\s+/g, " "); - const index = normalizedSource.indexOf(normalizedFragment, previousIndex + 1); - assert.ok(index >= 0, `missing or out-of-order release step: ${fragment}`); - previousIndex = index; - } -} - -function markdownSection(source, heading) { - const marker = `## ${heading}\n`; - const start = source.indexOf(marker); - assert.notEqual(start, -1, `missing Markdown section: ${heading}`); - const next = source.indexOf("\n## ", start + marker.length); - return source.slice(start + marker.length, next === -1 ? undefined : next); -} - async function releaseDocuments() { const [ compatibility, diff --git a/test/repository.test.mjs b/test/repository.test.mjs index 2ae3640..3b9fec7 100644 --- a/test/repository.test.mjs +++ b/test/repository.test.mjs @@ -16,64 +16,24 @@ import { forbiddenCheckoutRootClaudePluginComponentPaths, forbiddenClaudePluginPathSegments, } from "../script/claude-plugin-boundaries.mjs"; -import { - assertNoObservationAbsolutePathLeaks, - observedFileBytes, - observedFileTreeSha256, - renderManifestValidationEvidence, - renderStatePresenceNames, -} from "../script/claude-plugin-observation.mjs"; import { analyzerRelease as foundationPlanAnalyzerRelease, compilationTarget as foundationPlanTarget, - cliRevision as cliContractBaseline, - cliRuntimeSha256 as cliContractRuntimeDigest, compilerRelease as foundationPlanCompilerRelease, foundationPlanFormat, - rootOutputRecovery, - safeGithubReasonCodes, } from "../script/cli-contract/config.mjs"; const repository = path.dirname(path.dirname(fileURLToPath(import.meta.url))); const skillsDirectory = path.join(repository, "skills"); const evalsDirectory = path.join(repository, "evals"); const claudePluginDirectory = path.join(repository, ".claude-plugin"); -const claudePluginEvidence = path.join( - repository, - "evidence", - "2026-08-04-claude-code-plugin-install-smoke.md", -); -const claudePluginObservation = path.join( - repository, - "evidence", - "claude-code-plugin-install-observation.json", -); -const freshClaudeEvaluationEvidence = path.join( - repository, - "evidence", - "2026-08-04-fresh-claude-code-evaluations.md", -); -const homeInventoryOpeningResponse = path.join( - repository, - "evidence", - "2026-08-04-home-inventory-opening-response.txt", -); -const movieCatalogModelObservation = path.join( - repository, - "evidence", - "2026-08-04-movie-catalog-model-rehearsal.json", -); const claudePluginName = "firstdraft"; const claudeMarketplaceName = "firstdraft-skills"; -const portableSkillName = "create-full-stack-app"; const historicalFoundationPlanFormat = "firstdraft.foundation-plan.sketch/0.19"; const historicalFoundationPlanTarget = { id: "rails", profile: "rails-sketch/2026-08", }; -const historicalFoundationPlanAnalyzerRelease = "foundation-plan-rails/application-2026-08"; -const historicalFoundationPlanCompilerRelease = - "foundation-plan-rails/compiler-application-2026-08"; // These exact inputs remain linked by dated qualification receipts. const historicalPlanFixtures = new Set([ "appearance-issues.foundation-plan.json", @@ -85,95 +45,13 @@ const reviewedFixtureAnalyzerRelease = const reviewedFixtureCompilerRelease = "foundation-plan-rails/compiler-application-2026-08-28-reviewed-realization"; const currentFoundationPlanServiceBaseline = "ee38cafcff43d70fdb9f28626f25ebaecb257b0c"; -const generatedUiEvidenceBaseline = "00e92e397dfbb5bc4dfda69f0d1cf48c5e7beff8"; -const currentFoundationIosCoreRevision = "7365ba0bf7ea5e6c8e8223d24e54cf685b067950"; -const currentFoundationAndroidCoreRevision = "6a07e79197f2acbcaab9d15eb4dc61aa9ca5c94e"; const foundationPlanSchemaDigest = "5576ec5e10d108f0a2d0f9fa336249324642f092e4444f6c11e4ab738f3fa58b"; -const foundationPlanServerBaseline = - "35ad070beb36c66dc6480f36b33767caaed160a9"; const currentFoundationPlanSchemaBaseline = currentFoundationPlanServiceBaseline; -const priorNativeEvidenceBaseline = "9ff77985c821501f0174aec5da6192871395cd6b"; -const priorAndroidEvidenceBaseline = "89a2d6866f9448f4e75b58cac26f61c52daaa0b0"; -const previousSkillsCurrentTruthBaseline = - "160d33a5a7d9f9b2282729ecfd3b2e24a1123143"; -const previousSkillsCurrentTruthTree = - "6f3db12c017e884d8b14c66f7d82e64229ec2073"; -const previousFoundationPlanAnalyzerRelease = - "foundation-plan-rails/application-2026-08-27-codespace-ssh-qualification"; -const previousFoundationPlanCompilerRelease = - "foundation-plan-rails/compiler-application-2026-08-27-codespace-ssh-qualification"; -const currentCompilerServiceBaseline = - "6002be2685542fedf515879f940b97ad73b1a469"; -const discoverySmokeServiceBaseline = - "4007fc5ef0734e2fc3e3e59714919025bd73d621"; -const catalogPromotionBaseline = - "e0212cad0a89a8b0e38678e371389085f6ddc254"; -const pluginPatchCatalogPromotionBaseline = - "ff2f0863f85e1f95194c8e3fbe9986b56efb0ad1"; -const pluginReleaseBaseline = - "b3e53a240aaf79a776538e9b1410689d8a4e79ee"; -const compilationEvidenceCliBaseline = - "121272cd592055354d09a4fe90e55c3ca002770c"; -const compilationEvidenceCliRuntimeDigest = - "205e664df0ed9c7e63651a1c2c01e749a04d8879fe7f62cc4c1e13b66dce738d"; -const previousPublicCliContractBaseline = - "d38ef3e54a6476b3a91f22a17fe7bd47aa6d6d68"; -const previousPublicCliContractRuntimeDigest = - "0dec2ca75ce7862208fd093933d0954cbe9cbebc58dbc8fe6f589a1bee493098"; -const previousCliContractBaseline = - "e53eb38d7e8254e6ba1e660b38c5d32d0314be17"; -const previousCliContractRuntimeDigest = - "0983106d7c1054137d70dccb1091eeadd8272ffcca1f7bba1bde9c8028452fad"; -const historicalCliContractBaseline = - "f55edffc9e88924f9a4c95f41c4d0bc9b72422f8"; -const historicalCliContractRuntimeDigest = - "9e5a4bd0f16f49ab2e17c04f7defc59366f8fa073f772b310d8f684177890eab"; -const compilationProvenanceServiceBaseline = - "5811bb3013cf25072db74355597f60d85be3c05b"; -const productJourneySmokeBaseline = - "8ebfc2ed82a610e63f47eb985c23ab7e634fe94e"; -const historicalPluginInstallEvidenceBaseline = - "3777ae515bd366e7d6e55df0c2add3a7f12a9d12"; -const freshModelServiceBaseline = - "3a029a8b425addbbba4f56d9197878cc002752f4"; -const freshModelServiceTree = - "076415a4b1e34cc458a85186e1e335503eb30612"; -const freshModelPluginBaseline = - "b5c3897b240bfa3a9117d1a564d8e6b7d783e993"; -const marketplacePluginSourceBaseline = - "8ffbd9688f39118ddeeb48a3da7e5bc309b7be5e"; -const freshModelPluginRuntimeDigest = - "a5c3bfe0dd8d5396a692c4204c670e10cbc4b996883f76025d9e8a6586becc7b"; -const freshModelClaudeExecutableDigest = - "7a181f36ed0fc4fbac6cee4ecf2b615eff93d8b434221fff5d7c878dc5ebf380"; -const freshModelPublicationTree = - "5815d094e204f8b3928ff5b5467ef85e2551d109"; -const freshModelPublicationCommit = - "37cc23d7cf7a1448fb7dfd4be8aee27c6e389ead"; -const preparedCliPackage = "@firstdraft.com/cli@0.4.0"; -const previousPreparedCliPackage = "@firstdraft.com/cli@0.2.1"; const prettyJsonSha256 = (value) => createHash("sha256") .update(`${JSON.stringify(value, null, 2)}\n`) .digest("hex"); -const foundationIosCoreRevision = - "aa2ac902fa52abab51a4502953b7b962f949a21d"; -const foundationIosCoreArchiveDigest = - "0807e76cf02296af27d4eb1aae68e298beef162a7daa8a3da55d83e88ab6d748"; -const freshAgentEvidenceBaseline = - "16b056a6f55eb92cb6e5a6e02abd58e84b47abd5"; -const freshAgentSkillBaseline = - "5cad5acec23a983e6421d2d37420a74de63b47fb"; -const planPushErrorCodes = [ - "authentication_required", - "invalid_arguments", - "invalid_configuration", - "local_input_unreadable", - "request_outcome_unknown", - "server_rejected", - "local_state_not_saved", -]; const supportedScalarFieldTypes = [ "boolean", "date", @@ -186,11 +64,6 @@ const supportedScalarFieldTypes = [ "time_zone", "url", ]; -const supportedFieldTypes = [ - ...supportedScalarFieldTypes, - "enum", - "state_machine", -].sort(); const supportedFieldProperties = [ "subject_uuid", "key", @@ -205,40 +78,8 @@ const supportedFieldProperties = [ "encrypted_at_rest", "redact_from_logs", ]; -const fieldCapabilityProperties = [ - "required", - "default", - "notes", - "immutable", - "comparison", - "normalizations", - "encrypted_at_rest", - "redact_from_logs", -]; -const supportedReferenceProperties = [ - "subject_uuid", - "key", - "name", - "targets", - "required", - "one_to_one", - "on_referenced_deleted", - "default", - "immutable", - "realization", -]; -const supportedPredicateProperties = [ - "subject_uuid", - "key", - "name", - "expression", -]; test("documentation roles are routed and retrieval-sized", async () => { - const documentationMap = await readFile( - path.join(repository, "docs", "README.md"), - "utf8", - ); const readme = await readFile(path.join(repository, "README.md"), "utf8"); const releasing = await readFile( path.join(repository, "RELEASING.md"), @@ -271,21 +112,6 @@ test("documentation roles are routed and retrieval-sized", async () => { ]) { assert.ok(readme.includes(`(${route})`), `README.md must route to ${route}`); } - assert.match( - documentationMap, - /## Authority by question[\s\S]*?## Routes by task[\s\S]*?## Documentation roles/, - ); - assert.match( - documentationMap, - /Current operator procedure[\s\S]*?No completed chronology or historical shell transcripts/, - ); - assert.match( - releasing, - /evidence\/release-history\.md/, - ); - assert.doesNotMatch(releasing, /^## (?:Current 0\.1\.1|Completed)/m); - assert.doesNotMatch(releasing, /firstdraft-package-first\.XXXXXX/); - const evidenceFiles = (await readdir(path.join(repository, "evidence"), { withFileTypes: true, })) @@ -342,545 +168,6 @@ test("documentation roles are routed and retrieval-sized", async () => { } }); -test("revision pins remain exhaustive across coordination surfaces", async () => { - const readme = await readFile( - path.join(repository, "evidence", "repository-history.md"), - "utf8", - ); - assertRevisionTokens(readme, [ - foundationPlanServerBaseline, - compilationEvidenceCliBaseline, - previousPublicCliContractBaseline, - compilationProvenanceServiceBaseline, - productJourneySmokeBaseline, - freshModelServiceBaseline, - freshModelPluginBaseline, - foundationIosCoreRevision, - freshAgentEvidenceBaseline, - freshAgentSkillBaseline, - freshAgentSkillBaseline.slice(0, 7), - currentCompilerServiceBaseline, - discoverySmokeServiceBaseline, - historicalCliContractBaseline, - catalogPromotionBaseline, - pluginPatchCatalogPromotionBaseline, - pluginReleaseBaseline, - previousSkillsCurrentTruthBaseline, - previousSkillsCurrentTruthTree, - ]); - - const skillDirectory = path.join(skillsDirectory, "create-full-stack-app"); - const referencesDirectory = path.join( - skillDirectory, - "references", - ); - const referenceNames = (await readdir(referencesDirectory)) - .filter((file) => file.endsWith(".md")) - .sort(); - const references = await Promise.all( - referenceNames.map((file) => - readFile(path.join(referencesDirectory, file), "utf8"), - ), - ); - assertRevisionTokens(references.join("\n"), [ - currentFoundationPlanSchemaBaseline, - currentFoundationPlanServiceBaseline, - generatedUiEvidenceBaseline, - priorNativeEvidenceBaseline, - priorAndroidEvidenceBaseline, - currentFoundationIosCoreRevision, - currentFoundationAndroidCoreRevision, - catalogPromotionBaseline, - ]); - const skillSource = await readFile( - path.join(skillDirectory, "SKILL.md"), - "utf8", - ); - assertRevisionTokens(skillSource, []); - - const foundationPlanReference = await readFile( - path.join(referencesDirectory, "foundation-plan-020.md"), - "utf8", - ); - const diagnosticsReference = await readFile( - path.join(referencesDirectory, "diagnostics-and-recovery.md"), - "utf8", - ); - assert(readme.includes(`\`${previousPreparedCliPackage}\``)); - assert.match( - readme, - /CLI contract check requires the exact revision, runtime digest, and package version owned by\s+`script\/cli-contract\/config\.mjs`[\s\S]*?Follow the current checkout and reconciliation procedure in\s+`RELEASING\.md`/, - ); - assert.doesNotMatch( - readme, - /CLI contract check requires a checkout at the exact reviewed revision/, - ); - for (const source of [foundationPlanReference, diagnosticsReference]) { - const packageParagraph = source.split(/\n\s*\n/).find((paragraph) => - paragraph.includes(`\`${preparedCliPackage}\``), - ); - assert(packageParagraph, "the reference must identify the prepared CLI package"); - assert.doesNotMatch(packageParagraph, /\bis unpublished\b|^unpublished /m); - assert.match(source, /do not prove plugin(?:\/| or )catalog\s+publication/); - } - - const workflow = ( - await readFile(path.join(repository, ".github", "workflows", "ci.yml"), "utf8") - ).replace(/^.*uses:\s+\S+@[0-9a-f]{40}.*$/gm, ""); - assertRevisionTokens(workflow, []); - const contractConfig = await readFile( - path.join(repository, "script", "cli-contract", "config.mjs"), - "utf8", - ); - assertRevisionTokens(contractConfig, [cliContractBaseline]); - assert(contractConfig.includes(cliContractRuntimeDigest)); - assert(contractConfig.includes(foundationPlanCompilerRelease)); - assert(contractConfig.includes(foundationPlanAnalyzerRelease)); - assert(contractConfig.includes(foundationPlanTarget.profile)); - assertRevisionTokens( - await readFile(path.join(repository, "test", "repository.test.mjs"), "utf8"), - [ - foundationPlanServerBaseline, - currentFoundationPlanSchemaBaseline, - currentFoundationPlanServiceBaseline, - generatedUiEvidenceBaseline, - priorNativeEvidenceBaseline, - priorAndroidEvidenceBaseline, - currentFoundationIosCoreRevision, - currentFoundationAndroidCoreRevision, - previousSkillsCurrentTruthBaseline, - previousSkillsCurrentTruthTree, - currentCompilerServiceBaseline, - discoverySmokeServiceBaseline, - compilationEvidenceCliBaseline, - previousPublicCliContractBaseline, - previousCliContractBaseline, - historicalCliContractBaseline, - compilationProvenanceServiceBaseline, - productJourneySmokeBaseline, - historicalPluginInstallEvidenceBaseline, - freshModelServiceBaseline, - freshModelServiceTree, - freshModelPluginBaseline, - marketplacePluginSourceBaseline, - freshModelPublicationTree, - freshModelPublicationCommit, - foundationIosCoreRevision, - freshAgentEvidenceBaseline, - freshAgentSkillBaseline, - catalogPromotionBaseline, - pluginPatchCatalogPromotionBaseline, - pluginReleaseBaseline, - ], - ); - for (const relativePath of [ - ["evals", "create-full-stack-app", "cases.json"], - [ - "evals", - "create-full-stack-app", - "references", - "candidate-interview-protocol.md", - ], - ["script", "check"], - ["skills", "create-full-stack-app", "agents", "openai.yaml"], - ["test", "interview-evaluation-foundation.test.mjs"], - ["script", "support", "create-full-stack-app-evaluation.mjs"], - ]) { - assertRevisionTokens( - await readFile(path.join(repository, ...relativePath), "utf8"), - [], - ); - } -}); - -test("historical plugin receipts stay separate from current availability", async () => { - const candidateSkillPath = path.join( - skillsDirectory, - portableSkillName, - "SKILL.md", - ); - const candidateModelingGuidePath = path.join( - skillsDirectory, - portableSkillName, - "references", - "modeling-guide.md", - ); - const candidateFoundationPlanReferencePath = path.join( - skillsDirectory, - portableSkillName, - "references", - "foundation-plan-020.md", - ); - const [ - readme, - releasing, - candidateSkill, - candidateModelingGuide, - candidateFoundationPlanReference, - ] = await Promise.all([ - readFile( - path.join(repository, "evidence", "repository-history.md"), - "utf8", - ), - readFile( - path.join(repository, "evidence", "release-history.md"), - "utf8", - ), - readFile(candidateSkillPath, "utf8"), - readFile(candidateModelingGuidePath, "utf8"), - readFile(candidateFoundationPlanReferencePath, "utf8"), - ]); - const publishedSkill = gitBlobAtRevision( - pluginReleaseBaseline, - "skills/create-full-stack-app/SKILL.md", - ).toString("utf8"); - const publishedModelingGuide = gitBlobAtRevision( - pluginReleaseBaseline, - "skills/create-full-stack-app/references/modeling-guide.md", - ).toString("utf8"); - const publishedFoundationPlanReference = gitBlobAtRevision( - pluginReleaseBaseline, - "skills/create-full-stack-app/references/foundation-plan-019.md", - ).toString("utf8"); - - for (const source of [publishedSkill, publishedModelingGuide]) { - assert.match(source, /live [Pp]ublication remains unproved/); - } - for (const source of [candidateSkill, candidateModelingGuide]) { - assert.doesNotMatch(source, /live [Pp]ublication remains unproved/); - } - assert.match(candidateSkill, /CLI 0\.4\.0/); - assert.doesNotMatch(candidateSkill, /This source candidate is unreleased/); - assert.match( - candidateSkill, - /compatibility does not establish catalog selection/, - ); - assert.doesNotMatch(candidateModelingGuide, /dated staging (?:discovery|observation)/); - assert.match( - publishedFoundationPlanReference, - /no Plan GET or pull operation[\s\S]*?proven live Publish path/, - ); - assert.match( - publishedFoundationPlanReference, - /controlled product-journey smoke[\s\S]*?8ebfc2ed82a610e63f47eb985c23ab7e634fe94e[\s\S]*?packed reviewed[\s\S]*?CLI/, - ); - assert.doesNotMatch(candidateFoundationPlanReference, /proven live Publish path/); - assert.match( - candidateFoundationPlanReference, - new RegExp(`Current design and machine authority[\\s\\S]*?${currentFoundationPlanServiceBaseline}[\\s\\S]*?Implementation and observation evidence[\\s\\S]*?Older controlled smokes[\\s\\S]*?historical receipts[\\s\\S]*?must not be used to narrow or widen the current profile`), - ); - assert(!candidateFoundationPlanReference.includes(historicalCliContractBaseline)); - assert.match( - readme, - /immutable plugin 0\.1\.0 package[\s\S]*?b3e53a240aaf79a776538e9b1410689d8a4e79ee[\s\S]*?packaged `SKILL\.md` retains two pre-smoke negatives[\s\S]*?live GitHub publication remains outside the evidence boundary[\s\S]*?later Scaffold guidance[\s\S]*?publication remains unproved[\s\S]*?packaged modeling guide repeats the live-Publication negative[\s\S]*?`references\/foundation-plan-019\.md`[\s\S]*?no proven live Publish[\s\S]*?path[\s\S]*?old harness "reviewed"[\s\S]*?exact revision[\s\S]*?f55edffc9e88924f9a4c95f41c4d0bc9b72422f8[\s\S]*?`0\.1\.0-alpha\.2`[\s\S]*?four[\s\S]*?Publication\/Publish negatives do not disable `firstdraft plan compile`[\s\S]*?Treat all five[\s\S]*?Published[\s\S]*?plugin 0\.1\.1 corrects the four Publication\/Publish negatives in canonical source[\s\S]*?retains[\s\S]*?ambiguous "packed reviewed CLI" attribution[\s\S]*?acknowledged[\s\S]*?published-package limitation[\s\S]*?change immutable package bytes[\s\S]*?new[\s\S]*?SemVer[\s\S]*?recorded deterministic digest[\s\S]*?separate qualification[\s\S]*?full v14 qualification gaps/, - ); - assert.match( - releasing, - /immutable plugin 0\.1\.0 package[\s\S]*?b3e53a240aaf79a776538e9b1410689d8a4e79ee[\s\S]*?packaged `SKILL\.md` retains[\s\S]*?two pre-smoke negatives[\s\S]*?live GitHub publication outside the evidence boundary[\s\S]*?later[\s\S]*?Scaffold guidance says live publication remains unproved[\s\S]*?packaged modeling guide repeats the live-Publication[\s\S]*?negative[\s\S]*?`references\/foundation-plan-019\.md`[\s\S]*?no proven live Publish path[\s\S]*?old harness "reviewed"[\s\S]*?exact revision[\s\S]*?f55edffc9e88924f9a4c95f41c4d0bc9b72422f8[\s\S]*?`0\.1\.0-alpha\.2`[\s\S]*?four Publication\/Publish negatives do not[\s\S]*?disable `firstdraft plan compile`[\s\S]*?Treat all five[\s\S]*?Published plugin 0\.1\.1 corrects the four Publication\/Publish negatives in canonical source[\s\S]*?retains[\s\S]*?ambiguous "packed reviewed CLI" attribution[\s\S]*?acknowledged[\s\S]*?published-package limitation[\s\S]*?change immutable package bytes[\s\S]*?new[\s\S]*?SemVer[\s\S]*?recorded deterministic digest[\s\S]*?separate qualification[\s\S]*?full v14 gaps/, - ); -}); - -test("fresh Claude Code evidence is exact and bounded", async () => { - const evidence = await readFile(freshClaudeEvaluationEvidence, "utf8"); - const observationSource = await readFile( - movieCatalogModelObservation, - "utf8", - ); - const homeResponse = await readFile(homeInventoryOpeningResponse, "utf8"); - const observation = JSON.parse(observationSource); - assertRevisionTokens(evidence, [ - historicalCliContractBaseline, - freshModelServiceBaseline, - freshModelPluginBaseline, - ]); - assertRevisionTokens(homeResponse, []); - assertRevisionTokens(observationSource, [ - historicalCliContractBaseline, - freshModelServiceBaseline, - freshModelServiceTree, - freshModelPluginBaseline, - freshModelPublicationTree, - freshModelPublicationCommit, - ]); - assertNoObservationAbsolutePathLeaks({ - evidenceMarkdown: evidence, - homeResponse, - modelObservation: observation, - }); - - for (const source of [evidence, homeResponse, observationSource]) { - assert(!source.includes(repository)); - assert.doesNotMatch(source, /(?:\/Users\/|\/home\/|[A-Za-z]:\\)/); - assert.doesNotMatch(source, /\.firstdraft\/state\.json/); - assert.doesNotMatch( - source, - /(?:authorization|bearer|api[_-]?key|access[_-]?token|refresh[_-]?token|client[_-]?secret|BEGIN [A-Z ]+PRIVATE KEY)/i, - ); - } - assert.equal( - createHash("sha256").update(homeResponse).digest("hex"), - "ac9c699f8fee9848a5c5ab83a3383d08a9406f70aa41b1991d4ab036c2b8563e", - ); - assert.match( - homeResponse, - /What is one record\?[\s\S]*?one unique object per record[\s\S]*?one quantity-bearing record[\s\S]*?both as two distinct kinds/, - ); - - for (const value of [ - freshModelServiceBaseline, - freshModelPluginBaseline, - freshModelPluginRuntimeDigest, - freshModelClaudeExecutableDigest, - historicalCliContractBaseline, - historicalCliContractRuntimeDigest, - ]) { - assert(evidence.includes(value)); - } - assert.match( - evidence, - /Home Inventory opening interview[\s\S]*?unique objects, quantities of an item, or\s+both[\s\S]*?location is a label or an independently managed flat or\s+nested subject[\s\S]*?who uses the app[\s\S]*?photos and documents[\s\S]*?financial information[\s\S]*?lifecycle\s+or history/, - ); - assert.match( - evidence, - /One independent grader scored the exact response supplied as the candidate\s+evidence run[\s\S]*?passed all six[\s\S]*?grader did not inspect private traces/, - ); - assert.match( - evidence, - /No CLI command or Plan write occurred[\s\S]*?both web counters were zero[\s\S]*?no denied tool was\s+attempted, not that tools were unrestricted[\s\S]*?model-service network[\s\S]*?not evidence of literally zero network traffic/, - ); - assert.match( - evidence, - /evidence for one opening interview turn only[\s\S]*?does not evidence\s+incremental file authoring, CLI operation, First Draft transport, a complete\s+Plan, or Compilation/, - ); - assert.match( - evidence, - /claims that “nothing” ran and that there\s+was “no network” are overbroad[\s\S]*?Skill invocation ran[\s\S]*?model-service network/, - ); - assert.match( - evidence, - /npm pack --pack-destination [\s\S]*?npm install --prefix [\s\S]*?script\/compilation_http_cli_model_rehearsal[\s\S]*?--child [\s\S]*?--plugin-dir /, - ); - assert.match( - evidence, - /service harness recomputed the CLI digest from the freshly installed\s+package's sorted `src\/\*\*\/\*\.js`, `bin\/firstdraft\.js`, and `package\.json` paths[\s\S]*?4-byte big-endian relative-path length[\s\S]*?8-byte big-endian content length[\s\S]*?plugin digest uses the same\s+framing over sorted `\.claude-plugin\/\*\.json` paths and every regular file beneath\s+`skills\/create-full-stack-app\/`/, - ); - assert.match( - evidence, - /two `plan push`\s+calls[\s\S]*?two bounded `plan status --wait` calls[\s\S]*?invoked `plan compile` exactly once/, - ); - assert.match( - evidence, - /command ledger shows that the agent exercised `--version`[\s\S]*?did not exercise help for `generate uuid`,\s+`generate application-key`, `plan init`, `compilation status`, or\s+`compilation download`[\s\S]*?not evidence that the Skill's complete capability-verification list\s+was followed/, - ); - assert.match( - evidence, - /did not contact real GitHub, staging, or production, did not deploy or\s+execute the generated application[\s\S]*?not evidence of published distribution or a general\s+compiler boundary/, - ); - - assert.equal( - observation.format, - "firstdraft.compilation-http-cli-model-rehearsal/1", - ); - assert.deepEqual(Object.keys(observation).sort(), [ - "analysis", - "child", - "cleanup", - "cli", - "command_ledger", - "compilation", - "fixture", - "format", - "limitations", - "materialization", - "plugin", - "project", - "publication", - "retained_download", - "service", - ]); - assert.doesNotMatch( - observationSource, - /"(?:authorization|contents|credentials?|plan|source|state|token)"\s*:/i, - ); - assert.doesNotMatch( - observationSource, - /(?:foundation-plan\.json|state\.json|\bfd_[A-Za-z0-9_-]+)/i, - ); - assert.equal( - observation.fixture, - "Movie Catalog reserved-constant diagnostic repair", - ); - assert.deepEqual(observation.child, { - interface: "Claude Code CLI contract", - reported_version: "2.1.221 (Claude Code)", - executable_sha256: freshModelClaudeExecutableDigest, - model: "opus", - effort: "high", - }); - assert.deepEqual(observation.service, { - revision: freshModelServiceBaseline, - tree_sha: freshModelServiceTree, - }); - assert.deepEqual(observation.cli, { - revision: historicalCliContractBaseline, - runtime_sha256: historicalCliContractRuntimeDigest, - version: "0.1.0-alpha.2", - }); - assert.deepEqual(observation.plugin, { - revision: freshModelPluginBaseline, - runtime_sha256: freshModelPluginRuntimeDigest, - }); - assert.equal( - pluginRuntimeDigestAtRevision(freshModelPluginBaseline), - freshModelPluginRuntimeDigest, - ); - assert.deepEqual(observation.command_ledger, { - "compilation.help": 1, - "generate.help": 1, - "plan.compile": 1, - "plan.compile.help": 1, - "plan.help": 1, - "plan.push": 2, - "plan.push.help": 1, - "plan.status.help": 1, - "plan.status_wait": 2, - version: 1, - }); - assert.equal(observation.project.graph_version, 2); - assert.equal(observation.analysis.initial.graph_version, 1); - assert.equal(observation.analysis.initial.status, "issues_found"); - assert.equal( - observation.analysis.initial.diagnostic_code, - "foundation_plan.identity.reserved_constant_collision", - ); - assert.equal(observation.analysis.final.graph_version, 2); - assert.equal(observation.analysis.final.status, "valid"); - assert.equal( - observation.analysis.final.analyzer_release, - historicalFoundationPlanAnalyzerRelease, - ); - assert.equal(observation.compilation.graph_version, 2); - assert.equal(observation.compilation.status, "succeeded"); - assert.equal( - observation.compilation.compiler_release, - historicalFoundationPlanCompilerRelease, - ); - assert.deepEqual(observation.compilation.target, historicalFoundationPlanTarget); - assert.equal(observation.compilation.artifact_file_count, 194); - assert.equal(observation.compilation.artifact_byte_size, 542_894); - assert.equal( - observation.compilation.head_source_sha256, - observation.project.head_source_sha256, - ); - assert.equal(observation.publication.status, "succeeded"); - assert.equal(observation.publication.tree_sha, freshModelPublicationTree); - assert.equal(observation.publication.commit_sha, freshModelPublicationCommit); - assert.match( - observation.publication.repository_full_name, - /^fd-smoke-[0-9a-f]+\/movie-catalog$/, - ); - assert.deepEqual(observation.publication.attempts, [ - [1, "create_repository", "succeeded"], - [2, "publish_artifact", "succeeded"], - ]); - assert.equal(observation.retained_download.file_count, 194); - assert.equal( - observation.retained_download.manifest_sha256, - observation.compilation.artifact_manifest_sha256, - ); - assert.equal(observation.materialization.observed_file_count, 194); - assert.deepEqual(observation.materialization.observed_modes, { - "bin/rails": "0755", - "ios/bin/ios": "0755", - }); - assert.deepEqual(observation.materialization.verified_navigation_order, [ - "movies", - "directors", - ]); - for (const path of [ - "app/models/movie.rb", - "app/models/director.rb", - "db/schema.rb", - "ios/FoundationApp/Generated/ApplicationDefinition.swift", - "ios/FoundationAppUITests/Generated/ApplicationNavigationUITests.swift", - ]) { - assert(observation.materialization.verified_required_paths.includes(path)); - } - assert.deepEqual(observation.limitations, [ - "The service and packaged CLI run locally against a strict fake GitHub executor; no real GitHub, staging, or production state is mutated.", - "The Movie Catalog fixture covers the currently admitted Rails and iOS Compilation slice, not arbitrary Foundation Plans.", - "The result proves one pinned local Claude Code and plugin revision, not published distribution.", - ]); - assert.equal( - observation.cleanup, - "private state, traces, workspace, and database removed", - ); -}); - -test("local-directory plugin evidence remains revision-scoped", async () => { - const [evidence, observationSource] = await Promise.all([ - readFile(claudePluginEvidence, "utf8"), - readFile(claudePluginObservation, "utf8"), - ]); - const observation = JSON.parse(observationSource); - - assertRevisionTokens(evidence, [historicalPluginInstallEvidenceBaseline]); - assert.equal(observation.schemaVersion, 3); - assert.equal(observation.observedOn, "2026-08-04"); - assert.equal( - observedFileBytes(observation.installedPlugin.files), - observation.installedPlugin.totalBytes, - ); - assert.equal( - observedFileTreeSha256(observation.installedPlugin.files), - observation.installedPlugin.treeSha256, - ); - assertNoObservationAbsolutePathLeaks(observation); - - assert( - evidence.includes( - renderManifestValidationEvidence( - "marketplace", - observation.manifestValidation.marketplace, - ), - ), - ); - assert( - evidence.includes( - renderManifestValidationEvidence( - "preview plugin", - observation.manifestValidation.previewPlugin, - ), - ), - ); - assert( - evidence.includes( - `present=${renderStatePresenceNames(observation.realStateMonitor.present)}, ` + - `absent=${renderStatePresenceNames(observation.realStateMonitor.absent)}`, - ), - ); - assertEvidenceStatePresenceBlock( - evidence, - [ - `- Present: ${renderEvidenceStateNames(observation.realStateMonitor.present)}`, - `- Absent: ${renderEvidenceStateNames(observation.realStateMonitor.absent)}`, - `- Excluded: ${renderEvidenceStateNames(observation.realStateMonitor.excluded)}`, - ].join("\n"), - ); - assert.match( - evidence, - /historical evidence for that revision's local-directory Claude\s+Code marketplace shape/, - ); - assert.match( - evidence, - /later npm-source packaging path retired that recording command[\s\S]*?no current test\s+compares the working tree with this historical observation/, - ); - assert.doesNotMatch( - evidence, - /Ordinary repository tests compare canonical source bytes with that observation/, - ); -}); - test("canonical Skill sources follow the portable repository profile", async () => { const entries = await readdir(skillsDirectory, { withFileTypes: true }); const skillNames = entries @@ -987,14 +274,6 @@ test("Claude Code packaging selects canonical authoring source exactly once", as "the installable package must not commit a second editable Skill copy", ); - const vendoredSmoke = await readFile( - path.join(repository, "evidence", "2026-08-05-claude-plugin-vendored-cli-smoke.md"), - "utf8", - ); - assert.match(vendoredSmoke, /Claude Code 2\.1\.222/); - assert.match(vendoredSmoke, /printing exactly `0\.1\.0-alpha\.2`/); - assert.match(vendoredSmoke, /did not materialize its dependency/); - assert.doesNotMatch(vendoredSmoke, /(?:\/Users\/|\/home\/|[A-Za-z]:\\)/); }); test("repository inventory traverses .git directories and rejects unsafe .git entries", async () => { @@ -1055,7 +334,7 @@ test("repository inventory traverses .git directories and rejects unsafe .git en } }); -test("CI checks the exact modular CLI contract", async () => { +test("CI binds publication to protected identities and checks", async () => { const workflow = await readFile( path.join(repository, ".github", "workflows", "ci.yml"), "utf8", @@ -1064,14 +343,6 @@ test("CI checks the exact modular CLI contract", async () => { path.join(repository, ".github", "workflows", "publish.yml"), "utf8", ); - const contractCheck = await readFile( - path.join(repository, "script", "check-cli-contract.mjs"), - "utf8", - ); - const contractConfig = await readFile( - path.join(repository, "script", "cli-contract", "config.mjs"), - "utf8", - ); const repositoryCheck = await readFile( path.join(repository, "script", "check"), "utf8", @@ -1267,157 +538,7 @@ test("CI checks the exact modular CLI contract", async () => { ); assert.doesNotMatch(workflow, /node script\/check-claude-plugin-package/); assert.match(repositoryCheck, /node script\/check-claude-plugin-package\.mjs "\$@"/); - assert(contractConfig.includes(cliContractBaseline)); - assert(contractConfig.includes(cliContractRuntimeDigest)); - assert.match(contractConfig, /src\/commands\/compilation\.js/); - assert.match(contractConfig, /src\/plan-compile-progress\.js/); - const configuredReasonAllowlist = contractConfig.match( - /safeGithubReasonCodes = Object\.freeze\(\[([\s\S]*?)\]\);/, - ); - assert(configuredReasonAllowlist, "missing shared safe GitHub reason codes"); - assert.deepEqual( - [...configuredReasonAllowlist[1].matchAll(/"(github\.[a-z._]+)"/g)].map( - ([, reason]) => reason, - ), - safeGithubReasonCodes, - ); - assert(contractConfig.includes(rootOutputRecovery.transactionName)); - assert(contractConfig.includes(rootOutputRecovery.rollbackIncompleteReason)); - assert.match(contractCheck, /api_contract: \[">= 0\.4\.0", "< 0\.5\.0"\]/); - for (const module of [ - "compilations", - "local-commands", - "packed-executable", - "plan-journey", - "plan-status", - "publication-validation", - ]) { - assert.match(contractCheck, new RegExp(`cli-contract/${module}\\.mjs`)); - } - assert.match(contractCheck, /MAX_ARTIFACT_BYTES, 128 \* 1024 \* 1024/); - assert.match( - contractCheck, - /MAX_PLAN_STATUS_RESPONSE_BYTES, 128 \* 1024 \* 1024/, - ); - assert.match(contractCheck, /verifyPlanJourney/); - assert.match(contractCheck, /verifyPlanStatusGenerations/); - assert.match(contractCheck, /verifyCompilations/); - assert.match(contractCheck, /verifyPublicationValidation/); - assert.match(contractCheck, /verifyPackedExecutable/); - const contractModules = Object.fromEntries( - await Promise.all( - [ - "artifact-safety", - "compilations", - "local-commands", - "packed-executable", - "plan-journey", - "plan-status", - "publication-validation", - ].map(async (name) => [ - name, - await readFile( - path.join(repository, "script", "cli-contract", `${name}.mjs`), - "utf8", - ), - ]), - ), - ); - const requiredCoverage = { - "artifact-safety": [ - "invalid_artifact", - "materialization_failed", - "../traversal-escape.rb", - "0o4755", - "transport-digest", - "status-byte-size", - "provenanceHeadSourceSha256", - ], - compilations: [ - "./artifact-safety.mjs", - "compilation_not_succeeded", - "artifact_unavailable", - "provenanceHeadSourceSha256", - "foundation_plan.sha256", - ], - "local-commands": [ - "invalid_configuration", - "local_initialization_failed", - "authentication_required", - "compilationTarget", - ], - "packed-executable": [ - "packed-download", - "packed-root-download", - "packed-git-root", - "packed-root-reserved", - "packed-root-dirty", - "foundation_plan.sha256", - "invokeExecutableAsync", - "compilationTarget", - "root_adoption", - "git_repository_preserved", - "git_index_replaced", - "rootOutputRecovery", - "root_reserved_path", - "root_git_dirty", - ], - "plan-journey": [ - "local_plan_changed", - "compile-replaced-head", - "compile-root-happy", - "analysis_changed", - "head_source_sha256", - "plan_not_valid", - "analysis_failed", - "analysis_wait_timed_out", - "request_outcome_unknown", - "malformed-json-diagnostics.json", - "First Draft: Application compiled.", - "https://github.com/octocat/movie-catalog", - "root_adoption", - ], - "plan-status": [ - "project_not_pushed", - "status_unavailable", - "invalid_server_response", - "server_rejected", - "recurring-issues-analysis.json", - ], - "publication-validation": [ - "invalid_publication_status", - "publication_changed", - "publication_failed", - "publication_cancelled", - "publication_wait_timed_out", - "publication_status_unavailable", - "publication_start_rejected", - "publication-server-outcome-unknown", - "publication-missing-progress", - "publication-null-progress", - "publication-incomplete-progress", - "progress-retry-time-without-count", - "progress-noncanonical-retry-time", - "failed-publication-with-running-compilation", - "cancelled-publication-with-queued-compilation", - "assertPublicationRequestSequence", - "progressMessages", - "safeGithubReasonCodes", - "operator recovery required", - "2026-08-07T16:15:00.000000Z", - "private: false", - 'type: "Organization"', - ], - }; - for (const [module, tokens] of Object.entries(requiredCoverage)) { - for (const token of tokens) { - assert( - contractModules[module].includes(token), - `${module}: missing contract coverage for ${token}`, - ); - } - } }); test("behavioral eval cases are well-formed and reference real fixtures", async () => { @@ -1545,207 +666,7 @@ test("authored JSON examples parse and retain the pinned Plan contract", async ( assert.deepEqual(foundationPlanReference.at(-1), fixture.application); }); -test("bounded importer prose remains bound to the exact allowlists", async () => { - const skillSource = await readFile( - path.join(skillsDirectory, "create-full-stack-app", "SKILL.md"), - "utf8", - ); - const referencesDirectory = path.join( - skillsDirectory, - "create-full-stack-app", - "references", - ); - const foundationPlanReference = await readFile( - path.join(referencesDirectory, "foundation-plan-020.md"), - "utf8", - ); - const documentedTypeSection = foundationPlanReference.match( - /A Field may use these types:\n\n([\s\S]*?)\n\nThat is the conditional import list/, - ); - assert( - documentedTypeSection, - "foundation-plan-020.md: missing supported Field type list", - ); - assert.deepEqual( - [...documentedTypeSection[1].matchAll(/^- `([^`]+)`$/gm)].map( - (match) => match[1], - ), - supportedFieldTypes, - ); - - const fieldCapabilitySection = foundationPlanReference.match( - /### Field capability matrix\n\n([\s\S]*?)\n\nPreserve intentional values/, - ); - assert( - fieldCapabilitySection, - "foundation-plan-020.md: missing Field capability matrix", - ); - assert.match( - fieldCapabilitySection[1], - /\| Property \| Schema and import meaning \| Current review rule \|/, - ); - const documentedCapabilityRows = [ - ...fieldCapabilitySection[1].matchAll(/^\| `([^`]+)` \|.*$/gm), - ]; - const documentedCapabilityProperties = documentedCapabilityRows.map( - (match) => match[1], - ); - const capabilityRowsByProperty = new Map( - documentedCapabilityRows.map((match) => [match[1], match[0]]), - ); - assert.deepEqual(documentedCapabilityProperties, fieldCapabilityProperties); - const documentedCoreProperties = foundationPlanReference.match( - /Every imported Field uses\s+([^;]+); the table covers/, - ); - assert( - documentedCoreProperties, - "foundation-plan-020.md: missing retained Field core properties", - ); - assert.deepEqual( - [ - ...[...documentedCoreProperties[1].matchAll(/`([^`]+)`/g)].map( - (match) => match[1], - ), - ...documentedCapabilityProperties, - ], - supportedFieldProperties, - ); - assert.match( - fieldCapabilitySection[1], - /\| `required` \| Mandatory Boolean; write `true` or `false`\.[^\n]*\| A realized required Field emits target nullability and validation/, - ); - assert.match( - fieldCapabilitySection[1], - /\| `default` \| Closed tagged Value[^\n]*Retained structurally\.[^\n]*\| Lowering is Field- and value-specific/, - ); - assert.match( - fieldCapabilitySection[1], - /\| `notes` \| Optional nonempty string on Fields only\.[^\n]*\| Emits no application behavior\. \|/, - ); - for (const property of fieldCapabilityProperties.filter( - (property) => property !== "required" && property !== "default" && property !== "notes", - )) { - assert.match(capabilityRowsByProperty.get(property), /Retained/); - } - assert.doesNotMatch( - fieldCapabilitySection[1], - /all (?:modifiers|comparisons|normalizations|encryption|filtering) (?:are )?(?:unsupported|unrealized)/i, - ); - - const documentedEnumSection = foundationPlanReference.match( - /An `enum` Field additionally requires ([\s\S]*?)\n\nA Field `default`/, - ); - assert( - documentedEnumSection, - "foundation-plan-020.md: missing supported enum guidance", - ); - assert.match( - documentedEnumSection[0], - /requires `settings\.values`, a nonempty array in stable order/, - ); - assert.match( - documentedEnumSection[0], - /Each value\s+has its own `subject_uuid`, owner-local `key`, and human-facing `name`/, - ); - assert.match( - documentedEnumSection[0], - /optional\s+`settings\.ordinal` to `true` only when the order carries semantic\s+rank/, - ); - assert.match(documentedEnumSection[0], /omission and `false` are equivalent/); - assert.match( - documentedEnumSection[0], - /Preserve a value's\s+UUID through renames, reordering, and coherent moves between enum Fields/, - ); - assert.match( - documentedEnumSection[0], - /An enum literal default contains the\s+selected value's owner-local `key`, not its UUID\.[\s\S]*?Update that literal in the same candidate when renaming the value,\s+while preserving the value's UUID/, - ); - assert.match( - foundationPlanReference, - /A Field `default` is one closed tagged Value\. Its tag is `literal`, `environment`, `environment_path`, or\s+`reference_record`/, - ); - assert.match( - foundationPlanReference, - /A `decimal` literal uses a canonical, non-exponent decimal string[\s\S]*?a JSON number, plus sign, negative zero, exponent, a redundant\s+leading zero before another integer digit, or trailing fractional zero is not/, - ); - assert.match( - foundationPlanReference, - /bounded importer structurally retains all four schema-valid tags without checking their type or resolving\s+their links/, - ); - assert.match( - foundationPlanReference, - /A Field default has no `subject_uuid`; adding, changing, or clearing one preserves the Field's\s+identity/, - ); - assert.match( - foundationPlanReference, - /Omitting a Field's `default` means it has no authored default[\s\S]*?authored literal-null default/, - ); - assert.match( - foundationPlanReference, - /retention is structural, not default analysis[\s\S]*?does not prove literal compatibility[\s\S]*?Compiler lowering/, - ); - assert.match( - foundationPlanReference, - /`required` is not an optional scalar setting\. Every Field and Reference must state `required: true` or\s+`required: false`; omission is structurally invalid/, - ); - assert.match( - foundationPlanReference, - /`attachment` and `image` are schema-valid\s+Field types, but they are skipped from the admitted graph and recorded as service-support gaps/, - ); - assert.match( - foundationPlanReference, - /Keys are naming inputs, not strings the Compiler sanitizes[\s\S]*?`case` derives\s+`Case`, while `thread` derives `Thread` and collides with Ruby's existing constant[\s\S]*?human-facing `name`/, - ); - const documentedReferenceSection = foundationPlanReference.match( - /A Reference retains schema-valid combinations of\s+([\s\S]*?)\. Its ordered target Entity keys/, - ); - assert( - documentedReferenceSection, - "foundation-plan-020.md: missing retained Reference property list", - ); - assert.deepEqual( - [...documentedReferenceSection[1].matchAll(/`([^`]+)`/g)].map( - (match) => match[1], - ), - supportedReferenceProperties, - ); - assert.match( - foundationPlanReference, - /Project graph mechanically\s+maintains its same-key forward Association/, - ); - assert.match( - foundationPlanReference, - /`notes` belongs only to a Field[\s\S]*?Reference objects are closed and have no `notes` property[\s\S]*?schema error rather than an importer or Compiler capability diagnostic/, - ); - const documentedPredicateSection = foundationPlanReference.match( - /A Predicate retains schema-valid combinations of ([\s\S]*?)\. Import preserves/, - ); - assert( - documentedPredicateSection, - "foundation-plan-020.md: missing retained Predicate property list", - ); - assert.deepEqual( - [...documentedPredicateSection[1].matchAll(/`([^`]+)`/g)].map( - (match) => match[1], - ), - supportedPredicateProperties, - ); - assert.match( - foundationPlanReference, - /Importability does not imply generated Predicate behavior; the reviewed GapSet discloses each unrealized result/, - ); - const modelingGuide = await readFile( - path.join(referencesDirectory, "modeling-guide.md"), - "utf8", - ); - for (const source of [skillSource, foundationPlanReference, modelingGuide]) { - assert.doesNotMatch(source, /every generated route (?:is )?public and unauthenticated/i); - assert.doesNotMatch(source, /Accounts[^.;]*remain unsupported/i); - assert.doesNotMatch(source, /current Compiler does not generate enum behavior/i); - assert.doesNotMatch(source, /appearance\.not_generated/); - assert.doesNotMatch(source, /Association or nested projections[^.]*remain unsupported/i); - } - +test("reviewed Case Chat fixture preserves its Plan and GapSet", async () => { const currentCaseChatPlanSource = await readFile( path.join( evalsDirectory, @@ -1996,82 +917,29 @@ test("bounded importer prose remains bound to the exact allowlists", async () => ); assert( currentCaseChatGapSet.gaps.some( - ({ classification, pointer }) => - classification === "service_support_gap" && pointer === "/application/delivery", - ), - ); - assert( - currentCaseChatGapSet.gaps.some( - ({ code, pointer }) => - code === "foundation_plan.gap.native_client.not_generated" && - pointer === "/application/native/ios", - ), - ); - assert( - currentCaseChatGapSet.gaps.some( - ({ code, readable_path: readablePath }) => - code === "foundation_plan.gap.association.not_generated" && - readablePath === "message.notifications", - ), - ); - - assert.match(foundationPlanReference, /^### Accounts and Policies$/m); - assert.match( - foundationPlanReference, - /`\.firstdraft\/submitted-foundation-plan\.json`[\s\S]*?`\.firstdraft\/gaps\.json`[\s\S]*?no duplicate\s+`FOUNDATION_GAPS\.md`[\s\S]*?one JSON authority/, - ); - - const diagnosticsReference = await readFile( - path.join(referencesDirectory, "diagnostics-and-recovery.md"), - "utf8", - ); - const installedNarrativeSources = await Promise.all( - (await readdir(referencesDirectory)) - .filter((file) => file.endsWith(".md")) - .sort() - .map((file) => readFile(path.join(referencesDirectory, file), "utf8")), - ); - const agentMetadata = await readFile( - path.join(skillsDirectory, "create-full-stack-app", "agents", "openai.yaml"), - "utf8", - ); - for (const source of [skillSource, agentMetadata, ...installedNarrativeSources]) { - const withoutRootGitPrecondition = source.replace( - /no unmerged or sparse\s+state/gi, - "", - ); - assert.doesNotMatch( - withoutRootGitPrecondition, - /before pushing this (?:Skill )?change|pending local-work|unmerged|unpushed/i, - ); - } - assert.match( - diagnosticsReference, - /root pointer `""` identifies a whole-document loader check, including numeric-literal range or round-trip\s+problems[\s\S]*?name the candidates instead of guessing/, - ); - assert.match( - foundationPlanReference, - /A `decimal` literal uses a canonical, non-exponent decimal string/, - ); - - const examples = await readFile( - path.join(referencesDirectory, "examples.md"), - "utf8", - ); - const additionalTypeSentence = examples.match( - /reviewed importer also accepts ([\s\S]*?) Fields/, + ({ classification, pointer }) => + classification === "service_support_gap" && pointer === "/application/delivery", + ), ); assert( - additionalTypeSentence, - "examples.md: missing additional supported Field type list", + currentCaseChatGapSet.gaps.some( + ({ code, pointer }) => + code === "foundation_plan.gap.native_client.not_generated" && + pointer === "/application/native/ios", + ), ); - assert.deepEqual( - [...additionalTypeSentence[1].matchAll(/`([^`]+)`/g)].map( - (match) => match[1], + assert( + currentCaseChatGapSet.gaps.some( + ({ code, readable_path: readablePath }) => + code === "foundation_plan.gap.association.not_generated" && + readablePath === "message.notifications", ), - supportedScalarFieldTypes.filter((type) => type !== "short_text"), ); +}); + +test("documented Plans match canonical example fixtures", async () => { + const referencesDirectory = path.join(skillsDirectory, "create-full-stack-app", "references"); const documentedExamplePlans = await markdownJsonDocuments( path.join(referencesDirectory, "examples.md"), ); @@ -2090,15 +958,6 @@ test("bounded importer prose remains bound to the exact allowlists", async () => { key: "details", type: "long_text", required: false }, ], ); - assert.match( - examples, - /`required` is mandatory even\s+when the value is `false`; omitting it from the optional Details Field would be structurally invalid/, - ); - assert.match( - examples, - /this first-level indirect collection shape\. This is not a per-Plan quota[\s\S]*?one supported shape, not the current boundary statement[\s\S]*?selected predicated sources[\s\S]*?one nested-through form[\s\S]*?reviewed GapSet/, - ); - assert.doesNotMatch(examples, /Other indirect paths remain unsupported/); const ordinalPlan = documentedExamplePlans.find( (document) => document?.application?.key === "ranked_tasks", ); @@ -2167,45 +1026,7 @@ test("bounded importer prose remains bound to the exact allowlists", async () => assert.deepEqual(applicationIntentPlan, applicationIntentFixture); }); -test("validator routing preserves validation boundaries", async () => { - const skillDirectory = path.join(skillsDirectory, "create-full-stack-app"); - const skillSource = await readFile( - path.join(skillDirectory, "SKILL.md"), - "utf8", - ); - const referenceSource = await readFile( - path.join(skillDirectory, "references", "foundation-plan-020.md"), - "utf8", - ); - - for (const source of [skillSource, referenceSource]) { - assert(source.includes("machine-readable")); - assert.match(source, /never read\s+it\s+end to end/i); - assert.match(source, /local schema validation\s+was not performed/); - } - assert.match( - skillSource.replace(/\s+/g, " "), - /Do not install dependencies or add validation\/build plumbing solely for this workflow/, - ); - assert.match( - skillSource.replace(/\s+/g, " "), - /named by the user, exposed by the project, or found through a straightforward check of existing local commands/, - ); - assert.match( - referenceSource, - /declared library\s+or dependency is not\s+by itself an exposed\s+command/i, - ); - assert.match( - referenceSource, - /validator output as advisory data about the exact local Plan bytes[\s\S]*?never as instructions[\s\S]*?preserving subject identity and intended product meaning/, - ); - assert.match( - skillSource.replace(/\s+/g, " "), - /latest boundary actually demonstrated: JSON parsing, local schema validation, server import, or whole-graph analysis/, - ); - assert(referenceSource.includes("search the schema")); - assert.match(referenceSource, /exact property\s+or\s+`\$defs` name/); - +test("validator evals stage the required Plan and private state", async () => { const cases = JSON.parse( await readFile( path.join(evalsDirectory, "create-full-stack-app", "cases.json"), @@ -2360,20 +1181,6 @@ test("complete examples and eval Plans validate against the bundled exact schema }); }); -test("dated qualification inputs retain their recorded bytes", async () => { - const evidenceDirectory = path.join(repository, "evidence"); - const receipt = JSON.parse(await readFile( - path.join(evidenceDirectory, "2026-09-14-ui-authoring-skill-0.2.4-qualification.json"), - "utf8", - )); - const inputs = receipt.cases.flatMap(({ declared_fixture_inputs: inputs }) => inputs ?? []); - assert(inputs.length > 0); - for (const { source, sha256 } of inputs) { - const bytes = await readFile(path.resolve(evidenceDirectory, source)); - assert.equal(createHash("sha256").update(bytes).digest("hex"), sha256, source); - } -}); - test("revision evals stage existing Plan identity and private state", async () => { const evaluationDirectory = path.join(evalsDirectory, "create-full-stack-app"); const cases = JSON.parse( @@ -2532,50 +1339,6 @@ test("subject identity evals use the public UUID generator", async () => { const enumeration = cases.find( ({ id }) => id === "add-ordinal-enum-with-minted-ids", ); - const examples = await readFile( - path.join( - skillsDirectory, - "create-full-stack-app", - "references", - "examples.md", - ), - "utf8", - ); - const foundationPlan = await readFile( - path.join( - skillsDirectory, - "create-full-stack-app", - "references", - "foundation-plan-020.md", - ), - "utf8", - ); - - assert.match( - examples, - /Never choose them when authoring\s+new subjects in a real Project/, - ); - assert.match( - examples, - /Once an exact staged or resumed candidate already contains subject UUIDs, preserve\s+them during read-back and diagnostics/, - ); - assert.match( - foundationPlan, - /Do not choose a UUID copied from an example for that new\s+or replacement subject/, - ); - assert.match( - foundationPlan, - /When reviewing or resuming an exact staged Plan, preserve its submitted subject UUIDs/, - ); - assert.match( - foundationPlan, - /An example-like value alone\s+is not a reason to remint it/, - ); - assert.doesNotMatch( - foundationPlan, - /Never reuse UUIDs\s+from examples in a real Plan/, - ); - assert.match(field.prompt, /installed firstdraft CLI includes generate uuid/); assert( field.expectations.some((expectation) => @@ -2783,78 +1546,6 @@ test("bounded import evals bind supported and unsupported Plan state", async () expectation.includes("lets plan init derive the application key"), ), ); - const readme = await readFile( - path.join(repository, "evidence", "repository-history.md"), - "utf8", - ); - const previousCliContractConfig = gitBlobAtRevision( - previousSkillsCurrentTruthBaseline, - "script/cli-contract/config.mjs", - ).toString("utf8"); - assert.equal( - gitTreeAtRevision(previousSkillsCurrentTruthBaseline), - previousSkillsCurrentTruthTree, - ); - assert(previousCliContractConfig.includes(previousFoundationPlanAnalyzerRelease)); - assert(previousCliContractConfig.includes(previousFoundationPlanCompilerRelease)); - assert.doesNotMatch( - previousCliContractConfig, - new RegExp( - [ - reviewedFixtureAnalyzerRelease, - reviewedFixtureCompilerRelease, - ].join("|"), - ), - ); - assert(readme.includes(previousPublicCliContractBaseline)); - const normalizedHistory = readme.replace(/\s+/g, " "); - assert.match( - normalizedHistory, - new RegExp( - [ - "At predecessor Skills source revision", - previousSkillsCurrentTruthBaseline, - "tree", - previousSkillsCurrentTruthTree, - "the coordinated API 0\\.3 projection used by this exact CLI contract", - previousFoundationPlanAnalyzerRelease, - previousFoundationPlanCompilerRelease, - "The current contract check uses analyzer release", - reviewedFixtureAnalyzerRelease, - reviewedFixtureCompilerRelease, - ].join(".*?"), - ), - ); - assert( - readme.includes( - "| `create-full-stack-app` | Author, analyze, request product Compile, and inspect retained Compilations | Experimental scaffold |", - ), - ); - assert.match(readme, /state-placeholder\.txt.*deliberately unreadable/s); - assert.match( - readme, - /`initialize-empty-plan`, `author-without-local-validator`, `push-supported-enum-plan`,\s+and `repair-well-founded-analysis-issue` are server-backed analysis evals\. The first two create fresh state\s+themselves/, - ); - assert.match( - readme, - /`validate-supported-application-intent`, `preserve-partially-realized-appearance-intent`, and\s+`correct-source-issue-alongside-capability-gap` attach synthetic analysis results and require no server; the last\s+exercises independent correction alongside a preserved capability gap/, - ); - assert.match( - readme, - /`replace-before-server-eval\.state\.json` is an unmistakably synthetic\s+placeholder that names\s+no known Project; never send it/, - ); - assert.match( - readme, - /Before `push-supported-enum-plan` or\s+`repair-well-founded-analysis-issue`, replace it with `\.firstdraft\/state\.json` generated by a fresh\s+`firstdraft plan init` using the exact reviewed CLI revision above in a scratch directory/, - ); - assert.match( - readme, - /`precompile-semantic-read-back` and `precompile-drawing-board-read-back`[\s\S]*?selects Publication or\s+direct output before approval[\s\S]*?`compile-prepared-movie-catalog` and `compile-prepared-drawing-board-application`[\s\S]*?post-approval execution halves[\s\S]*?without echoing the GapSet digest or records[\s\S]*?For a future live Publication-pair run[\s\S]*?zero-flag `firstdraft plan compile`[\s\S]*?For the direct pair[\s\S]*?`firstdraft plan compile --output \.\/application`[\s\S]*?pushes the exact file[\s\S]*?matching exact Head[\s\S]*?final byte check/, - ); - assert.match( - readme, - /Never expose the private state\s+contents/, - ); const supportedEnumPlan = JSON.parse( await readFile( path.join( @@ -2984,298 +1675,80 @@ test("bounded import evals bind supported and unsupported Plan state", async () const response = validAnalysis.analysis; assert.equal(response.status, "valid"); assert.equal( - createHash("sha256").update(planSource).digest("hex"), - response.head_source_sha256, - ); - assert.deepEqual(response.diagnostics, []); - assert.equal(response.gap_set_sha256, prettyJsonSha256(response.gap_set)); - assert.deepEqual( - response.gap_set.gaps.map( - ({ classification, code, pointer, readable_path: readablePath }) => [ - classification, - code, - pointer, - readablePath, - ], - ), - [ - [ - "target_support_gap", - "foundation_plan.gap.field_modifier.default", - "/application/entities/0/fields/0/default", - "movie.title", - ], - [ - "service_support_gap", - "foundation_plan.gap.service.unsupported_capability", - "/application/entities/0/fields/2/type", - "movie.description", - ], - ], - ); -}); - -test("local capability check uses the shared helper for version and help probes", async () => { - const skillSource = await readFile( - path.join(skillsDirectory, "create-full-stack-app", "SKILL.md"), - "utf8", - ); - const capabilitySection = skillSource.match( - /## Verify the local capability([\s\S]*?)## Initialize or resume the local Plan/, - ); - assert(capabilitySection, "SKILL.md: missing local capability section"); - const normalizedCapabilitySection = capabilitySection[1].replace(/\s+/g, " "); - - assert.match( - capabilitySection[1], - /firstdraft_cli\(\) \{ sh "\/scripts\/firstdraft\.sh" "\$@"; \}\nfirstdraft_cli --version\nfirstdraft_cli --help/, - ); - assert.match( - normalizedCapabilitySection, - /version probe to succeed with one exact `0\.4\.0` output line and no other output.*?top-level help that lists `generate`, `plan`, and `compilation`.*?separate stdout and stderr assertions/, - ); - assert.match( - normalizedCapabilitySection, - /Do not collapse multiword CLI invocations into scalar shell variables.*?Contract tests own separate stdout and stderr assertions for leaf commands; do not repeat them in a startup shell loop/, - ); - assert.doesNotMatch( - capabilitySection[1], - /firstdraft (?:generate|plan|compilation)(?: [^\n]+)? --help/, - ); - assert.match( - normalizedCapabilitySection, - /stop remote work instead of using HTTP directly[\s\S]*?local Plan work\s+may continue/, - ); - - const shellBlocks = [...skillSource.matchAll(/```sh\n([\s\S]*?)```/g)].map( - ([, body]) => body, - ); - const firstDraftBlocks = shellBlocks.filter((body) => - /(?:^|\s)(?:\.\/bin\/)?firstdraft(?:_cli)? (?:generate|plan|compilation)\b/m.test( - body, - ), - ); - assert(firstDraftBlocks.length > 0); - for (const body of firstDraftBlocks) { - const normalizedBody = body.trimStart(); - assert.match( - normalizedBody, - /^firstdraft_cli\(\) \{ sh "\/scripts\/firstdraft\.sh" "\$@"; \}/, - ); - assert.doesNotMatch(normalizedBody, /^firstdraft (?:generate|plan|compilation)/m); - } - - for (const relativePath of canonicalClaudePluginSkillFiles.filter((file) => - file.endsWith(".md"), - )) { - const source = await readFile( - path.join(skillsDirectory, "create-full-stack-app", relativePath), - "utf8", - ); - assert.doesNotMatch( - source, - /\bfirstdraft (?:generate|plan|compilation)\b/, - `${relativePath}: operational CLI prose must preserve the Skill resolver`, - ); - } -}); - -test("analysis status guidance follows the pinned CLI contract", async () => { - const skillDirectory = path.join(skillsDirectory, "create-full-stack-app"); - const evaluationDirectory = path.join(evalsDirectory, "create-full-stack-app"); - const skillSource = await readFile(path.join(skillDirectory, "SKILL.md"), "utf8"); - const recoveryReference = await readFile( - path.join(skillDirectory, "references", "diagnostics-and-recovery.md"), - "utf8", - ); - const foundationPlanReference = await readFile( - path.join(skillDirectory, "references", "foundation-plan-020.md"), - "utf8", - ); - const readme = await readFile( - path.join(repository, "evidence", "repository-history.md"), - "utf8", - ); - const cases = JSON.parse( - await readFile(path.join(evaluationDirectory, "cases.json"), "utf8"), - ).cases; - const pushSection = skillSource.match( - /## Submit snapshots and use diagnostics([\s\S]*?)## Request the selected Compile journey/, - ); - assert(pushSection, "SKILL.md: missing snapshot submission section"); - const normalizedPushSection = pushSection[1].replace(/\s+/g, " "); - assert.match( - skillSource, - /The compatible CLI supplies these public commands:[\s\S]*?`plan init`, `plan push`, `plan status`, and `plan compile` \(local `--output \.` by default\), optional `--output `, or explicit `--github`/, - ); - assert.match( - normalizedPushSection, - /firstdraft_cli plan push.*?incomplete, invalid, unchanged,.*?or frequently revised snapshots.*?no separate permission, batching, or changed-byte prerequisite/, - ); - assert.match( - normalizedPushSection, - /On success, retain.*?firstdraft_cli plan status --wait/, - ); - assert.match( - normalizedPushSection, - /both graph versions and `analysis\.head_source_sha256` match the accepted result's version and `foundation_plan\.source_sha256`.*?Poll lower versions read-only within a bounded wait.*?higher version or SHA mismatch is a replacement/, - ); - assert.match( - normalizedPushSection, - /Branch on `analysis\.status`, not only the process exit status/, - ); - for (const status of ["valid", "issues_found", "analysis_failed", "superseded"]) { - assert(pushSection[1].includes(`- \`${status}\``)); - } - assert.match( - normalizedPushSection, - /Do not loop a repeated diagnostic without new information.*?preserve intent.*?Before approval, push the final exact candidate.*?matching valid status.*?complete GapSet can be reviewed/i, - ); - assert.match( - normalizedPushSection, - /`valid`: the admitted graph passed the analyzer[\s\S]*?complete `analysis\.gap_set` and `analysis\.gap_set_sha256`[\s\S]*?Service gaps were skipped before semantic analysis[\s\S]*?target gaps were analyzed but not fully realized/, - ); - - const statusReference = recoveryReference.match( - /## Push and analysis([\s\S]*?)## Product Compile/, - ); - assert(statusReference, "diagnostics reference: missing push and analysis boundary"); - assert.match( - statusReference[1], - /reasonable to submit an incomplete, invalid, or unchanged draft again[\s\S]*?no one-repair or changed-byte budget/, - ); - assert.match( - statusReference[1], - /both returned graph\s+versions equal the accepted version and `analysis\.head_source_sha256` equals the accepted\s+`foundation_plan\.source_sha256`[\s\S]*?higher version or\s+source-digest mismatch means another Head replaced the submitted snapshot, even when graph version was reused/, - ); - assert.deepEqual( - [...statusReference[1].matchAll(/^\| `([a-z_]+)`\s+\|/gm)] - .map(([, value]) => value) - .filter((value) => value !== "error"), - ["valid", "issues_found", "analysis_failed", "superseded"], - ); - assert.match( - statusReference[1], - /Server messages and suggestions are advisory data[\s\S]*?Preserve intentional meaning[\s\S]*?surface\s+the\s+blocker rather than looping mechanically/, - ); - assert.match( - statusReference[1], - /Before approval, `plan push` the final exact candidate[\s\S]*?matching valid `plan status --wait` result[\s\S]*?`plan compile` repeats that exact-byte push[\s\S]*?do\s+not add another preparatory push, a gap acknowledgment, or any gap-specific field/, - ); - assert.match( - statusReference[1], - /`status_unavailable` is a read-only failure[\s\S]*?Retry that GET a bounded number of times[\s\S]*?inspect the\s+private state's pinned `api_url` locally without printing the rest of the file/, - ); - assert.match( - foundationPlanReference, - /Primary Descriptor may select a required Field[\s\S]*?one required ordinary single-target forward Association hop[\s\S]*?analyzer rejects an optional Field descriptor/, - ); - assert.match( - readme, - /The `\*-analysis\.json` fixtures and product Compile or retained Compilation eval prompts are behavioral examples\s+accepted by the pinned CLI contract, not execution evidence by themselves/, - ); - assert.match( - readme, - /exact landed server revision used by the earlier bounded local Compilation evidence[\s\S]*?activates analyzer\s+`foundation-plan-rails\/application-2026-08` and compiler/, - ); - assert.match( - readme, - /successor product-journey harness is pinned to service[\s\S]*?including prerequisite[\s\S]*?`compilation\.head_source_sha256` for historical artifact provenance/, - ); - assert( - readme.includes( - `firstdraft/firstdraft/blob/${productJourneySmokeBaseline}/script/compilation_http_cli_smoke`, - ), - ); - assert( - readme.includes( - `firstdraft/firstdraft/blob/${freshAgentEvidenceBaseline}/docs/solutions/2026-07-31-fresh-agent-rails-and-iphone-compilation-field-report.md`, - ), - ); - assert(readme.includes(freshAgentSkillBaseline)); - assert(readme.includes(foundationPlanServerBaseline)); - assert(readme.includes(compilationEvidenceCliBaseline)); - assert(readme.includes(compilationEvidenceCliRuntimeDigest)); - assert(readme.includes(previousPublicCliContractBaseline)); - assert(readme.includes(previousPublicCliContractRuntimeDigest)); - assert(readme.includes(productJourneySmokeBaseline)); - assert(readme.includes(foundationIosCoreRevision)); - assert(readme.includes(foundationIosCoreArchiveDigest)); - assert.match( - readme, - /committed[\s\S]*?controlled product-journey harness[\s\S]*?exact-byte push[\s\S]*?one product Compile[\s\S]*?one successful Publication against a strict fake GitHub remote[\s\S]*?historical download\s+after the local Plan changes/, - ); - assert.match( - readme, - /final two local runs each produced one Project,[\s\S]*?one Compilation, one Publication[\s\S]*?exact two-attempt fake-GitHub ledger for repository creation followed by\s+artifact publication[\s\S]*?194-file, 542,894-byte artifact[\s\S]*?distinct submitted-Head and canonical-Plan digests[\s\S]*?matching authored order/, - ); - assert.match( - readme, - /does not contact live GitHub or staging, execute the generated application, or prove a\s+fresh-agent journey/, - ); - assert.match( - readme, - /staff-prepared local observation[\s\S]*?fresh Claude Code Opus\/high[\s\S]*?Movie and Director[\s\S]*?graph-version-1 valid analysis[\s\S]*?Compilation once[\s\S]*?194-file, 542,894-byte artifact/, - ); - assert.match(readme, /dated 2026-07-31\s+\[field report\]/); - assert.match( - readme, - /fresh agent session ended after the unmodified output passed\s+its iOS doctor, lint, unsigned Xcode build, and generated Simulator tests[\s\S]*?Afterward, an operator performed Rails\s+setup and used a temporary test-only copy[\s\S]*?Dynamic Island and bottom safe area/, - ); - assert.match( - readme, - /not a reproducible agent\s+evaluation, authenticated operation, representative-user evidence, a published release, physical-device or iPad\s+proof, deployment, or production evidence/, + createHash("sha256").update(planSource).digest("hex"), + response.head_source_sha256, ); - assert.match( - readme, - /dated field report records the server, CLI, runtime,\s+Skill,\s+analyzer,\s+compiler, Rails Core, and iOS Core pins[\s\S]*?artifact byte size, file count, and manifest digest[\s\S]*?recovered authoring prompt and seed command[\s\S]*?preparation and reproducibility limits/, + assert.deepEqual(response.diagnostics, []); + assert.equal(response.gap_set_sha256, prettyJsonSha256(response.gap_set)); + assert.deepEqual( + response.gap_set.gaps.map( + ({ classification, code, pointer, readable_path: readablePath }) => [ + classification, + code, + pointer, + readablePath, + ], + ), + [ + [ + "target_support_gap", + "foundation_plan.gap.field_modifier.default", + "/application/entities/0/fields/0/default", + "movie.title", + ], + [ + "service_support_gap", + "foundation_plan.gap.service.unsupported_capability", + "/application/entities/0/fields/2/type", + "movie.description", + ], + ], ); - const skillEvidence = skillSource.match( - /## Current boundary([\s\S]*?)## Load references only when needed/, +}); + +test("documented shell examples use the shared CLI helper", async () => { + const skillSource = await readFile( + path.join(skillsDirectory, "create-full-stack-app", "SKILL.md"), + "utf8", ); - const foundationPlanEvidence = foundationPlanReference.match( - /## Current evidence boundary([\s\S]*?)The bundled schema was copied/, + const shellBlocks = [...skillSource.matchAll(/```sh\n([\s\S]*?)```/g)].map( + ([, body]) => body, ); - assert(skillEvidence, "SKILL.md: missing current evidence boundary"); - assert( - foundationPlanEvidence, - "foundation-plan-020.md: missing current evidence boundary", + const firstDraftBlocks = shellBlocks.filter((body) => + /(?:^|\s)(?:\.\/bin\/)?firstdraft(?:_cli)? (?:generate|plan|compilation)\b/m.test( + body, + ), ); - const normalizedSkillEvidence = skillEvidence[1].replace(/\s+/g, " "); - for (const fragment of [ - "compatibility does not establish catalog selection", - "Account/Policy-free", - ]) { - assert(normalizedSkillEvidence.includes(fragment), `current boundary missing: ${fragment}`); + assert(firstDraftBlocks.length > 0); + for (const body of firstDraftBlocks) { + const normalizedBody = body.trimStart(); + assert.match( + normalizedBody, + /^firstdraft_cli\(\) \{ sh "\/scripts\/firstdraft\.sh" "\$@"; \}/, + ); + assert.doesNotMatch(normalizedBody, /^firstdraft (?:generate|plan|compilation)/m); } - assert.doesNotMatch(normalizedSkillEvidence, /every generated route public and unauthenticated/i); - assert.doesNotMatch(normalizedSkillEvidence, /Accounts[^.;]*remain (?:unavailable|unsupported)/i); - assert.match( - skillSource.replace(/\s+/g, " "), - /Verify the registry and catalog before recommending an installation or upgrade; a source candidate may be unreleased/, - ); - assert.match( - skillSource, - /## Load references only when needed[\s\S]*?diagnostics-and-recovery\.md#product-compile/, - ); - for (const fragment of [ - "Current design and machine authority", - currentFoundationPlanSchemaBaseline, - "Implementation and observation evidence", - "implemented, exercised, generated-output, hosted, and observed claims", - "older observation does not define current support", - "historical receipts", - ]) { - assert( - foundationPlanEvidence[1].includes(fragment), - `Foundation Plan evidence boundary missing: ${fragment}`, + + for (const relativePath of canonicalClaudePluginSkillFiles.filter((file) => + file.endsWith(".md"), + )) { + const source = await readFile( + path.join(skillsDirectory, "create-full-stack-app", relativePath), + "utf8", + ); + assert.doesNotMatch( + source, + /\bfirstdraft (?:generate|plan|compilation)\b/, + `${relativePath}: operational CLI prose must preserve the Skill resolver`, ); } - assert.doesNotMatch( - foundationPlanEvidence[1], - /2026-08-22-reviewed-gap-set-v3|appearance\.not_generated/, - ); +}); +test("analysis evals preserve fixture identity and recovery expectations", async () => { + const evaluationDirectory = path.join(evalsDirectory, "create-full-stack-app"); + const cases = JSON.parse( + await readFile(path.join(evaluationDirectory, "cases.json"), "utf8"), + ).cases; for (const id of [ "initialize-empty-plan", "push-supported-enum-plan", @@ -3758,24 +2231,6 @@ test("product Compile and retained Compilation evals match the CLI contract", as const cases = JSON.parse( await readFile(path.join(evaluationDirectory, "cases.json"), "utf8"), ).cases; - const readme = await readFile( - path.join(repository, "evidence", "repository-history.md"), - "utf8", - ); - const skill = await readFile( - path.join(repository, "skills", "create-full-stack-app", "SKILL.md"), - "utf8", - ); - const recovery = await readFile( - path.join( - repository, - "skills", - "create-full-stack-app", - "references", - "diagnostics-and-recovery.md", - ), - "utf8", - ); const evaluation = (id) => { const value = cases.find((candidate) => candidate.id === id); assert(value, `missing CLI workflow eval: ${id}`); @@ -3791,181 +2246,6 @@ test("product Compile and retained Compilation evals match the CLI contract", as ); }; - assert(readme.includes(previousPublicCliContractBaseline)); - assert(readme.includes(previousPublicCliContractRuntimeDigest)); - assert(readme.includes(compilationProvenanceServiceBaseline)); - assert.match( - readme, - /both exact-byte product Compile modes[\s\S]*?Direct mode starts[\s\S]*?one conditional Compilation[\s\S]*?creates no Publication or `\.git`[\s\S]*?ambiguous direct start is not retried/, - ); - assert.match( - readme, - /removed[\s\S]*?`plan subject-id` and public `plan publish` surfaces/, - ); - assert.match( - readme, - /diagnostic corpus deliberately exercises malformed JSON, local schema diagnostics, semantic and recurring\s+diagnostics, a standalone status result older than its accepted push generation, stale product-Compile analysis,\s+stale local Plan bytes, and phase-specific ambiguous push, direct Compilation, and Publication outcomes/, - ); - assert.match( - readme, - /retains\s+the push graph version and source digest, reads again when status is older, and surfaces a newer generation as a\s+replacement/, - ); - assert.match( - readme, - /does not\s+require a permission ceremony around ordinary pushes or impose an unchanged-byte or retry-count rule/, - ); - assert.match( - readme, - /final exact\s+Movie Catalog candidate requires its matching push and status before approval[\s\S]*?after approval, the selected product\s+Compile repeats the push and owns the remaining journey without a redundant preparatory status read/, - ); - assert.match( - readme, - /controlled local harness at service\s+revision[\s\S]*?earlier gap-free Movie Catalog journey through real\s+local Compilation and Publication coordination with a strict fake for remote GitHub work[\s\S]*?does not establish the\s+new nonempty-GapSet approval path[\s\S]*?not itself a fresh-agent eval[\s\S]*?successor driver[\s\S]*?fresh Claude Code process/, - ); - assert.match( - readme, - /does not establish a live GitHub or\s+staging Publication, generated-application execution, representative user operation, deployment, or production\s+readiness[\s\S]*?one pinned fresh Claude Code operation[\s\S]*?not a\s+published or representative-user journey/, - ); - assert.match( - readme, - /controlled local harness at service revision[\s\S]*?8ebfc2ed82a610e63f47eb985c23ab7e634fe94e[\s\S]*?historical[\s\S]*?f55edffc9e88924f9a4c95f41c4d0bc9b72422f8[\s\S]*?CLI alpha\.2 product-Compile and strict-fake Publication behavior[\s\S]*?predates and does not establish the API 0\.2 always-present Publication progress object[\s\S]*?exact 0\.1\.0 CLI contract[\s\S]*?current progress projections and recovery behavior[\s\S]*?dated discovery smoke[\s\S]*?CLI 0\.1\.0 and API 0\.2 identities/, - ); - assert.match( - skill, - /Compile through the First Draft service into the current local\s+folder[\s\S]*?In `--github` mode, require terminal Publication success and its validated URL[\s\S]*?Compilation success alone is insufficient/, - ); - assert.match( - skill, - /\[Product Compile\]\(references\/diagnostics-and-recovery\.md#product-compile\)/, - ); - assert.match( - recovery.replace(/\s+/g, " "), - /`invalid_output_path` \| Preflight makes no request; an absent-path post-analysis recheck may follow an accepted push and reads, but no Compilation starts\. Preserve owner material and correct only the reported root precondition or choose an absent path/, - ); - const normalizedRecovery = recovery.replace(/\s+/g, " "); - assert.match( - normalizedRecovery, - /CLI emits only state-changing lines.*?prefixes every line with `First Draft: `/, - ); - assert.match( - normalizedRecovery, - /non-null `retry_at` identifies the exact scheduled time to report/, - ); - assert.match( - normalizedRecovery, - /positive `retry_count`.*?null `retry_at` means automatic work is parked and needs operator attention/, - ); - assert.match( - normalizedRecovery, - /Zero with both nullable fields null means no current wait or safe reason is projected/, - ); - assert.match( - normalizedRecovery, - /Publication follow is bounded to ten minutes.*?Four minutes alone remains inside that window.*?timeout stops only the current invocation's wait, not retained work/, - ); - assert.match( - normalizedRecovery, - /While one `plan compile` invocation polls it, do not launch a concurrent Compile.*?invocation that reached that retained Publication exits.*?Publication-phase outcome unknown, status unavailable, wait timeout.*?wait for it to exit.*?conditional singleton PUT is the documented reconciliation path.*?exception does not apply to an outcome-unknown Plan push.*?no Plan GET.*?There is no separate public Publication status command/, - ); - assert.match( - normalizedRecovery, - /`invalid_publication_status` is different: unchanged replay cannot repair its protocol mismatch.*?reconcile the coordinated CLI\/service versions first/, - ); - assert.match( - skill.replace(/\s+/g, " "), - /`--github` success prints only the repository URL.*?never recover one from private state or unvalidated output/, - ); - assert.match( - recovery, - /reserves standard output for one validated private GitHub repository URL on success[\s\S]*?Do not call a nonterminal GitHub phase "still compiling"/, - ); - assert.match( - recovery, - /end standard error with exactly one JSON object[\s\S]*?Remove only one leading contiguous block of complete lines[\s\S]*?exact `First Draft: ` prefix[\s\S]*?any other prefix or suffix,[\s\S]*?interleaved output fail closed/, - ); - for (const field of ["phase", "retry_at", "retry_count", "reason_code"]) { - assert(recovery.includes(`| \`${field}\` |`)); - } - for (const phase of [ - "compiling", - "preparing_repository", - "github_preflight", - "creating_repository", - "preparing_repository_reconciliation", - "reconciling_repository", - "preparing_artifact", - "publishing_artifact", - "preparing_publication_reconciliation", - "reconciling_publication", - "completed", - "failed", - "cancelled", - ]) { - assert(recovery.includes(`\`${phase}\``), `missing Publication phase ${phase}`); - } - const reasonAllowlist = recovery.match( - /The reason-code allowlist is ([\s\S]*?)\.\n\nA non-null `retry_at`/, - ); - assert(reasonAllowlist, "missing safe Publication reason allowlist"); - assert.deepEqual( - [...reasonAllowlist[1].matchAll(/`(github\.[a-z._]+)`/g)].map( - ([, reason]) => reason, - ), - safeGithubReasonCodes, - ); - for (const message of [ - "Analyzing Foundation Plan...", - "Foundation Plan analysis valid.", - "Compiling application...", - "Application compiled.", - "Application compilation failed.", - "Application compilation cancelled.", - "Preparing private GitHub repository...", - "Checking GitHub access...", - "Checking GitHub access (reason: CODE; retry count: N; next retry: TIMESTAMP).", - "Checking GitHub access (reason: CODE; retry count: N; automatic retries paused; operator recovery required).", - "Creating private GitHub repository...", - "Preparing to verify GitHub repository creation...", - "Verifying GitHub repository creation...", - "Preparing compiled application...", - "Publishing compiled application to GitHub...", - "Preparing to verify GitHub publication...", - "Verifying GitHub publication...", - "GitHub publication complete.", - "GitHub publication failed.", - "GitHub publication cancelled.", - ]) { - assert(recovery.includes(`\`${message}\``), `missing progress message ${message}`); - } - assert.match( - recovery, - /emits only state-changing lines, suppresses consecutive duplicate text, and prefixes every line with\s+`First Draft: `/, - ); - assert.match( - recovery, - /HTTP status\s+by itself does not prove that an account lacks provisioning or that an endpoint does not exist[\s\S]*?not a\s+basis for a support recommendation/, - ); - assert.match( - recovery, - /`invalid_publication_status` also leaves the singleton result unverified[\s\S]*?retrying unchanged cannot repair a\s+protocol mismatch[\s\S]*?reconcile compatible CLI\/service versions/, - ); - assert.match( - recovery, - /`publication_start_rejected` is a validated non-timeout 4xx result and establishes only that Publication success was\s+not verified[\s\S]*?does not establish whether this request reached the service's start boundary,[\s\S]*?left retained or remote work,[\s\S]*?rejection alone authorizes no replay, concurrent Compile, or direct mutation[\s\S]*?A 408 or 5xx response to the start\s+request is not this family[\s\S]*?outcome as unknown/, - ); - assert.match( - recovery, - /`publication_failed` and `publication_cancelled` are terminal[\s\S]*?inspect\s+`current\.compilation\.status` before reporting the failed stage[\s\S]*?failed or cancelled Compilation means GitHub work\s+was not reached[\s\S]*?When Compilation succeeded,[\s\S]*?remote processing may have left a repository or commit/, - ); - assert.match( - recovery, - /`github\.preflight_unclassified` says only that a retained legacy retry had no classified\s+reason[\s\S]*?`github\.preflight_unavailable\.\*` fallback identifies the coarse pre-claim stage[\s\S]*?does not expose the exception or establish a provider cause/, - ); - assert.match( - recovery, - /failed or cancelled Compilation\s+means GitHub Publication work was not reached[\s\S]*?failed or cancelled Publication paired with a succeeded Compilation\s+is a later GitHub delivery outcome/, - ); - const schemaRepair = evaluation("repair-local-schema-diagnostic"); hasExpectation(schemaRepair, "instancePath", "application.key"); hasExpectation(schemaRepair, "one complete parseable Plan snapshot"); @@ -4342,64 +2622,6 @@ test("recovery evals stage and preserve existing Plan state", async () => { ); } - const recoveryReference = await readFile( - path.join( - skillsDirectory, - "create-full-stack-app", - "references", - "diagnostics-and-recovery.md", - ), - "utf8", - ); - const skillSource = await readFile( - path.join(skillsDirectory, "create-full-stack-app", "SKILL.md"), - "utf8", - ); - const recoverySection = skillSource.match( - /## Recover from failures([\s\S]*?)## Hand off the result/, - ); - assert(recoverySection, "SKILL.md: missing recovery section"); - const normalizedRecoverySection = recoverySection[1].replace(/\s+/g, " "); - assert.match( - normalizedRecoverySection, - /Branch on its stable `error` and structured fields, not the human-readable `detail`/, - ); - assert.match( - recoveryReference.replace(/\s+/g, " "), - /`422 server_rejected` binds them to `response.source_sha256`.*?submitted bytes.*?Correct a well-founded source problem while preserving unrelated meaning and subject identity.*?submit an incomplete, invalid, or unchanged draft again/, - ); - assert.match( - recoverySection[1], - /\[stable error family\]\(references\/diagnostics-and-recovery\.md#stable-error-families\)/, - ); - assert.match(recoveryReference, /CLI contract configuration.*script\/cli-contract\/config\.mjs/); - const stableErrors = recoveryReference.match( - /## Stable error families([\s\S]*?)## Ambiguous mutations/, - ); - assert(stableErrors, "diagnostics reference: missing stable error families"); - for (const code of planPushErrorCodes) { - assert( - stableErrors[1].includes(`| \`${code}\` |`), - `diagnostics reference: missing plan push error ${code}`, - ); - } - assert.match( - recoveryReference, - /Branch on the object's stable `error` and structured fields rather than the\s+human-readable `detail`/, - ); - assert.match( - recoveryReference, - /`local_state_not_saved` is the only handled envelope that can include private `recovery_state`/, - ); - assert.match( - recoveryReference, - /Unknown, absent, malformed, or additional output after removing only recognized complete `First Draft: ` lines is\s+not a trusted recovery envelope/, - ); - assert.doesNotMatch( - recoveryReference, - /The Plan may have been accepted; local state was not changed\./, - ); - const evaluationsByError = { authentication_required: "authentication-required-stop", invalid_arguments: "invalid-push-arguments", @@ -4420,18 +2642,6 @@ test("recovery evals stage and preserve existing Plan state", async () => { `${id}: missing error-code branch expectation`, ); } - assert.doesNotMatch( - recoveryReference, - /Could not read the local First Draft Plan or state\. No network request was made\./, - ); - assert.match( - recoveryReference, - /Let the user configure `FIRSTDRAFT_API_TOKEN` outside the conversation[\s\S]*?Do not request its value, print it, place\s+it on a command line, or persist it in project files/, - ); - assert.match( - recoveryReference, - /user confirms authentication is configured[\s\S]*?resume[\s\S]*?already requested operation without asking for fresh authorization/, - ); const authenticationEvaluation = cases.find( ({ id }) => id === "authentication-required-stop", ); @@ -4488,45 +2698,6 @@ test("initialization recovery consumes the prepared CLI error envelope", async ( const cases = JSON.parse( await readFile(path.join(evaluationDirectory, "cases.json"), "utf8"), ).cases; - const skillSource = await readFile( - path.join(skillsDirectory, "create-full-stack-app", "SKILL.md"), - "utf8", - ); - const recoveryReference = await readFile( - path.join( - skillsDirectory, - "create-full-stack-app", - "references", - "diagnostics-and-recovery.md", - ), - "utf8", - ); - const initializationSection = skillSource.match( - /## Initialize or resume the local Plan([\s\S]*?)## Interview and author incrementally/, - ); - const initializationReference = recoveryReference.match( - /## Local state and credentials([\s\S]*?)## Push and analysis/, - ); - - assert(initializationSection, "SKILL.md: missing initialization section"); - assert(initializationReference, "diagnostics reference: missing initialization boundary"); - assert.match( - initializationSection[1], - /If initialization\s+fails,\s+follow the stable error in the recovery reference[\s\S]*?Preserve any partial `\.firstdraft\/`\s+directory/, - ); - assert.match( - initializationSection[1], - /If `\.firstdraft\/` already exists,[\s\S]*?confirm with project-relative metadata[\s\S]*?regular and readable/, - ); - assert.match( - initializationReference[1], - /`invalid_arguments`[\s\S]*?No local files were written[\s\S]*?`local_initialization_failed`[\s\S]*?may be incomplete/, - ); - assert.match( - initializationReference[1], - /An existing `\.firstdraft\/` is not disposable scratch space[\s\S]*?do not\s+reinitialize over partial, damaged, or existing state/, - ); - const hasExpectation = (evaluation, fragment) => evaluation.expectations.some((expectation) => expectation.includes(fragment), @@ -4827,38 +2998,6 @@ async function filesUnder( return files; } -function renderEvidenceStateNames(names) { - assert(Array.isArray(names), "evidence state names must be an array"); - assert( - names.every((name) => typeof name === "string" && name.length > 0), - "evidence state names must contain only nonempty strings", - ); - return names.length > 0 - ? names.map((name) => `\`${name}\``).join(", ") - : "(none)"; -} - -function assertEvidenceStatePresenceBlock(source, expectedBlock) { - const section = source.match( - /The pre-smoke presence\s+summary was exactly:\n\n((?:- [^\n]+\n)+)/, - ); - assert( - section, - "packaging evidence must render the canonical state-presence block", - ); - assert.equal( - section[1], - `${expectedBlock}\n`, - "packaging evidence state-presence bullets differ from the observation", - ); -} - -function revisionTokens(source) { - return [ - ...new Set(source.match(/\b(?:[0-9a-f]{40}|[0-9a-f]{7})\b/g) ?? []), - ].sort(); -} - function workflowJobSource(source, name) { const marker = `\n ${name}:\n`; const start = source.indexOf(marker); @@ -4878,90 +3017,6 @@ function workflowJobSource(source, name) { return source.slice(start, end); } -function assertRevisionTokens(source, expected) { - assert.deepEqual(revisionTokens(source), [...new Set(expected)].sort()); -} - -function pluginRuntimeDigestAtRevision(revision) { - const relativePaths = gitTreePathsAtRevision( - revision, - ".claude-plugin", - "skills/create-full-stack-app", - ).filter( - (relativePath) => - /^\.claude-plugin\/[^/]+\.json$/.test(relativePath) || - relativePath.startsWith("skills/create-full-stack-app/"), - ); - const digest = createHash("sha256"); - for (const relativePath of relativePaths) { - const source = gitBlobAtRevision(revision, relativePath); - const pathLength = Buffer.alloc(4); - pathLength.writeUInt32BE(Buffer.byteLength(relativePath)); - const sourceLength = Buffer.alloc(8); - sourceLength.writeBigUInt64BE(BigInt(source.length)); - digest.update(pathLength); - digest.update(relativePath); - digest.update(sourceLength); - digest.update(source); - } - return digest.digest("hex"); -} - -function gitBlobAtRevision(revision, relativePath) { - const blob = spawnSync("git", ["show", `${revision}:${relativePath}`], { - cwd: repository, - encoding: "buffer", - maxBuffer: 10 * 1024 * 1024, - }); - assert.equal( - blob.status, - 0, - `git show failed for ${revision}:${relativePath}: ` + - spawnBufferText(blob.stderr), - ); - return blob.stdout; -} - -function gitTreeAtRevision(revision) { - const tree = spawnSync("git", ["rev-parse", `${revision}^{tree}`], { - cwd: repository, - encoding: "utf8", - }); - assert.equal( - tree.status, - 0, - `git rev-parse failed for ${revision}: ${tree.stderr.trim()}`, - ); - return tree.stdout.trim(); -} - -function gitTreePathsAtRevision(revision, ...roots) { - const tree = spawnSync( - "git", - [ - "ls-tree", - "-r", - "--name-only", - "-z", - revision, - "--", - ...roots, - ], - { cwd: repository, encoding: "buffer" }, - ); - assert.equal( - tree.status, - 0, - `git ls-tree failed for ${revision}: ${spawnBufferText(tree.stderr)}`, - ); - const relativePaths = spawnBufferText(tree.stdout) - .split("\0") - .filter(Boolean) - .sort(); - assert(relativePaths.length > 0, `no Git tree paths found for ${revision}`); - return relativePaths; -} - function trackedFiles() { const result = spawnSync("git", ["ls-files", "-z"], { cwd: repository,