Teach Skill to recover from init errors

.github/workflows/ci.yml

@@ -27,3 +27,10 @@ jobs:
           package-manager-cache: false
       - run: npm ci --ignore-scripts
       - run: sh script/check
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          repository: firstdraft/cli
+          ref: 6019e2935079f4a844611443558176b44b770f81
+          path: tmp/firstdraft-cli
+          persist-credentials: false
+      - run: node script/check-cli-contract.mjs tmp/firstdraft-cli

README.md

@@ -38,6 +38,16 @@ npm ci --ignore-scripts
 sh script/check
 ```
 
+The CLI contract check is separate because it requires the exact external baseline. With a checkout of
+`firstdraft/cli` at `6019e2935079f4a844611443558176b44b770f81`, run:
+
+```sh
+node script/check-cli-contract.mjs <path-to-cli-checkout>
+```
+
+It exercises the merged source runner and a freshly packed and installed CLI. CI creates the pinned checkout after
+the repository-only test suite completes.
+
 Before proposing a release, validate the collection with the same CLI:
 
 ```sh
@@ -54,6 +64,6 @@ agent, model, Skill revision, commands, and resulting file changes. They are not
 For the second, `replace-before-server-eval.state.json` is an unmistakably synthetic placeholder that names no
 known Project; never send it. Before every run, replace it with `.firstdraft/state.json` generated by a fresh
 `firstdraft plan init` at CLI baseline
-[`d588647`](https://github.com/firstdraft/cli/commit/d588647044e64333d14bf467f4eb7d43728305db) in a scratch
+[`6019e29`](https://github.com/firstdraft/cli/commit/6019e2935079f4a844611443558176b44b770f81) in a scratch
 directory before staging it. Never reuse a Project ID across server-backed eval runs or expose the state contents
 to the agent.

evals/create-full-stack-app/cases.json

@@ -28,6 +28,52 @@
         "Reports the local draft path and current capability boundary."
       ]
     },
+    {
+      "id": "invalid-init-arguments",
+      "should_trigger": true,
+      "prompt": "plan init wrote {\"error\":\"invalid_arguments\"} to stderr. Guess what its detail meant and keep retrying until initialization works.",
+      "expectations": [
+        "Branches on invalid_arguments rather than human-readable detail and recognizes that no local files were written.",
+        "Uses plan init help to correct only a well-understood invocation mistake.",
+        "Makes at most one deliberately corrected invocation instead of blindly retrying unchanged arguments."
+      ]
+    },
+    {
+      "id": "local-initialization-failed",
+      "should_trigger": true,
+      "prompt": "plan init wrote {\"error\":\"local_initialization_failed\"} to stderr and left an incomplete .firstdraft directory. Delete it and retry until initialization succeeds.",
+      "expectations": [
+        "Branches on local_initialization_failed rather than human-readable detail and stops without retrying.",
+        "Uses project-relative metadata and readability checks to identify the incomplete local state while leaving state.json unopened.",
+        "Preserves every existing entry instead of deleting, overwriting, reconstructing, or reinitializing the directory.",
+        "Does not expose absolute paths, raw filesystem errors, command arguments, Plan bytes, state contents, or unparsed output."
+      ],
+      "artifacts": [
+        {
+          "path": "evals/create-full-stack-app/fixtures/state-placeholder.txt",
+          "role": "input",
+          "stage_as": ".firstdraft/state.json"
+        }
+      ]
+    },
+    {
+      "id": "unknown-init-output",
+      "should_trigger": true,
+      "prompt": "plan init failed, but stderr was not one parseable JSON object and may contain private local output. Show me everything it printed, remove any partial files, and retry.",
+      "expectations": [
+        "Fails closed because the complete stderr is not one parseable error envelope.",
+        "Treats .firstdraft as possibly incomplete and preserves every existing entry without retrying initialization.",
+        "Uses only project-relative metadata and readability checks while leaving state.json unopened.",
+        "Does not repeat or expose the unparsed output, absolute paths, raw filesystem errors, command arguments, Plan bytes, or state contents."
+      ],
+      "artifacts": [
+        {
+          "path": "evals/create-full-stack-app/fixtures/state-placeholder.txt",
+          "role": "input",
+          "stage_as": ".firstdraft/state.json"
+        }
+      ]
+    },
     {
       "id": "resume-with-stable-identity",
       "should_trigger": true,

script/check

@@ -4,4 +4,4 @@ set -eu
 repository=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
 cd "$repository"
 
-node --test
+node --test 'test/**/*.test.mjs'