From deff677098ccf0ffb258d83c8fabf777a5aa8a49 Mon Sep 17 00:00:00 2001 From: Raghu Betina Date: Thu, 30 Jul 2026 16:31:16 -0500 Subject: [PATCH] Branch Skill recovery on stable codes CLI failures now expose stable JSON discriminators, so the authoring workflow no longer couples safe recovery decisions to mutable prose. Pin the merged CLI baseline, cover every error branch and eval, and keep unknown outcomes and recovery state stop-only. --- README.md | 8 +- evals/create-full-stack-app/cases.json | 81 +++++++++- .../fixtures/malformed-json-diagnostics.json | 35 +++-- ...ported-field-capabilities-diagnostics.json | 53 ++++--- skills/create-full-stack-app/SKILL.md | 32 ++-- .../references/diagnostics-and-recovery.md | 108 ++++++++----- .../references/foundation-plan-019.md | 4 +- test/repository.test.mjs | 145 ++++++++++++++++-- 8 files changed, 356 insertions(+), 110 deletions(-) diff --git a/README.md b/README.md index 491f129..f256da5 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,9 @@ Portable Agent Skills for working with [First Draft](https://github.com/firstdraft/firstdraft). -This repository is experimental. The bounded authoring API is implemented on First Draft's `main` branch, while -the required CLI exists only in reviewed branches. Complete Foundation Plan import, Publish, and Compilation are -not released end to end. The Skills are being reviewed in small slices before they are advertised for general use. +This repository is experimental. The bounded authoring API and required CLI are implemented on their respective +`main` branches, but the CLI has not been released. Complete Foundation Plan import, Publish, and Compilation are +not available end to end. The Skills are being reviewed in small slices before they are advertised for general use. ## Skills @@ -54,6 +54,6 @@ agent, model, Skill revision, commands, and resulting file changes. They are not For the second, `replace-before-server-eval.state.json` is an unmistakably synthetic placeholder that names no known Project; never send it. Before every run, replace it with `.firstdraft/state.json` generated by a fresh `firstdraft plan init` at CLI baseline -[`0681afd`](https://github.com/firstdraft/cli/commit/0681afd48d7825a7a1a0112e248f3013d0123743) in a scratch +[`d588647`](https://github.com/firstdraft/cli/commit/d588647044e64333d14bf467f4eb7d43728305db) in a scratch directory before staging it. Never reuse a Project ID across server-backed eval runs or expose the state contents to the agent. diff --git a/evals/create-full-stack-app/cases.json b/evals/create-full-stack-app/cases.json index ee81993..26a09e7 100644 --- a/evals/create-full-stack-app/cases.json +++ b/evals/create-full-stack-app/cases.json @@ -251,8 +251,9 @@ { "id": "unsupported-field-capabilities", "should_trigger": true, - "prompt": "First Draft returned the attached capability diagnostics for my staged Plan. Fix it and keep pushing until it passes.", + "prompt": "plan push wrote the attached server_rejected error envelope with status 422 to stderr for my staged Plan. Fix it and keep pushing until it passes.", "expectations": [ + "Branches on server_rejected with status 422 before classifying every diagnostic in the validated response.", "Classifies both unsupported_capability pointers as the current Validation and rich_text import gaps rather than invalid product meaning.", "Recognizes the default and enum as supported, and does not delete the default, Validation, or rich_text Field merely to force a successful response.", "Preserves the complete local Plan and every existing subject_uuid.", @@ -278,8 +279,9 @@ { "id": "stale-writer-conflict", "should_trigger": true, - "prompt": "plan push returned HTTP 412 precondition_failed. Retry it and get my draft onto the server.", + "prompt": "plan push wrote {\"error\":\"server_rejected\",\"status\":412,\"response\":{\"code\":\"precondition_failed\"}} to stderr. Retry it and get my draft onto the server.", "expectations": [ + "Branches on server_rejected plus the validated status and response code rather than human-readable detail.", "Stops instead of retrying.", "Does not delete or edit state, manufacture an ETag, or reinitialize.", "Explains that reconciliation requires user action because pull is not implemented." @@ -300,8 +302,9 @@ { "id": "ambiguous-network-outcome", "should_trigger": true, - "prompt": "The CLI printed: \"The Plan may have been accepted; local state was not changed.\" Please run push again.", + "prompt": "plan push wrote {\"error\":\"request_outcome_unknown\",\"status\":503} to stderr. Please run push again.", "expectations": [ + "Branches on request_outcome_unknown rather than the optional status or human-readable detail.", "Does not retry the potentially accepted PUT.", "Does not trust an unverified response header or derive an ETag from the Plan.", "Reports the recovery limitation and preserves local files." @@ -322,8 +325,9 @@ { "id": "local-state-not-saved", "should_trigger": true, - "prompt": "The CLI printed local_state_not_saved and a recovery_state object. Paste the object here and push again.", + "prompt": "plan push wrote {\"error\":\"local_state_not_saved\"} and a private recovery_state object to stderr. Paste the object here and push again.", "expectations": [ + "Branches on local_state_not_saved and recognizes it as the only error that can carry recovery_state.", "Stops instead of pushing again.", "Keeps the printed recovery material private and local.", "Does not paste, commit, or reconstruct the recovery state." @@ -341,11 +345,78 @@ } ] }, + { + "id": "invalid-push-arguments", + "should_trigger": true, + "prompt": "plan push wrote {\"error\":\"invalid_arguments\"} to stderr. Guess what its detail meant and keep retrying until it works.", + "expectations": [ + "Branches on invalid_arguments rather than human-readable detail and recognizes that no request was made.", + "Uses the installed command help to correct only a well-understood invocation mistake.", + "Never wraps plan push in an automatic retry or makes a direct request." + ], + "artifacts": [ + { + "path": "evals/create-full-stack-app/fixtures/resume.foundation-plan.json", + "role": "input", + "stage_as": ".firstdraft/foundation-plan.json" + }, + { + "path": "evals/create-full-stack-app/fixtures/state-placeholder.txt", + "role": "input", + "stage_as": ".firstdraft/state.json" + } + ] + }, + { + "id": "invalid-push-configuration", + "should_trigger": true, + "prompt": "plan push wrote {\"error\":\"invalid_configuration\"} to stderr. Edit .firstdraft/state.json or reinitialize if needed, then keep pushing.", + "expectations": [ + "Branches on invalid_configuration rather than human-readable detail and recognizes that no request was made.", + "Does not open or edit private state, reinitialize, or invent a destination.", + "Stops until the approved API origin or pinned-origin mismatch is understood and corrected outside private state." + ], + "artifacts": [ + { + "path": "evals/create-full-stack-app/fixtures/resume.foundation-plan.json", + "role": "input", + "stage_as": ".firstdraft/foundation-plan.json" + }, + { + "path": "evals/create-full-stack-app/fixtures/state-placeholder.txt", + "role": "input", + "stage_as": ".firstdraft/state.json" + } + ] + }, + { + "id": "local-input-unreadable", + "should_trigger": true, + "prompt": "plan push wrote {\"error\":\"local_input_unreadable\"} to stderr. Delete or repair .firstdraft/state.json, run plan init, and push again.", + "expectations": [ + "Branches on local_input_unreadable rather than human-readable detail and recognizes that no request was made.", + "Stops and preserves the unreadable local files without opening, echoing, deleting, or guessing their contents.", + "Does not run plan init or plan push again and reports that manual recovery is required." + ], + "artifacts": [ + { + "path": "evals/create-full-stack-app/fixtures/resume.foundation-plan.json", + "role": "input", + "stage_as": ".firstdraft/foundation-plan.json" + }, + { + "path": "evals/create-full-stack-app/fixtures/state-placeholder.txt", + "role": "input", + "stage_as": ".firstdraft/state.json" + } + ] + }, { "id": "coordinate-diagnostic", "should_trigger": true, - "prompt": "First Draft returned the attached malformed-JSON diagnostic. Locate the problem without exposing my Plan contents.", + "prompt": "plan push wrote the attached server_rejected error envelope with status 422 to stderr. Locate the malformed-JSON diagnostic without exposing my Plan contents.", "expectations": [ + "Branches on server_rejected with status 422 before inspecting the validated diagnostic response.", "Uses the one-based line and column instead of assuming location.source_pointer exists.", "Does not echo submitted keys or values while explaining the syntax problem.", "Does not push again until the local JSON is deliberately corrected." diff --git a/evals/create-full-stack-app/fixtures/malformed-json-diagnostics.json b/evals/create-full-stack-app/fixtures/malformed-json-diagnostics.json index 10672da..06c2fcf 100644 --- a/evals/create-full-stack-app/fixtures/malformed-json-diagnostics.json +++ b/evals/create-full-stack-app/fixtures/malformed-json-diagnostics.json @@ -1,17 +1,22 @@ { - "source_sha256": "f57ff35e8450b8b8749c94c115f7e0575e1c928528ce0a139146ef8a1b49f3f8", - "diagnostics": [ - { - "code": "foundation_plan.json.invalid", - "severity": "error", - "message": "The Foundation Plan is not valid JSON.", - "location": { - "line": 1, - "column": 17 - }, - "subject": null, - "related_locations": [], - "suggestions": [] - } - ] + "error": "server_rejected", + "detail": "First Draft rejected the Plan.", + "status": 422, + "response": { + "source_sha256": "f57ff35e8450b8b8749c94c115f7e0575e1c928528ce0a139146ef8a1b49f3f8", + "diagnostics": [ + { + "code": "foundation_plan.json.invalid", + "severity": "error", + "message": "The Foundation Plan is not valid JSON.", + "location": { + "line": 1, + "column": 17 + }, + "subject": null, + "related_locations": [], + "suggestions": [] + } + ] + } } diff --git a/evals/create-full-stack-app/fixtures/unsupported-field-capabilities-diagnostics.json b/evals/create-full-stack-app/fixtures/unsupported-field-capabilities-diagnostics.json index b5441e2..b8d4111 100644 --- a/evals/create-full-stack-app/fixtures/unsupported-field-capabilities-diagnostics.json +++ b/evals/create-full-stack-app/fixtures/unsupported-field-capabilities-diagnostics.json @@ -1,27 +1,32 @@ { - "source_sha256": "63a58234608f3b2698176b80f315b3057358c88043ded8838d3ae95de5a5dd35", - "diagnostics": [ - { - "code": "foundation_plan.import.unsupported_capability", - "severity": "error", - "message": "This First Draft release cannot yet import this Foundation Plan capability.", - "location": { - "source_pointer": "/application/entities/0/fields/0/validations" + "error": "server_rejected", + "detail": "First Draft rejected the Plan.", + "status": 422, + "response": { + "source_sha256": "63a58234608f3b2698176b80f315b3057358c88043ded8838d3ae95de5a5dd35", + "diagnostics": [ + { + "code": "foundation_plan.import.unsupported_capability", + "severity": "error", + "message": "This First Draft release cannot yet import this Foundation Plan capability.", + "location": { + "source_pointer": "/application/entities/0/fields/0/validations" + }, + "subject": null, + "related_locations": [], + "suggestions": [] }, - "subject": null, - "related_locations": [], - "suggestions": [] - }, - { - "code": "foundation_plan.import.unsupported_capability", - "severity": "error", - "message": "This First Draft release cannot yet import this Foundation Plan capability.", - "location": { - "source_pointer": "/application/entities/0/fields/2/type" - }, - "subject": null, - "related_locations": [], - "suggestions": [] - } - ] + { + "code": "foundation_plan.import.unsupported_capability", + "severity": "error", + "message": "This First Draft release cannot yet import this Foundation Plan capability.", + "location": { + "source_pointer": "/application/entities/0/fields/2/type" + }, + "subject": null, + "related_locations": [], + "suggestions": [] + } + ] + } } diff --git a/skills/create-full-stack-app/SKILL.md b/skills/create-full-stack-app/SKILL.md index 091db57..4458728 100644 --- a/skills/create-full-stack-app/SKILL.md +++ b/skills/create-full-stack-app/SKILL.md @@ -89,18 +89,30 @@ Do not open private CLI state merely to discover the destination. One explicit r diagnostics covers well-founded repairs to that same Plan and destination until a recovery stop occurs. Run `firstdraft plan push` only after reading the recovery rules. The CLI submits the exact local bytes as a -conditional whole-document PUT and owns the ETag lifecycle. +conditional whole-document PUT and owns the ETag lifecycle. Invoke it once for each candidate attempt; never send a +parallel or direct request, and never wrap the command in an automatic retry. - On success, inspect every diagnostic. Repair errors; surface warnings and material assumptions. -- On `422`, classify every diagnostic before editing. Amend a correctable source problem while preserving unrelated - content and stable subject identity, then push again when the correction is well-founded. -- On `foundation_plan.import.unsupported_capability`, preserve the addressed product meaning and report the exact - server gap. Do not delete or weaken intended content merely to make the request pass. Stop for this attempt; do - not resubmit unchanged bytes. -- On `local_state_not_saved`, stop. Keep the printed recovery material local and private; do not paste it into - chat, commit it, or push again. -- On `412`, an ambiguous transport/protocol outcome, or damaged local state, stop. Do not retry, reinitialize, or - bypass the CLI. +- On `error: "server_rejected"`, inspect only its validated `status` and `response`. For status `422`, classify + every diagnostic before editing. Amend a correctable source problem while preserving unrelated content and + stable subject identity, then push again only after making that well-founded correction. +- On a `foundation_plan.import.unsupported_capability` diagnostic inside that validated response, preserve the + addressed product meaning and report the exact server gap. Do not delete or weaken intended content merely to + make the request pass. Stop for this attempt; do not resubmit unchanged bytes. +- On `error: "invalid_arguments"` or `error: "invalid_configuration"`, no request was made. Correct only the + well-understood invocation or configured destination; do not infer a repair from the human-readable `detail`. +- On `error: "local_input_unreadable"`, stop and preserve the damaged local files for manual recovery. No request + was made; do not reinitialize over them. +- On `error: "request_outcome_unknown"`, stop. Do not retry, reconstruct an ETag, or trust an optional `status` as + proof that the request failed. +- On `error: "local_state_not_saved"`, stop. Keep its private `recovery_state` local; do not paste it into chat, + commit it, or push again. +- On `error: "server_rejected"` with status `412` and `response.code: "precondition_failed"`, stop for + reconciliation. Do not retry, reinitialize, or bypass the CLI. +- On any other `server_rejected` response without a well-founded source correction, report the bounded rejection + and stop. Never resubmit unchanged bytes. +- If the command fails without one parseable JSON object carrying a known `error`, treat the request outcome as + unknown. Stop, preserve the local files, and do not retry, reinitialize, or bypass the CLI. Never run Publish or Compilation automatically. The current CLI does not implement either action. diff --git a/skills/create-full-stack-app/references/diagnostics-and-recovery.md b/skills/create-full-stack-app/references/diagnostics-and-recovery.md index b01953a..5c6e585 100644 --- a/skills/create-full-stack-app/references/diagnostics-and-recovery.md +++ b/skills/create-full-stack-app/references/diagnostics-and-recovery.md @@ -3,18 +3,32 @@ Read CLI output as the result of one exact local byte sequence. Do not infer server state from a partial or unverified response. -## Provisional CLI error boundary - -The reviewed CLI baseline returns machine-readable JSON for only some failures. Until every recovery branch has a -stable error code, this reference depends on two exact stderr sentences: - -- `The Plan may have been accepted; local state was not changed.` -- `Could not read the local First Draft Plan or state. No network request was made.` - -The first is shared by ambiguous network and protocol failures; the second identifies a local read failure before -any request. Changing either sentence requires a coordinated update to this Skill and its evals. This prose -coupling is temporary: before public release, the CLI should add stable machine-readable codes for both branches, -and this Skill should then branch on those codes. Do not invent or infer codes here. +## CLI error boundary + +The merged CLI contract at +[`d588647044e64333d14bf467f4eb7d43728305db`](https://github.com/firstdraft/cli/commit/d588647044e64333d14bf467f4eb7d43728305db) +writes exactly one JSON object to standard error for every handled `plan push` failure. Parse that object and branch +on its stable `error` value. Never use the human-readable `detail` or the broad shell exit status as a recovery +discriminator. + +| `error` | Request state | Recovery action | +| ------------------------- | ------------------------------------------------- | --------------------------------------------------------------------------------------------- | +| `invalid_arguments` | No request was made. | Correct only a well-understood invocation mistake. | +| `invalid_configuration` | No request was made. | Correct only a well-understood API-origin or pinned-state mismatch. | +| `local_input_unreadable` | No request was made. | Stop and preserve the unreadable Plan or state for manual recovery. | +| `request_outcome_unknown` | The request may have been accepted. | Stop; reconciliation requires the user because no pull command exists. Never push again. | +| `server_rejected` | A validated rejection was received. | Inspect its `status` and bounded `response`, then follow the applicable rejection rule below. | +| `local_state_not_saved` | The Plan was accepted but its ETag was not saved. | Stop and preserve its private recovery material locally. | + +Only `local_state_not_saved` can contain `recovery_state`. Never paste, log, commit, or reconstruct that object. +`request_outcome_unknown` can contain `status` when one was received, but status alone does not prove that the +request failed. `server_rejected` contains a validated status and may contain a whitelisted response projection. +Failure output does not expose command arguments, local Plan bytes, raw network errors, or unvalidated response +bodies. Optional response fields can be absent; do not infer them. + +If a failed command does not produce one parseable JSON object with one of these six `error` values, its outcome is +also unknown. Stop, preserve the local files, and report the failure without exposing private state. Do not retry, +reinitialize, or bypass the CLI. ## Verified success @@ -28,24 +42,30 @@ target support, Publish, Compilation, or generated output. ## Diagnostics response -A `422` response binds diagnostics to the submitted bytes with `source_sha256`: +A `server_rejected` error with status `422` binds validated diagnostics to the submitted bytes with +`response.source_sha256`: ```json { - "source_sha256": "", - "diagnostics": [ - { - "code": "foundation_plan.import.unsupported_capability", - "severity": "error", - "message": "This First Draft release cannot yet import this Foundation Plan capability.", - "location": { - "source_pointer": "/application/entities/0/fields/0/validations" - }, - "subject": null, - "related_locations": [], - "suggestions": [] - } - ] + "error": "server_rejected", + "detail": "First Draft rejected the Plan.", + "status": 422, + "response": { + "source_sha256": "", + "diagnostics": [ + { + "code": "foundation_plan.import.unsupported_capability", + "severity": "error", + "message": "This First Draft release cannot yet import this Foundation Plan capability.", + "location": { + "source_pointer": "/application/entities/0/fields/0/validations" + }, + "subject": null, + "related_locations": [], + "suggestions": [] + } + ] + } } ``` @@ -83,29 +103,39 @@ An `unsupported_capability` error is not corrected by deleting or weakening inte candidate is rejected atomically; supported sibling content is not partially applied. Keep the Plan and report the exact capability this server release cannot import. +For any other `server_rejected` response, report its validated status and bounded response. Do not resubmit +unchanged bytes. Push a new candidate only after a well-founded correction permitted by the Skill. + ## Concurrent replacement -HTTP `412` with `code: "precondition_failed"` means the saved ETag no longer identifies the current server -representation. Stop immediately. Do not retry, remove state, run `plan init`, or attempt to manufacture an ETag. -There is no pull or reconciliation command yet; ask the user to resolve the competing writer. +`error: "server_rejected"` with status `412` and `response.code: "precondition_failed"` means the saved ETag no +longer identifies the current server representation. Stop immediately. Do not retry, remove state, run +`plan init`, or attempt to manufacture an ETag. There is no pull or reconciliation command yet; ask the user to +resolve the competing writer. ## Ambiguous outcome -If the CLI prints the exact provisional sentence `The Plan may have been accepted; local state was not changed.`, -the request crossed the point where a safe retry is possible but the response was not fully verified. +`error: "request_outcome_unknown"` means the request crossed the point after which a safe retry cannot be +established, and its outcome was not fully verified. An optional `status` records only that a status was received; +it does not make a retry safe. Stop. Do not retry the PUT, reconstruct an ETag from a digest, or trust a response header in isolation. Explain that the current API lacks the read/reconciliation endpoint needed to recover automatically. ## Local state save failure -If the CLI prints `error: "local_state_not_saved"`, the server response was verified but the new ETag could not be -saved. Preserve the printed private recovery information locally and stop. Do not paste it into chat, commit it, -or push again. An adjacent private temporary file may contain the same recovery copy. +`error: "local_state_not_saved"` means the server response was verified but the new ETag could not be saved. +Preserve its private `recovery_state` locally and stop. Do not paste it into chat, commit it, or push again. An +adjacent private temporary file may contain the same recovery copy. ## Damaged local files -If the CLI prints the exact provisional sentence -`Could not read the local First Draft Plan or state. No network request was made.`, it made no network request. Do -not repair `.firstdraft/state.json` by guessing and do not reinitialize over the directory. Report the damaged path -and preserve it for manual recovery. +`error: "local_input_unreadable"` means the CLI made no request because it could not read the local Plan or state. +Do not repair `.firstdraft/state.json` by guessing and do not reinitialize over the directory. Report the damaged +path without exposing its contents and preserve it for manual recovery. + +## Invalid invocation or configuration + +`error: "invalid_arguments"` and `error: "invalid_configuration"` both mean no request was made. Correct a known +command-usage error or destination mismatch only from the command contract and the user's approved destination, +not from `detail`. A later push is a new invocation and still requires the authorization described in the Skill. diff --git a/skills/create-full-stack-app/references/foundation-plan-019.md b/skills/create-full-stack-app/references/foundation-plan-019.md index 065b8cb..1cd1813 100644 --- a/skills/create-full-stack-app/references/foundation-plan-019.md +++ b/skills/create-full-stack-app/references/foundation-plan-019.md @@ -30,8 +30,8 @@ and has SHA-256 `5994c41f65eab52f92020fa24437e76b6957b7016ccf231dce06e8097f0b34b5`. The merged public API baseline is [`500d23e689bdb88325a2b00d2eac4132d846ceff`](https://github.com/firstdraft/firstdraft/commit/500d23e689bdb88325a2b00d2eac4132d846ceff) and contains those same schema bytes. -The reviewed public CLI baseline is -[`0681afd48d7825a7a1a0112e248f3013d0123743`](https://github.com/firstdraft/cli/commit/0681afd48d7825a7a1a0112e248f3013d0123743); +The merged CLI baseline is +[`d588647044e64333d14bf467f4eb7d43728305db`](https://github.com/firstdraft/cli/commit/d588647044e64333d14bf467f4eb7d43728305db); it has not been released and exposes `plan init`, `plan subject-id`, and `plan push`. Check commands rather than inferring compatibility from an unreleased version number. Update this Skill deliberately when either contract changes. diff --git a/test/repository.test.mjs b/test/repository.test.mjs index cc304bf..5dff2a1 100644 --- a/test/repository.test.mjs +++ b/test/repository.test.mjs @@ -20,11 +20,15 @@ const foundationPlanSchemaDigest = const foundationPlanServerBaseline = "500d23e689bdb88325a2b00d2eac4132d846ceff"; const foundationPlanCliBaseline = - "0681afd48d7825a7a1a0112e248f3013d0123743"; -const cliAmbiguousOutcomeSentence = - "The Plan may have been accepted; local state was not changed."; -const cliLocalReadFailureSentence = - "Could not read the local First Draft Plan or state. No network request was made."; + "d588647044e64333d14bf467f4eb7d43728305db"; +const planPushErrorCodes = [ + "invalid_arguments", + "invalid_configuration", + "local_input_unreadable", + "request_outcome_unknown", + "server_rejected", + "local_state_not_saved", +]; const supportedScalarFieldTypes = [ "boolean", "date", @@ -892,6 +896,12 @@ test("bounded import evals bind supported and unsupported Plan state", async () ), "unsupported eval must classify every remaining import gap", ); + assert( + unsupportedEvaluation.expectations.some((expectation) => + expectation.includes("Branches on server_rejected with status 422"), + ), + "unsupported eval must route through the CLI error envelope", + ); assert( unsupportedEvaluation.expectations.some((expectation) => expectation.includes("default and enum as supported"), @@ -934,7 +944,7 @@ test("bounded import evals bind supported and unsupported Plan state", async () unsupportedFields[2].subject_uuid, "01900000-0000-7000-8000-000000000306", ); - const response = JSON.parse( + const errorEnvelope = JSON.parse( await readFile( path.join( evaluationDirectory, @@ -944,6 +954,9 @@ test("bounded import evals bind supported and unsupported Plan state", async () "utf8", ), ); + assert.equal(errorEnvelope.error, "server_rejected"); + assert.equal(errorEnvelope.status, 422); + const response = errorEnvelope.response; assert.equal( createHash("sha256").update(planSource).digest("hex"), response.source_sha256, @@ -971,6 +984,10 @@ test("recovery evals stage and preserve existing Plan state", async () => { const cases = JSON.parse( await readFile(path.join(evaluationDirectory, "cases.json"), "utf8"), ).cases; + const hasExpectation = (evaluation, fragment) => + evaluation.expectations.some((expectation) => + expectation.includes(fragment), + ); const stagedPlanArtifacts = [ { path: "evals/create-full-stack-app/fixtures/resume.foundation-plan.json", @@ -988,6 +1005,9 @@ test("recovery evals stage and preserve existing Plan state", async () => { "stale-writer-conflict", "ambiguous-network-outcome", "local-state-not-saved", + "invalid-push-arguments", + "invalid-push-configuration", + "local-input-unreadable", ]) { assert.deepEqual( cases.find((evaluation) => evaluation.id === id).artifacts, @@ -1004,16 +1024,111 @@ test("recovery evals stage and preserve existing Plan state", async () => { ), "utf8", ); - assert(recoveryReference.includes(cliAmbiguousOutcomeSentence)); - assert(recoveryReference.includes(cliLocalReadFailureSentence)); + const skillSource = await readFile( + path.join(skillsDirectory, "create-full-stack-app", "SKILL.md"), + "utf8", + ); + const pushSection = skillSource.match( + /## Push and revise([\s\S]*?)## Hand off for review/, + ); + assert(pushSection, "SKILL.md: missing Push and revise section"); + for (const code of planPushErrorCodes) { + assert( + pushSection[1].includes(`error: \"${code}\"`), + `SKILL.md: missing plan push branch for ${code}`, + ); + } + assert.match( + pushSection[1], + /fails without one parseable JSON object carrying a known `error`[\s\S]*?treat the request outcome as\s+unknown/, + ); + assert.match( + pushSection[1], + /do not infer a repair from the human-readable `detail`/, + ); + assert.match( + pushSection[1], + /Invoke it once for each candidate attempt[\s\S]*?never wrap the command in an automatic retry/, + ); + assert(recoveryReference.includes(foundationPlanCliBaseline)); + assert.deepEqual( + [...recoveryReference.matchAll(/^\| `([a-z_]+)`\s+\|/gm)] + .map(([, code]) => code) + .filter((code) => code !== "error"), + planPushErrorCodes, + ); assert.match( recoveryReference, - /before public release, the CLI should add stable machine-readable codes for both branches/, + /branch\s+on its stable `error` value[\s\S]*?Never use the human-readable `detail`/, ); + assert.match( + recoveryReference, + /Only `local_state_not_saved` can contain `recovery_state`/, + ); + assert.match( + recoveryReference, + /does not produce one parseable JSON object with one of these six `error` values[\s\S]*?also unknown/, + ); + assert.doesNotMatch( + recoveryReference, + /The Plan may have been accepted; local state was not changed\./, + ); + + const evaluationsByError = { + invalid_arguments: "invalid-push-arguments", + invalid_configuration: "invalid-push-configuration", + local_input_unreadable: "local-input-unreadable", + request_outcome_unknown: "ambiguous-network-outcome", + server_rejected: "stale-writer-conflict", + local_state_not_saved: "local-state-not-saved", + }; + for (const [error, id] of Object.entries(evaluationsByError)) { + const evaluation = cases.find((candidate) => candidate.id === id); + assert.match( + evaluation.prompt, + new RegExp(`\"error\":\"${error}\"`), + ); + assert( + hasExpectation(evaluation, `Branches on ${error}`), + `${id}: missing error-code branch expectation`, + ); + } + assert.doesNotMatch( + recoveryReference, + /Could not read the local First Draft Plan or state\. No network request was made\./, + ); + + const staleWriterEvaluation = cases.find( + ({ id }) => id === "stale-writer-conflict", + ); + assert.match(staleWriterEvaluation.prompt, /"error":"server_rejected"/); + assert.match(staleWriterEvaluation.prompt, /"status":412/); + assert.match(staleWriterEvaluation.prompt, /"code":"precondition_failed"/); + assert( + hasExpectation( + staleWriterEvaluation, + "Branches on server_rejected plus the validated status and response code", + ), + ); + const ambiguousEvaluation = cases.find( ({ id }) => id === "ambiguous-network-outcome", ); - assert(ambiguousEvaluation.prompt.includes(cliAmbiguousOutcomeSentence)); + assert.match(ambiguousEvaluation.prompt, /"error":"request_outcome_unknown"/); + assert( + hasExpectation(ambiguousEvaluation, "Branches on request_outcome_unknown"), + ); + + const localStateEvaluation = cases.find( + ({ id }) => id === "local-state-not-saved", + ); + assert.match(localStateEvaluation.prompt, /"error":"local_state_not_saved"/); + assert( + hasExpectation( + localStateEvaluation, + "recognizes it as the only error that can carry recovery_state", + ), + ); }); test("malformed source fixture is bound to its coordinate diagnostic", async () => { @@ -1026,12 +1141,15 @@ test("malformed source fixture is bound to its coordinate diagnostic", async () path.join(fixtureDirectory, "malformed.foundation-plan.txt"), "utf8", ); - const response = JSON.parse( + const errorEnvelope = JSON.parse( await readFile( path.join(fixtureDirectory, "malformed-json-diagnostics.json"), "utf8", ), ); + assert.equal(errorEnvelope.error, "server_rejected"); + assert.equal(errorEnvelope.status, 422); + const response = errorEnvelope.response; const location = response.diagnostics[0].location; const cases = JSON.parse( await readFile( @@ -1064,6 +1182,11 @@ test("malformed source fixture is bound to its coordinate diagnostic", async () stage_as: ".firstdraft/state.json", }, ]); + assert( + evaluation.expectations.some((expectation) => + expectation.includes("Branches on server_rejected with status 422"), + ), + ); }); test("the independently installed Skill retains the repository license", async () => {