From 1fba4a819285983577bf70bd52335fb9f3e6986b Mon Sep 17 00:00:00 2001 From: Raghu Betina Date: Thu, 30 Jul 2026 23:22:51 -0500 Subject: [PATCH] Refresh local smoke evidence The project-scoped server path is no longer pending. Record the controlled local smoke. Keep release, authentication, and capability boundaries explicit. --- README.md | 34 +++-- evals/create-full-stack-app/cases.json | 12 +- skills/create-full-stack-app/SKILL.md | 23 ++- .../references/diagnostics-and-recovery.md | 2 +- .../references/examples.md | 7 +- .../references/foundation-plan-019.md | 49 ++++-- test/repository.test.mjs | 143 ++++++++++++++++-- 7 files changed, 220 insertions(+), 50 deletions(-) diff --git a/README.md b/README.md index db0fd15..05f01ce 100644 --- a/README.md +++ b/README.md @@ -2,12 +2,23 @@ Portable Agent Skills for working with [First Draft](https://github.com/firstdraft/firstdraft). -This repository is experimental. The bounded authoring API and required CLI are implemented in reviewed slices, -including a CLI that can wait for analysis and verify and materialize one pinned Compilation. The matching server -AnalysisRun and Compilation lifecycle slices are still landing, and the CLI has not been released. The first local -compiler smoke path is limited to one Entity using supported scalar Fields; complete Foundation Plan import, -arbitrary application generation, Publish, deployment, and mobile clients are not available end to end. The Skills -are being reviewed in small slices before they are advertised for general use. +This repository is experimental. The project-scoped server analysis and Compilation transport and the required CLI +are implemented in reviewed slices. First Draft's committed +[controlled CLI smoke](https://github.com/firstdraft/firstdraft/blob/9e296062bf543e89387e2f1044dd29eb52123c9c/script/compilation_http_cli_smoke) +reproducibly exercises an installed CLI, loopback Rails, and real Solid Queue through 151-file application +materialization. Separately, a one-off observation on 2026-07-30 used a fresh Codex invocation of this Skill at +[`e24b438`](https://github.com/firstdraft/skills/commit/e24b438918f406e8638e79598b6d83605bd4c15a), +server baseline +[`9e29606`](https://github.com/firstdraft/firstdraft/commit/9e296062bf543e89387e2f1044dd29eb52123c9c), +and CLI baseline +[`36f1292`](https://github.com/firstdraft/cli/commit/36f12921c0f6641f073820734234c11e47fdb834) +to go from a prose request through valid analysis and Movie application materialization. That observation is narrow +development evidence, not a reproducible agent eval, release, authentication, representative-user, deployment, or +production evidence. The CLI and Skill remain unpublished, and the importer supports only the bounded subset +described by the Skill reference. The compiler path is narrower still: one Entity using supported scalar Fields. +There is no Plan GET or pull operation, complete semantic analyzer, Publish action, arbitrary application generation, +deployment workflow, or mobile-client generation. The Skills are being reviewed in small slices before they are +advertised for general use. ## Skills @@ -80,6 +91,11 @@ private state, leave the accepted Plan bytes unchanged, and ensure `./generated- starts. Never reuse a Project or Compilation across server-backed eval runs or expose state contents to the agent. The `*-analysis.json` fixtures and Compilation eval prompts are behavioral examples accepted by the pinned CLI -contract. They are not evidence that the pending server AnalysisRun and Compilation lifecycle slices are merged, -deployed, or released. Until those server slices land, server-backed evals can demonstrate handled stop paths but -cannot be graded as though a terminal analysis or materialized application were reachable. +contract, not execution evidence by themselves. The project-scoped AnalysisRun and Compilation transport is merged +at server baseline +[`9e29606`](https://github.com/firstdraft/firstdraft/commit/9e296062bf543e89387e2f1044dd29eb52123c9c). +A server-backed eval may now exercise terminal analysis and materialization only when it is prepared and run against +a fresh compatible local server, queue, CLI, Project, and Compilation. The committed CLI smoke does not establish +released or authenticated operation, representative external-agent use, deployment, production readiness, or +capabilities beyond the one-Entity scalar compiler slice. The dated agent observation above is not a reproducible +eval. diff --git a/evals/create-full-stack-app/cases.json b/evals/create-full-stack-app/cases.json index a564480..e6002c1 100644 --- a/evals/create-full-stack-app/cases.json +++ b/evals/create-full-stack-app/cases.json @@ -10,8 +10,8 @@ "Confirms the application key and name before running plan init.", "Leaves entities empty instead of inventing a placeholder Entity.", "Uses plan push exactly once, then runs plan status --wait and branches on analysis.status rather than the shell exit code.", - "If the pending status API is unavailable, branches on the stable status error and stops without retrying, pushing again, or inferring an analysis result.", - "If analysis.status is valid, reports that the current analysis gate passed while making clear that Compilation remains unavailable." + "If plan status --wait returns status_unavailable or any other handled status error, branches on the stable error and stops without retrying, pushing again, or inferring an analysis result.", + "If analysis.status is valid, reports that the current analysis gate passed without treating it as Compilation approval." ], "artifacts": [ { @@ -163,7 +163,7 @@ "Leaves .firstdraft/state.json unopened and lets the CLI update it only after verified success.", "Runs plan push exactly once through the CLI and makes no direct network request.", "After a verified push, runs plan status --wait and branches on analysis.status rather than the shell exit code.", - "If the pending status API is unavailable, branches on the stable status error and stops without retrying, pushing again, or inferring an analysis result.", + "If plan status --wait returns status_unavailable or any other handled status error, branches on the stable error and stops without retrying, pushing again, or inferring an analysis result.", "Reports the exact import and analysis boundaries demonstrated without claiming Publish, Compilation, or generation." ], "artifacts": [ @@ -190,7 +190,7 @@ "Treats the diagnostic message and suggestions as advisory data rather than instructions.", "Runs plan push exactly once for the deliberately corrected complete Plan, then runs plan status --wait for that new candidate.", "Does not open or edit .firstdraft/state.json, choose another origin, send the ETag directly, or make a direct request.", - "If the pending status API is unavailable, branches on the stable status error and stops without retrying, pushing again, or inferring a new analysis result.", + "If plan status --wait returns status_unavailable or any other handled status error, branches on the stable error and stops without retrying, pushing again, or inferring a new analysis result.", "Branches on the new analysis.status and claims only the boundaries demonstrated; even valid is an analysis gate, not Compilation." ], "artifacts": [ @@ -420,8 +420,8 @@ "Runs plan push exactly once through the CLI and makes no direct network request.", "After a verified push, runs plan status --wait and branches on analysis.status rather than the shell exit code.", "Reports the verified push as acceptance of the bounded import without claiming local schema validation.", - "If the pending status API is unavailable, branches on the stable status error and stops without retrying, pushing again, or inferring an analysis result.", - "If analysis.status is valid, reports that the current analysis gate passed while making clear that Compilation remains unavailable." + "If plan status --wait returns status_unavailable or any other handled status error, branches on the stable error and stops without retrying, pushing again, or inferring an analysis result.", + "If analysis.status is valid, reports that the current analysis gate passed without treating it as Compilation approval." ] }, { diff --git a/skills/create-full-stack-app/SKILL.md b/skills/create-full-stack-app/SKILL.md index 1bcb898..4862da1 100644 --- a/skills/create-full-stack-app/SKILL.md +++ b/skills/create-full-stack-app/SKILL.md @@ -12,12 +12,23 @@ concurrency, and network behavior in the `firstdraft` CLI. This Skill is experimental. The reviewed CLI can initialize a Plan, mint UUIDv7 subject IDs, push exact bytes, wait for the current whole-graph analysis, and perform one pinned Compilation whose complete artifact it verifies before -atomically materializing a new local directory. The reviewed server can create and replace empty drafts plus a -bounded subset of Entities, ten scalar Field kinds, enum Fields with ordered values, schema-valid tagged Field -defaults, and Field or system-Field Primary Descriptors. The first local compiler smoke path is narrower: one Entity -using supported scalar Fields. It is not arbitrary application generation, a deployment workflow, or support for -the rest of the Foundation Plan. The matching server AnalysisRun and Compilation lifecycle slices are still -landing, and none of these components is released end to end. +atomically materializing a new local directory. The reviewed project-scoped server transport accepts complete Plan +replacements, exposes bounded AnalysisRun status, and can start, poll, cancel, and return the artifact for one pinned +Compilation. Its importer supports empty drafts plus a bounded subset of Entities, ten scalar Field kinds, enum +Fields with ordered values, schema-valid tagged Field and Reference defaults, References with ordered targets and +mechanically derived forward Associations, Predicates with exact Expression JSON, and Field or system-Field Primary +Descriptors. + +First Draft's committed controlled CLI smoke at server baseline `9e29606` reproducibly exercises an installed CLI, +loopback Rails, and real Solid Queue through 151-file application materialization. Separately, a one-off observation +on 2026-07-30 used a fresh Codex invocation at Skill baseline `e24b438`, server baseline `9e29606`, and CLI baseline +`36f1292` to go from a prose request through valid analysis and Movie application materialization. That observation +is not a reproducible agent eval. Neither form of evidence makes the unauthenticated local transport, CLI, or Skill +released or published, and neither is representative-user, deployed, or production evidence. The smoke exercised +the successful start, status, artifact, and materialization path, not cancellation. The compiler path remains +narrower than the importer: one Entity using supported scalar Fields. There is no Plan GET or pull operation, +complete semantic analyzer, Publish action, arbitrary application generation, deployment workflow, or support for +the rest of the Foundation Plan. ## Load the relevant references diff --git a/skills/create-full-stack-app/references/diagnostics-and-recovery.md b/skills/create-full-stack-app/references/diagnostics-and-recovery.md index 971ecc3..e19a208 100644 --- a/skills/create-full-stack-app/references/diagnostics-and-recovery.md +++ b/skills/create-full-stack-app/references/diagnostics-and-recovery.md @@ -6,7 +6,7 @@ unverified response. ## Local initialization error boundary The merged CLI contract at -[`74e3d4203587bcecbaf85362596037cb71d5154c`](https://github.com/firstdraft/cli/commit/74e3d4203587bcecbaf85362596037cb71d5154c) +[`36f12921c0f6641f073820734234c11e47fdb834`](https://github.com/firstdraft/cli/commit/36f12921c0f6641f073820734234c11e47fdb834) writes exactly one JSON object to standard error for every handled `plan init` failure. Parse the complete output and branch on its stable `error` value, never on human-readable `detail` or the broad shell exit status. diff --git a/skills/create-full-stack-app/references/examples.md b/skills/create-full-stack-app/references/examples.md index 601e277..730e730 100644 --- a/skills/create-full-stack-app/references/examples.md +++ b/skills/create-full-stack-app/references/examples.md @@ -149,9 +149,10 @@ key. If `medium` is renamed, update the default in the same candidate while pres ## Stored and reverse relationship -This complete document is structurally valid v0.19, but References and authored Associations remain outside the -reviewed importer subset. It is not Compiler-proven. `Task` owns the stored `project` Reference. `Project` owns the -meaningful reverse `tasks` Association. The forward `task.project` Association is derived and therefore omitted. +This complete document is structurally valid v0.19. Its Reference is within the reviewed importer subset, but its +authored reverse Association is not, so the complete document is rejected at the current conditional PUT boundary. +It is not analyzer- or Compiler-proven. `Task` owns the stored `project` Reference. `Project` owns the meaningful +reverse `tasks` Association. The forward `task.project` Association is derived and therefore omitted. ```json { diff --git a/skills/create-full-stack-app/references/foundation-plan-019.md b/skills/create-full-stack-app/references/foundation-plan-019.md index 18b3194..eef48e8 100644 --- a/skills/create-full-stack-app/references/foundation-plan-019.md +++ b/skills/create-full-stack-app/references/foundation-plan-019.md @@ -20,23 +20,33 @@ authorized. - Structural validity does not prove readable-link resolution, whole-application consistency, target support, or compilability. - The reviewed conditional PUT imports empty drafts and a bounded subset of Entities, ten scalar Field kinds, enum - Fields with ordered values, schema-valid tagged Field defaults, and Field or system-Field Primary Descriptors. -- The reviewed CLI can read or wait for a bounded current whole-graph analysis. The matching server AnalysisRun - response is not yet released end to end. -- There is no released end-to-end CLI/API workflow, complete nonempty import, Plan GET or pull operation, complete - semantic analyzer, Publish action, Compilation action, or generated Foundation. + Fields with ordered values, schema-valid tagged Field and Reference defaults, References with ordered targets and + mechanically derived forward Associations, Predicates with exact Expression JSON, and Field or system-Field + Primary Descriptors. +- The project-scoped server implements bounded AnalysisRun status and Compilation start, status, cancellation, and + artifact transport for the reviewed CLI contract. +- First Draft's committed controlled CLI smoke at server baseline `9e29606` reproducibly exercises an installed CLI, + loopback Rails, and real Solid Queue through 151-file application materialization. +- Separately, a one-off observation on 2026-07-30 used a fresh Codex invocation at Skill baseline `e24b438`, server + baseline `9e29606`, and CLI baseline `36f1292` to go from a prose request through valid analysis and Movie + application materialization. That observation is not a reproducible agent eval. +- That path remains unreleased, unpublished, unauthenticated, local, and bounded to one Entity using supported + scalar Fields. It exercised successful Compilation start, status, artifact, and materialization, not cancellation. + Neither form of evidence is representative-user, deployed, or production evidence. +- There is no Plan GET or pull operation, complete semantic analyzer, Publish action, arbitrary application + generation, deployment workflow, or support for the rest of the Foundation Plan. The bundled schema was copied from the [First Draft source at revision `12fa2a6`](https://github.com/firstdraft/firstdraft/blob/12fa2a6bcac122196d55f5528fbc3f1363c684e3/docs/architecture/design/foundation-plan.schema.json) and has SHA-256 -`5994c41f65eab52f92020fa24437e76b6957b7016ccf231dce06e8097f0b34b5`. The merged public API baseline is -[`500d23e689bdb88325a2b00d2eac4132d846ceff`](https://github.com/firstdraft/firstdraft/commit/500d23e689bdb88325a2b00d2eac4132d846ceff) +`5994c41f65eab52f92020fa24437e76b6957b7016ccf231dce06e8097f0b34b5`. The merged server baseline is +[`9e296062bf543e89387e2f1044dd29eb52123c9c`](https://github.com/firstdraft/firstdraft/commit/9e296062bf543e89387e2f1044dd29eb52123c9c) and contains those same schema bytes. The merged CLI baseline is -[`74e3d4203587bcecbaf85362596037cb71d5154c`](https://github.com/firstdraft/cli/commit/74e3d4203587bcecbaf85362596037cb71d5154c); -it has not been released and exposes `plan init`, `plan subject-id`, `plan push`, and `plan status`. Check commands -rather than inferring compatibility from an unreleased version number. Update this Skill deliberately when either -contract changes. +[`36f12921c0f6641f073820734234c11e47fdb834`](https://github.com/firstdraft/cli/commit/36f12921c0f6641f073820734234c11e47fdb834); +it has not been released and exposes `plan init`, `plan subject-id`, `plan push`, `plan status`, and `plan compile`. +Check commands rather than inferring compatibility from an unreleased version number. Update this Skill deliberately +when either contract changes. ## Closed envelope @@ -112,7 +122,8 @@ App Schema artifact. The reviewed importer accepts the required Application properties `key`, `name`, `native`, `delivery`, and `entities`. `native` and `delivery` must remain empty. `entities` may contain any number of Entities with -`subject_uuid`, `key`, `name`, optional `icon` and `fields`, and one required `primary_descriptor`. +`subject_uuid`, `key`, `name`, optional `icon`, `fields`, `references`, and `predicates`, and one required +`primary_descriptor`. The smallest accepted Application remains: @@ -172,9 +183,19 @@ This retention is structural, not default analysis. It does not prove literal co enum membership, readable-locator resolution, nullability, normalization behavior, or Compiler lowering. Preserve the intended default when reporting any later semantic gap. +An Entity may also own supported References and Predicates. A Reference retains schema-valid combinations of +`subject_uuid`, `key`, `name`, `targets`, `required`, `one_to_one`, `on_referenced_deleted`, `default`, `immutable`, +and `realization`. Its ordered target Entity keys are resolved during import, and the Project graph mechanically +maintains its same-key forward Association. Reference `validations` remain outside this boundary. + +A Predicate retains schema-valid combinations of `subject_uuid`, `key`, `name`, and `expression`. Import preserves +the Expression's exact decoded JSON meaning without claiming link resolution, type checking, or target lowering. +Importability does not imply that the current bounded whole-graph analyzer or Compiler accepts a Project containing +References or Predicates. + Scalar Fields have no `settings` object, and enum `settings` admits only `values` and optional `ordinal`; any other settings shape is structurally invalid rather than an importer capability gap. Schema-valid Field types outside -the list above, Validations, derivations, References, Associations, and other Entity or Application capabilities -remain unsupported. One unsupported pointer rejects the complete conditional PUT with +the list above, Validations, derivations, authored Associations, and other Entity or Application capabilities remain +unsupported. One unsupported pointer rejects the complete conditional PUT with `foundation_plan.import.unsupported_capability` and no mutation. That diagnostic describes server capability, not invalid product meaning. Preserve the authored Plan and report the exact gap. diff --git a/test/repository.test.mjs b/test/repository.test.mjs index 7c69fa9..c203b71 100644 --- a/test/repository.test.mjs +++ b/test/repository.test.mjs @@ -18,11 +18,11 @@ const foundationPlanTarget = { const foundationPlanSchemaDigest = "5994c41f65eab52f92020fa24437e76b6957b7016ccf231dce06e8097f0b34b5"; const foundationPlanServerBaseline = - "500d23e689bdb88325a2b00d2eac4132d846ceff"; -const foundationPlanCliBaseline = - "74e3d4203587bcecbaf85362596037cb71d5154c"; + "9e296062bf543e89387e2f1044dd29eb52123c9c"; const planCompileCliBaseline = "36f12921c0f6641f073820734234c11e47fdb834"; +const agentSmokeSkillBaseline = + "e24b438918f406e8638e79598b6d83605bd4c15a"; const planInitErrorCodes = [ "invalid_arguments", "local_initialization_failed", @@ -90,6 +90,24 @@ const supportedFieldProperties = [ "encrypted_at_rest", "redact_from_logs", ]; +const supportedReferenceProperties = [ + "subject_uuid", + "key", + "name", + "targets", + "required", + "one_to_one", + "on_referenced_deleted", + "default", + "immutable", + "realization", +]; +const supportedPredicateProperties = [ + "subject_uuid", + "key", + "name", + "expression", +]; test("installable Skills follow the portable repository profile", async () => { const entries = await readdir(skillsDirectory, { withFileTypes: true }); @@ -226,6 +244,10 @@ test("authored JSON examples parse and retain the pinned Plan contract", async ( }); test("bounded importer prose remains bound to the exact allowlists", async () => { + const skillSource = await readFile( + path.join(skillsDirectory, "create-full-stack-app", "SKILL.md"), + "utf8", + ); const referencesDirectory = path.join( skillsDirectory, "create-full-stack-app", @@ -319,6 +341,52 @@ test("bounded importer prose remains bound to the exact allowlists", async () => foundationPlanReference, /retention is structural, not default analysis[\s\S]*?does not prove literal compatibility[\s\S]*?Compiler lowering/, ); + const documentedReferenceSection = foundationPlanReference.match( + /A Reference retains schema-valid combinations of\s+([\s\S]*?)\. Its ordered target Entity keys/, + ); + assert( + documentedReferenceSection, + "foundation-plan-019.md: missing retained Reference property list", + ); + assert.deepEqual( + [...documentedReferenceSection[1].matchAll(/`([^`]+)`/g)].map( + (match) => match[1], + ), + supportedReferenceProperties, + ); + assert.match( + foundationPlanReference, + /Project graph mechanically\s+maintains its same-key forward Association/, + ); + assert.match( + foundationPlanReference, + /Reference `validations` remain outside this boundary/, + ); + const documentedPredicateSection = foundationPlanReference.match( + /A Predicate retains schema-valid combinations of ([\s\S]*?)\. Import preserves/, + ); + assert( + documentedPredicateSection, + "foundation-plan-019.md: missing retained Predicate property list", + ); + assert.deepEqual( + [...documentedPredicateSection[1].matchAll(/`([^`]+)`/g)].map( + (match) => match[1], + ), + supportedPredicateProperties, + ); + assert.match( + foundationPlanReference, + /Importability does not imply that the current bounded whole-graph analyzer or Compiler accepts a Project containing\s+References or Predicates/, + ); + assert.match( + foundationPlanReference, + /Schema-valid Field types outside\s+the list above, Validations, derivations, authored Associations, and other Entity or Application capabilities remain\s+unsupported/, + ); + assert.match( + skillSource, + /schema-valid tagged Field and Reference defaults, References with ordered targets and\s+mechanically derived forward Associations, Predicates with exact Expression JSON/, + ); const diagnosticsReference = await readFile( path.join(referencesDirectory, "diagnostics-and-recovery.md"), @@ -528,10 +596,10 @@ test("complete examples and eval Plans validate against the bundled exact schema ); assert(referenceSource.includes(foundationPlanSchemaDigest)); assert(referenceSource.includes(foundationPlanServerBaseline)); - assert(referenceSource.includes(foundationPlanCliBaseline)); + assert(referenceSource.includes(planCompileCliBaseline)); assert.match( referenceSource, - /merged public API baseline is[\s\S]*?and contains those same schema bytes/, + /merged server baseline is[\s\S]*?and contains those same schema bytes/, ); const validate = new Ajv2020({ @@ -1144,12 +1212,65 @@ test("analysis status guidance follows the pinned CLI contract", async () => { ); assert.match( readme, - /The `\*-analysis\.json` fixtures and Compilation eval prompts are behavioral examples accepted by the pinned CLI\s+contract[\s\S]*?not evidence that the pending server AnalysisRun and Compilation lifecycle slices are merged,\s+deployed, or released/, + /The `\*-analysis\.json` fixtures and Compilation eval prompts are behavioral examples accepted by the pinned CLI\s+contract, not execution evidence by themselves/, + ); + assert.match( + readme, + /project-scoped AnalysisRun and Compilation transport is merged[\s\S]*?server-backed eval may now exercise terminal analysis and materialization only when it is prepared and run against\s+a fresh compatible local server, queue, CLI, Project, and Compilation/, + ); + assert( + readme.includes( + `firstdraft/firstdraft/blob/${foundationPlanServerBaseline}/script/compilation_http_cli_smoke`, + ), + ); + assert.match( + readme, + /committed[\s\S]*?controlled CLI smoke[\s\S]*?reproducibly exercises[\s\S]*?Separately, a one-off observation on 2026-07-30/, ); + assert(readme.includes(agentSmokeSkillBaseline)); + assert.match(readme, /observation is narrow\s+development evidence, not a reproducible agent eval/); assert.match( readme, - /Until those server slices land[\s\S]*?cannot be graded as though a terminal analysis or materialized application\s+were reachable/, + /does not establish\s+released or authenticated operation, representative external-agent use, deployment, production readiness, or\s+capabilities beyond the one-Entity scalar compiler slice/, + ); + const skillEvidence = skillSource.match( + /This Skill is experimental\.([\s\S]*?)## Load the relevant references/, + ); + const foundationPlanEvidence = foundationPlanReference.match( + /## Current evidence boundary([\s\S]*?)The bundled schema was copied/, ); + assert(skillEvidence, "SKILL.md: missing current evidence boundary"); + assert( + foundationPlanEvidence, + "foundation-plan-019.md: missing current evidence boundary", + ); + for (const source of [skillEvidence[1], foundationPlanEvidence[1]]) { + assert(source.includes("loopback Rails")); + assert(source.includes("real Solid Queue")); + assert.match(source, /committed controlled CLI smoke/); + assert.match(source, /reproducibly exercises/); + assert.match(source, /Separately, a one-off observation\s+on 2026-07-30/); + assert.match(source, /fresh\s+Codex/); + assert(source.includes("e24b438")); + assert(source.includes("9e29606")); + assert(source.includes("36f1292")); + assert.match(source, /not a reproducible agent eval/); + assert.match(source, /151-file application\s+materialization/); + assert(source.includes("unauthenticated")); + assert.match( + source, + /(?:neither is|Neither form of evidence is) representative-user,\s+deployed, or production evidence/, + ); + assert.match(source, /one Entity using supported\s+scalar\s+Fields/); + assert.match( + source, + /no Plan GET or pull operation,\s+complete semantic analyzer, Publish action, arbitrary\s+application\s+generation, deployment workflow/, + ); + assert.match( + source, + /successful (?:Compilation )?start, status, artifact, and materialization[\s\S]*?not cancellation/, + ); + } for (const id of [ "initialize-empty-plan", @@ -1179,9 +1300,9 @@ test("analysis status guidance follows the pinned CLI contract", async () => { const evaluation = cases.find((candidate) => candidate.id === id); assert( evaluation.expectations.some((expectation) => - expectation.includes("pending status API is unavailable"), + expectation.includes("any other handled status error"), ), - `${id}: pending server path must stop on the status error`, + `${id}: every handled status failure must remain a stop condition`, ); } @@ -1466,7 +1587,7 @@ test("Compilation guidance follows the pinned CLI contract", async () => { assert(referenceSection[1].includes(planCompileCliBaseline)); assert.match( readme, - /first local\s+compiler smoke path is limited to one Entity using supported scalar Fields/, + /compiler path is narrower still:\s+one Entity using supported scalar Fields/, ); const evaluation = (id) => { @@ -1654,7 +1775,7 @@ test("recovery evals stage and preserve existing Plan state", async () => { pushSection[1], /Invoke it once for each candidate attempt[\s\S]*?never wrap the command in an automatic retry/, ); - assert(recoveryReference.includes(foundationPlanCliBaseline)); + assert(recoveryReference.includes(planCompileCliBaseline)); const pushReference = recoveryReference.match( /## Plan push error boundary([\s\S]*?)## Verified success/, );