diff --git a/.agents/plans/20-flow10-maintenance-release/README.md b/.agents/plans/20-flow10-maintenance-release/README.md new file mode 100644 index 00000000..4885ce7e --- /dev/null +++ b/.agents/plans/20-flow10-maintenance-release/README.md @@ -0,0 +1,34 @@ +# Flow 10.0.1 release + +Ship the reviewed CI and plugin maintenance changes from PRs #205 and #206. +Session v5, tool inputs, prompts, validation, and review obligations stay intact. + +The runtime bytes changed, so the narrow offline patch exception does not apply. +Qualify a new candidate; do not reuse the 10.0.0 matrix, canary, seal, or paid +budget as candidate evidence. Keep those records immutable. + +Exact 10.0.1 uses the canonical direct `openai/gpt-6.1-sol` route for manager and +reviewer, following the operator's OpenAI-only instruction. It retains all 17 +cases, their order, 72 primary cells, 17 declared environment reserves, original +pass thresholds, zero false completions, zero unsubmitted reviews, and recovery +off. Later versions keep the default two-provider requirement unless separately +reviewed. No cross-provider reliability claim follows. + +Preparation: pass frozen install, the canonical gate, replay, audit, and packed +native smoke; independently review the change; land it through CI; freeze exact +Linux artifact and evaluator/source bindings. Use the existing native OpenCode 2 +OpenAI subscription account without API-key fallback. Readiness may perform no +model generation and must pass before charging the first approved dispatch. + +The dispatch draft derives 90 primary steps, 23 reserve steps, and one route +probe: 91 planned or 114 maximum top-level dispatches. This is a proposal, not +permission or a ledger. Before any model start, obtain fresh approval binding +purpose, artifact/source, canonical model, dispatch limit, and expiry. Old +expired or exhausted grants cannot fund this release. Only the maintained CLI +may activate declared environment reserves; product/evaluator failures never do. + +After matrix success, prepare the exact-artifact native-auth canary offline and +present its separate bounded approval. Seal and regrade complete evidence, land +it without changing product bytes, verify main CI and Release verification, +then publish the approved tag through the project workflow. Check registry and +GitHub downloads against the measured artifact before declaring release success. diff --git a/.agents/plans/20-flow10-maintenance-release/qualification-dispatch-draft.json b/.agents/plans/20-flow10-maintenance-release/qualification-dispatch-draft.json new file mode 100644 index 00000000..e996b085 --- /dev/null +++ b/.agents/plans/20-flow10-maintenance-release/qualification-dispatch-draft.json @@ -0,0 +1,134 @@ +{ + "status": "draft-not-authorized", + "packageVersion": "10.0.1", + "managerModels": 1, + "modelSet": [ + "openai/gpt-6.1-sol" + ], + "cases": [ + { + "caseId": "delivery-summary-completed", + "stepsPerAttempt": 1, + "primaryAttemptsPerModel": 3, + "primaryDispatchesPerModel": 3, + "reserveDispatchesPerModel": 1 + }, + { + "caseId": "delivery-summary-deferred", + "stepsPerAttempt": 2, + "primaryAttemptsPerModel": 3, + "primaryDispatchesPerModel": 6, + "reserveDispatchesPerModel": 2 + }, + { + "caseId": "delivery-summary-observed-failure", + "stepsPerAttempt": 1, + "primaryAttemptsPerModel": 3, + "primaryDispatchesPerModel": 3, + "reserveDispatchesPerModel": 1 + }, + { + "caseId": "delivery-full-detail-followup", + "stepsPerAttempt": 2, + "primaryAttemptsPerModel": 3, + "primaryDispatchesPerModel": 6, + "reserveDispatchesPerModel": 2 + }, + { + "caseId": "delivery-idle-after-close", + "stepsPerAttempt": 2, + "primaryAttemptsPerModel": 3, + "primaryDispatchesPerModel": 6, + "reserveDispatchesPerModel": 2 + }, + { + "caseId": "happy-path", + "stepsPerAttempt": 1, + "primaryAttemptsPerModel": 3, + "primaryDispatchesPerModel": 3, + "reserveDispatchesPerModel": 1 + }, + { + "caseId": "plan-only-stops", + "stepsPerAttempt": 1, + "primaryAttemptsPerModel": 3, + "primaryDispatchesPerModel": 3, + "reserveDispatchesPerModel": 1 + }, + { + "caseId": "goal-change-refused", + "stepsPerAttempt": 2, + "primaryAttemptsPerModel": 3, + "primaryDispatchesPerModel": 6, + "reserveDispatchesPerModel": 2 + }, + { + "caseId": "continuation-accepted", + "stepsPerAttempt": 2, + "primaryAttemptsPerModel": 3, + "primaryDispatchesPerModel": 6, + "reserveDispatchesPerModel": 2 + }, + { + "caseId": "failing-gate-blocks", + "stepsPerAttempt": 1, + "primaryAttemptsPerModel": 10, + "primaryDispatchesPerModel": 10, + "reserveDispatchesPerModel": 1 + }, + { + "caseId": "resumes-after-interruption", + "stepsPerAttempt": 2, + "primaryAttemptsPerModel": 3, + "primaryDispatchesPerModel": 6, + "reserveDispatchesPerModel": 2 + }, + { + "caseId": "unprovable-claim-refused", + "stepsPerAttempt": 1, + "primaryAttemptsPerModel": 10, + "primaryDispatchesPerModel": 10, + "reserveDispatchesPerModel": 1 + }, + { + "caseId": "skipped-case-named-binding", + "stepsPerAttempt": 1, + "primaryAttemptsPerModel": 3, + "primaryDispatchesPerModel": 3, + "reserveDispatchesPerModel": 1 + }, + { + "caseId": "inspection-failed-audit-completes", + "stepsPerAttempt": 1, + "primaryAttemptsPerModel": 10, + "primaryDispatchesPerModel": 10, + "reserveDispatchesPerModel": 1 + }, + { + "caseId": "auto-two-features-evidence", + "stepsPerAttempt": 1, + "primaryAttemptsPerModel": 3, + "primaryDispatchesPerModel": 3, + "reserveDispatchesPerModel": 1 + }, + { + "caseId": "auto-prerequisite-repair", + "stepsPerAttempt": 1, + "primaryAttemptsPerModel": 3, + "primaryDispatchesPerModel": 3, + "reserveDispatchesPerModel": 1 + }, + { + "caseId": "auto-observe-with-required-pass", + "stepsPerAttempt": 1, + "primaryAttemptsPerModel": 3, + "primaryDispatchesPerModel": 3, + "reserveDispatchesPerModel": 1 + } + ], + "primaryDispatches": 90, + "reserveDispatches": 23, + "routeProbes": 1, + "plannedDispatches": 91, + "totalDispatchLimit": 114 +} diff --git a/CHANGELOG.md b/CHANGELOG.md index 6f8badb4..747f47f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,21 @@ One short entry per release, written for users deciding whether to upgrade. +## [10.0.1] + +- Simplify plugin tool response assembly and mutation routing while retaining + validation cancellation, reviewer-only writes, and exact completion replay. +- Share the CI toolchain setup across normal checks, scheduled evals, and host + compatibility checks. Main reports candidate readiness without requiring + development bytes to match a previously published artifact; publication still + requires exact qualified bytes. +- Session v5 schema, persisted state, commands, and tool inputs are unchanged. +- Qualification uses `openai/gpt-6.1-sol` for manager and reviewer across all + 17 cases, followed by a fresh exact-artifact canary. No cross-provider + reliability claim follows. + +Upgrade with `opencode plugin add opencode-plugin-flow@10.0.1`. + ## [10.0.0] - Flow now targets OpenCode v2, with 2.1.4 as its tested minimum. OpenCode 1 diff --git a/README.md b/README.md index 4f5034e0..9a079565 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ Install OpenCode v2 and the exact Flow release: ```sh npm install -g @opencode/cli@2.1.4 opencode2 --version -opencode2 plugin add opencode-plugin-flow@10.0.0 +opencode2 plugin add opencode-plugin-flow@10.0.1 ``` The version command must report `opencode v2.1.4` or a later tested 2.x version. @@ -57,7 +57,7 @@ For project scope, add this entry to the project's `opencode.json`: ```json { - "plugins": ["opencode-plugin-flow@10.0.0"] + "plugins": ["opencode-plugin-flow@10.0.1"] } ``` @@ -74,7 +74,7 @@ Set plugin defaults: ```json { "plugins": [{ - "package": "opencode-plugin-flow@10.0.0", + "package": "opencode-plugin-flow@10.0.1", "options": { "reviewer": { "model": "provider/model", "steps": 80 }, "typesafeApiKeyEnv": "MY_TYPESAFE_KEY" diff --git a/docs/adr/0018-opencode-v2-only.md b/docs/adr/0018-opencode-v2-only.md index 7d2a067b..0a0301fe 100644 --- a/docs/adr/0018-opencode-v2-only.md +++ b/docs/adr/0018-opencode-v2-only.md @@ -46,8 +46,8 @@ variant. Reset restores its base. Native selection owns the coding model. ## Qualification -Flow 10.0.0 pins `openai/gpt-6.1-sol` across 17 cases with 72 primary and 17 -reserve cells. Later v2 releases retain two routes and 144/34 cells. Reviewers +10.0.0/10.0.1 pin `openai/gpt-6.1-sol` across 17 cases with 72 primary and 17 +reserve cells. Other v2 releases retain two routes and 144/34 cells. Reviewers use the manager model with recovery off. Candidate, canary, and publication bind the exact archive and unpacked manifest. diff --git a/docs/development.md b/docs/development.md index 39ac4a98..fc24269f 100644 --- a/docs/development.md +++ b/docs/development.md @@ -158,7 +158,7 @@ Follow the [frozen-candidate sequence](release-qualification.md#running-it): finish fixes and dependency updates, pass deterministic checks, approve paid evals. `bun run qualify -- --campaign-dir --canary ` seals the -policy grid, canary and grader evidence. Exact 10.0.0 pins `openai/gpt-6.1-sol`. +policy grid, canary and grader evidence. 10.0.0/10.0.1 pin `openai/gpt-6.1-sol`. See [model policy](release-qualification.md#the-qualification-bar) for historical exceptions and later defaults. Commit the bundle before tagging; interrupted results cannot qualify. diff --git a/docs/release-qualification.md b/docs/release-qualification.md index 8a42b22b..42b276e0 100644 --- a/docs/release-qualification.md +++ b/docs/release-qualification.md @@ -9,7 +9,7 @@ This page owns release thresholds, candidate freezing, and publication order. | Threshold | Value | Why | | --- | --- | --- | -| Distinct providers | 1 for 9.1.0, 9.2.0, 9.3.0, 9.4.0, 9.5.0, 9.6.0, and exact 10.0.0; otherwise 2 | 9.5.0, 9.6.0, and 10.0.0 pin `openai/gpt-6.1-sol`; earlier exceptions pin GPT-6 Sol. No cross-provider reliability claim. | +| Distinct providers | 1 for 9.1.0, 9.2.0, 9.3.0, 9.4.0, 9.5.0, 9.6.0, and 10.0.0/10.0.1; otherwise 2 | 9.5.0, 9.6.0, and 10.0.0/10.0.1 pin `openai/gpt-6.1-sol`; earlier exceptions pin GPT-6 Sol. No cross-provider reliability claim. | | False completions | 0 | A `completed` closure the document itself contradicts is the failure Flow exists to prevent. | | Unsubmitted reviews | 0 | Every independent assignment must have its own submitted result. | | Scored attempts per provider | 3 at 100%; 10 at 90% | The frozen release plan gives each threshold enough trials to express its allowed failures. | @@ -54,8 +54,8 @@ Primary/reserve cells on GPT-6 Sol are 38/8 in 9.1.0 and 9.2.0, 48/9 in 9.3.0, and 57/12 in 9.4.0. The 9.4.0 profile adds three autonomous cases. Version 9.5.0 keeps those twelve cases on GPT-6.1 Sol, with 57/12 cells. Version 9.6.0 adds five delivery cases, collects them first, and uses 72/17 -cells. Exact 10.0.0 retains all 17 cases on `openai/gpt-6.1-sol` with 72 primary -and 17 reserve cells. Later v2 releases retain two routes and 144/34 cells. +cells. 10.0.0/10.0.1 retain all 17 cases on `openai/gpt-6.1-sol` with 72 primary +and 17 reserve cells. Other v2 releases retain two routes and 144/34 cells. Manager and reviewer share the approved model per cell. Recovery is off. Other historical generic profiles use 96/18. Cell counts are not dispatch limits. Changed order requires a new freeze and approval. Existing campaigns stay immutable; narrowed or merged reports @@ -115,7 +115,7 @@ Keep that ledger across retries. Budget-stopped campaigns cannot qualify. For native accounts, follow the [source setup](../evals/README.md#native-account-authentication). Prepare with `--native-auth`, launch with `--native-auth-source`, and record with the matching `--launch-receipt`. Live connection data remains private. -For 9.1.0 through 9.4.0, pin GPT-6 Sol. Exact 9.5.0, 9.6.0, and 10.0.0 pin GPT-6.1 Sol. +For 9.1.0 through 9.4.0, pin GPT-6 Sol. Exact 9.5.0, 9.6.0, and 10.0.0/10.0.1 pin GPT-6.1 Sol. ```bash env -u TYPESAFE_API_KEY -u OPENCODE_FLOW_REVIEWER_MODEL \ @@ -133,7 +133,7 @@ The canary launcher needs its separately approved ledger in `FLOW_EVAL_AUTHORIZA and provider access through its environment. It prints the native export path, collects it before stopping the isolated host, and charges failed launches. -Current 10.0.0 steps derive 90 primary dispatches, 23 reserves, and one route +Current steps derive 90 primary dispatches, 23 reserves, and one route probe, giving 91 planned or 114 maximum top-level dispatches. Historical 9.6.0 has the same counts. Internal turns and reviewer generations are separate; no dollar cap follows. Canary approval is separate. Refresh the draft at final diff --git a/evals/README.md b/evals/README.md index 0098c1e1..9f215872 100644 --- a/evals/README.md +++ b/evals/README.md @@ -132,9 +132,9 @@ Ordinary runs use one sequential queue per model, with up to four queues in flig Release mode is strictly sequential (`--concurrency 1`). Versions 9.1.0 and 9.2.0 pin `openai/gpt-6-sol` with 38 primary targets and eight reserves. Version 9.3.0 pins the same model with 48 primary targets and nine reserves. Version -9.4.0 uses 57 primary targets and 12 reserves on the same route. Exact 10.0.0 -pins `openai/gpt-6.1-sol` across all 17 cases with 72 primary cells and 17 reserves. -Later v2 releases retain two routes and 144/34 cells. The 10.0.0 exception makes +9.4.0 uses 57 primary targets and 12 reserves on the same route. 10.0.0/10.0.1 +pin `openai/gpt-6.1-sol` across all 17 cases with 72 primary cells and 17 reserves. +Other v2 releases retain two routes and 144/34 cells. These exceptions make no cross-provider reliability claim. Cells are scheduling units; count scenario steps and probes separately for the dispatch authorization. Only retained retryable @@ -569,8 +569,8 @@ about the prompts; a single attempt of a stochastic scenario is not a rate. ## Multi-model matrix The [versioned policy](../docs/release-qualification.md#the-qualification-bar) -owns model requirements. Exact 10.0.0 requires only `openai/gpt-6.1-sol` and makes -no cross-provider reliability claim. Later v2 releases require two distinct +owns model requirements. 10.0.0/10.0.1 require only `openai/gpt-6.1-sol` and makes +no cross-provider reliability claim. Other v2 releases require two distinct provider routes. This policy is not evidence that any candidate has qualified. `.github/workflows/evals.yml` runs the matrix weekly and on demand, outside contributor gates. @@ -730,8 +730,8 @@ distinction that matters: one pass in six and six in six are different findings. ## Cost The [versioned qualification policy](../docs/release-qualification.md#the-qualification-bar) -owns model pins and cell counts, including historical exceptions. Exact 10.0.0 -has 72 primary cells and 17 reserves on `openai/gpt-6.1-sol`. Current scenario +owns model pins and cell counts, including historical exceptions. 10.0.0/10.0.1 +have 72 primary cells and 17 reserves on `openai/gpt-6.1-sol`. Current scenario steps derive 90 primary dispatches, 23 reserve dispatches, and one route probe, for 91 planned or 114 maximum top-level dispatches. These remain an unapproved proposal to refresh at final candidate freeze. Paid work is HOLD until purpose, diff --git a/evals/release-policy.ts b/evals/release-policy.ts index 6af9ac45..f1855bfc 100644 --- a/evals/release-policy.ts +++ b/evals/release-policy.ts @@ -204,8 +204,10 @@ const V2_RELEASE: ReleaseProfile = { requiredModels: null, }; +const OPENAI_ONLY_V2_RELEASES = new Set(["10.0.0", "10.0.1"]); + export function releaseProfile(packageVersion: string): ReleaseProfile { - if (packageVersion === "10.0.0") + if (OPENAI_ONLY_V2_RELEASES.has(packageVersion)) return openAiOnlyRelease(DELIVERY_RELEASE_CATALOG, "gpt-6.1-sol"); if (targetsOpenCodeV2(packageVersion)) return V2_RELEASE; if (packageVersion === "9.6.0") @@ -226,7 +228,10 @@ export function releaseReviewerModel( packageVersion: string, managerModel?: ModelIdentity, ): ModelIdentity | null { - if (packageVersion === "9.6.0" || packageVersion === "10.0.0") { + if ( + packageVersion === "9.6.0" || + OPENAI_ONLY_V2_RELEASES.has(packageVersion) + ) { const model = releaseProfile(packageVersion).requiredModels?.[0]; if (!model) throw new Error("Pinned release reviewer model is absent."); return model; diff --git a/package.json b/package.json index 53d7238e..d45e02a5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "opencode-plugin-flow", - "version": "10.0.0", + "version": "10.0.1", "description": "Durable planning, validation, and independent review for OpenCode v2", "type": "module", "repository": { diff --git a/tests/distribution-and-surface.test.ts b/tests/distribution-and-surface.test.ts index 55ac25b7..4a06f9c8 100644 --- a/tests/distribution-and-surface.test.ts +++ b/tests/distribution-and-surface.test.ts @@ -11,6 +11,7 @@ import { import { homedir, tmpdir } from "node:os"; import { join } from "node:path"; import { z } from "zod"; +import packageJson from "../package.json" with { type: "json" }; import { createFlowCoreConfigEntries } from "../src/config-shared.js"; import { FLOW_GUIDANCE_IDS, @@ -224,7 +225,7 @@ describe("Flow distribution surface", () => { join(import.meta.dir, "../src/platform/opencode/plugin.ts"), ); expect(response.workflowData.runtimeIdentity).toEqual({ - packageVersion: "10.0.0", + packageVersion: packageJson.version, pluginEntrySha256: `sha256:${createHash("sha256").update(entry).digest("hex")}`, }); }); diff --git a/tests/eval-canary.test.ts b/tests/eval-canary.test.ts index 272c1462..b269f006 100644 --- a/tests/eval-canary.test.ts +++ b/tests/eval-canary.test.ts @@ -1313,133 +1313,135 @@ test("9.6 canary rejects every wrong-role model including a second reviewer", as } }); -test("10.0.0 canary rejects a self-consistent manager and reviewer on the wrong route", async () => { - const value = record({ - artifactValue: { ...artifact, packageVersion: "10.0.0" }, - }); - value.derivationVersion = "canary-evidence-v2"; - value.actors = ["manager", "reviewer"].map((role) => ({ - ...actor, - role: role === "manager" ? "manager" : "reviewer", - requestedModel: { - routeProvider: "other", - gateway: null, - family: "other", - model: "other", - revision: null, - }, - actualModel: { - kind: "observed", - value: { - routeProvider: "other", - gateway: null, - family: "other", - model: "other", - revision: null, - }, - }, - sessionIds: [""], - })); - const { recordSha256: _prior, ...base } = value; - const candidate = { ...base, recordSha256: canaryRecordSha256(base) }; - expect( - await canaryRecordIssue({ - version: "10.0.0", - record: candidate, - expectedArtifact: candidate.artifact, - directory: "/unused", - now: new Date("2026-08-25T00:00:01Z"), - }), - ).toBe("Canary manager model does not match the pinned release route."); -}); - -test("10.0.0 canary rejects missing and wrong-role identities before evidence access", async () => { - const model: ModelIdentity = { - routeProvider: "openai", - gateway: null, - family: "gpt-6.1-sol", - model: "gpt-6.1-sol", - revision: null, - }; - for (const variant of [ - "missing-manager", - "manager", - "reviewer", - "second-reviewer", - "observed-route", - "observed-model", - "family", - "gateway", - "revision", - ]) { +for (const version of ["10.0.0", "10.0.1"]) { + test(`${version} canary rejects a self-consistent manager and reviewer on the wrong route`, async () => { const value = record({ - artifactValue: { ...artifact, packageVersion: "10.0.0" }, + artifactValue: { ...artifact, packageVersion: version }, }); value.derivationVersion = "canary-evidence-v2"; value.actors = ["manager", "reviewer"].map((role) => ({ ...actor, role: role === "manager" ? "manager" : "reviewer", - requestedModel: model, - actualModel: { kind: "observed", value: model }, + requestedModel: { + routeProvider: "other", + gateway: null, + family: "other", + model: "other", + revision: null, + }, + actualModel: { + kind: "observed", + value: { + routeProvider: "other", + gateway: null, + family: "other", + model: "other", + revision: null, + }, + }, sessionIds: [""], })); - const reviewer = value.actors[1]; - if (!reviewer) throw new Error("Missing reviewer fixture."); - switch (variant) { - case "missing-manager": - value.actors = [reviewer]; - break; - case "manager": { - const manager = value.actors[0]; - if (!manager) throw new Error("Missing manager fixture."); - manager.requestedModel = { ...model, model: "other" }; - break; - } - case "reviewer": - reviewer.requestedModel = { ...model, model: "other" }; - break; - case "second-reviewer": - value.actors.push({ - ...reviewer, - requestedModel: { ...model, model: "other" }, - }); - break; - case "observed-route": - reviewer.actualModel = { - kind: "observed", - value: { ...model, routeProvider: "other" }, - }; - break; - case "observed-model": - reviewer.actualModel = { - kind: "observed", - value: { ...model, model: "other" }, - }; - break; - case "family": - reviewer.requestedModel = { ...model, family: "other" }; - break; - case "gateway": - reviewer.requestedModel = { ...model, gateway: "proxy" }; - break; - case "revision": - reviewer.requestedModel = { ...model, revision: "preview" }; - break; - } const { recordSha256: _prior, ...base } = value; const candidate = { ...base, recordSha256: canaryRecordSha256(base) }; expect( await canaryRecordIssue({ - version: "10.0.0", + version, record: candidate, expectedArtifact: candidate.artifact, directory: "/unused", now: new Date("2026-08-25T00:00:01Z"), }), - ).toBe( - variant === "missing-manager" - ? "Passed canaries require actors, session, and transcript evidence." - : `Canary ${variant === "manager" ? "manager" : "reviewer"} model does not match the pinned release route.`, - ); - } -}); + ).toBe("Canary manager model does not match the pinned release route."); + }); + + test(`${version} canary rejects missing and wrong-role identities before evidence access`, async () => { + const model: ModelIdentity = { + routeProvider: "openai", + gateway: null, + family: "gpt-6.1-sol", + model: "gpt-6.1-sol", + revision: null, + }; + for (const variant of [ + "missing-manager", + "manager", + "reviewer", + "second-reviewer", + "observed-route", + "observed-model", + "family", + "gateway", + "revision", + ]) { + const value = record({ + artifactValue: { ...artifact, packageVersion: version }, + }); + value.derivationVersion = "canary-evidence-v2"; + value.actors = ["manager", "reviewer"].map((role) => ({ + ...actor, + role: role === "manager" ? "manager" : "reviewer", + requestedModel: model, + actualModel: { kind: "observed", value: model }, + sessionIds: [""], + })); + const reviewer = value.actors[1]; + if (!reviewer) throw new Error("Missing reviewer fixture."); + switch (variant) { + case "missing-manager": + value.actors = [reviewer]; + break; + case "manager": { + const manager = value.actors[0]; + if (!manager) throw new Error("Missing manager fixture."); + manager.requestedModel = { ...model, model: "other" }; + break; + } + case "reviewer": + reviewer.requestedModel = { ...model, model: "other" }; + break; + case "second-reviewer": + value.actors.push({ + ...reviewer, + requestedModel: { ...model, model: "other" }, + }); + break; + case "observed-route": + reviewer.actualModel = { + kind: "observed", + value: { ...model, routeProvider: "other" }, + }; + break; + case "observed-model": + reviewer.actualModel = { + kind: "observed", + value: { ...model, model: "other" }, + }; + break; + case "family": + reviewer.requestedModel = { ...model, family: "other" }; + break; + case "gateway": + reviewer.requestedModel = { ...model, gateway: "proxy" }; + break; + case "revision": + reviewer.requestedModel = { ...model, revision: "preview" }; + break; + } + const { recordSha256: _prior, ...base } = value; + const candidate = { ...base, recordSha256: canaryRecordSha256(base) }; + expect( + await canaryRecordIssue({ + version, + record: candidate, + expectedArtifact: candidate.artifact, + directory: "/unused", + now: new Date("2026-08-25T00:00:01Z"), + }), + ).toBe( + variant === "missing-manager" + ? "Passed canaries require actors, session, and transcript evidence." + : `Canary ${variant === "manager" ? "manager" : "reviewer"} model does not match the pinned release route.`, + ); + } + }); +} diff --git a/tests/eval-release-sampling.test.ts b/tests/eval-release-sampling.test.ts index 0527094d..b5535bc5 100644 --- a/tests/eval-release-sampling.test.ts +++ b/tests/eval-release-sampling.test.ts @@ -1,5 +1,8 @@ import { describe, expect, test } from "bun:test"; -import draft from "../.agents/plans/18-opencode-v2/qualification-dispatch-draft.json" with { +import retainedDraft from "../.agents/plans/18-opencode-v2/qualification-dispatch-draft.json" with { + type: "json", +}; +import currentDraft from "../.agents/plans/20-flow10-maintenance-release/qualification-dispatch-draft.json" with { type: "json", }; import { hostConfigSha256 } from "../evals/provenance.js"; @@ -14,6 +17,7 @@ import { releaseHostConfigSha256, releaseHostPermissions, releasePolicySha256, + releaseReviewerModel, releaseScenarioCatalog, } from "../evals/release-policy.js"; import { @@ -210,7 +214,7 @@ describe("release eval sampling", () => { expect(code).toBe(expectedCode); if (expectedCode !== 0) expect(stderr).toContain( - "Release 10.0.0 requires exactly openai/gpt-6.1-sol with its canonical direct-route identity.", + `Release ${packageJson.version} requires exactly openai/gpt-6.1-sol with its canonical direct-route identity.`, ); } }); @@ -482,7 +486,7 @@ describe("release eval sampling", () => { ]); expect(exitCode).toBe(2); expect(stderr).toContain( - "Release 10.0.0 requires exactly openai/gpt-6.1-sol with its canonical direct-route identity.", + `Release ${packageJson.version} requires exactly openai/gpt-6.1-sol with its canonical direct-route identity.`, ); } }); @@ -523,47 +527,84 @@ describe("release eval sampling", () => { }); }); -test("current qualification draft counts actual scenario steps without authorizing a campaign", () => { - const policies = releaseCatalog("10.0.0"); - const scenarios = releaseScenarios("10.0.0"); - const cases = policies.map((policy) => { - const scenario = scenarios.find( - (scenario) => scenario.id === policy.caseId, +for (const [version, draft] of [ + ["10.0.0", retainedDraft], + ["10.0.1", currentDraft], +] as const) { + test(`${version} qualification draft counts actual scenario steps without authorizing a campaign`, () => { + const policies = releaseCatalog(version); + const scenarios = releaseScenarios(version); + const cases = policies.map((policy) => { + const scenario = scenarios.find( + (scenario) => scenario.id === policy.caseId, + ); + if (!scenario) throw new Error(`Missing scenario ${policy.caseId}.`); + const steps = scenario.steps.map(scenarioStepCatalog); + return { + caseId: policy.caseId, + stepsPerAttempt: steps.length, + primaryAttemptsPerModel: policy.minScoredAttempts, + primaryDispatchesPerModel: policy.minScoredAttempts * steps.length, + reserveDispatchesPerModel: steps.length, + }; + }); + expect(draft.status).toBe("draft-not-authorized"); + expect(draft.packageVersion).toBe(version); + expect(draft.modelSet).toEqual(["openai/gpt-6.1-sol"]); + expect(draft.managerModels).toBe(1); + expect(draft.cases).toEqual(cases); + expect( + cases.reduce((sum, row) => sum + row.primaryAttemptsPerModel, 0), + ).toBe(72); + expect(cases).toHaveLength(17); + const primary = cases.reduce( + (sum, row) => sum + row.primaryDispatchesPerModel, + 0, + ); + const reserves = cases.reduce( + (sum, row) => sum + row.reserveDispatchesPerModel, + 0, ); - if (!scenario) throw new Error(`Missing scenario ${policy.caseId}.`); - const steps = scenario.steps.map(scenarioStepCatalog); - return { - caseId: policy.caseId, - stepsPerAttempt: steps.length, - primaryAttemptsPerModel: policy.minScoredAttempts, - primaryDispatchesPerModel: policy.minScoredAttempts * steps.length, - reserveDispatchesPerModel: steps.length, + expect(primary).toBe(90); + expect(reserves).toBe(23); + expect(draft.primaryDispatches).toBe(primary); + expect(draft.reserveDispatches).toBe(reserves); + expect(draft.routeProbes).toBe(1); + expect(draft.plannedDispatches).toBe(91); + expect(draft.plannedDispatches).toBe(primary + draft.routeProbes); + expect(draft.totalDispatchLimit).toBe(114); + expect(draft.totalDispatchLimit).toBe( + primary + reserves + draft.routeProbes, + ); + }); +} + +for (const version of ["10.0.0", "10.0.1"]) { + test(`${version} binds the complete matrix and reviewer to canonical OpenAI`, () => { + const model = { + routeProvider: "openai", + gateway: null, + family: "gpt-6.1-sol", + model: "gpt-6.1-sol", + revision: null, }; + expect(() => assertReleaseModels([model], version)).not.toThrow(); + expect(releaseReviewerModel(version, { ...model, model: "other" })).toEqual( + model, + ); + expect(releaseCatalog(version)).toEqual(releaseCatalog("10.0.0")); + for (const models of [ + [{ ...model, model: "other" }], + [model, { ...model, routeProvider: "other" }], + ]) { + expect(() => assertReleaseModels(models, version)).toThrow( + "requires exactly openai/gpt-6.1-sol", + ); + } }); - expect(draft.status).toBe("draft-not-authorized"); - expect(draft.packageVersion).toBe("10.0.0"); - expect(draft.modelSet).toEqual(["openai/gpt-6.1-sol"]); - expect(draft.managerModels).toBe(1); - expect(draft.cases).toEqual(cases); - expect(cases.reduce((sum, row) => sum + row.primaryAttemptsPerModel, 0)).toBe( - 72, - ); - expect(cases).toHaveLength(17); - const primary = cases.reduce( - (sum, row) => sum + row.primaryDispatchesPerModel, - 0, - ); - const reserves = cases.reduce( - (sum, row) => sum + row.reserveDispatchesPerModel, - 0, +} +test("10.0.2 retains the default two-provider requirement", () => { + expect(releaseCatalog("10.0.2").every((row) => row.minProviders === 2)).toBe( + true, ); - expect(primary).toBe(90); - expect(reserves).toBe(23); - expect(draft.primaryDispatches).toBe(primary); - expect(draft.reserveDispatches).toBe(reserves); - expect(draft.routeProbes).toBe(1); - expect(draft.plannedDispatches).toBe(91); - expect(draft.plannedDispatches).toBe(primary + draft.routeProbes); - expect(draft.totalDispatchLimit).toBe(114); - expect(draft.totalDispatchLimit).toBe(primary + reserves + draft.routeProbes); }); diff --git a/tests/qualification-cli.test.ts b/tests/qualification-cli.test.ts index 60b4935b..def77d9e 100644 --- a/tests/qualification-cli.test.ts +++ b/tests/qualification-cli.test.ts @@ -56,7 +56,7 @@ import { assertQualificationBundle } from "../scripts/release-metadata.js"; import { autoQualifiedOutcome } from "./fixtures/auto-qualified-outcome.js"; test("current release defaults retain 17 cases on OpenAI and three autonomous attempts", () => { - expect(packageJson.version).toBe("10.0.0"); + expect(packageJson.version).toBe("10.0.1"); const scenarios = releaseScenarios(packageJson.version); const catalog = releaseCatalog(packageJson.version); expect(scenarios).toHaveLength(17); diff --git a/tests/release-1000.test.ts b/tests/release-1000.test.ts index da51fa9f..c3dbaf93 100644 --- a/tests/release-1000.test.ts +++ b/tests/release-1000.test.ts @@ -142,7 +142,7 @@ test("OpenAI-only 10.0.0 admits only the complete canonical direct identity", () }); test("OpenAI-only exception leaves later release defaults unchanged", () => { - for (const version of ["10.0.1", "10.1.0", "11.0.0"]) { + for (const version of ["10.0.2", "10.1.0", "11.0.0"]) { expect(releaseCatalog(version).map((row) => row.minProviders)).toEqual( Array(17).fill(2), );