diff --git a/.agents/plans/20-flow10-maintenance-release/README.md b/.agents/plans/20-flow10-maintenance-release/README.md
new file mode 100644
index 00000000..4885ce7e
--- /dev/null
+++ b/.agents/plans/20-flow10-maintenance-release/README.md
@@ -0,0 +1,34 @@
+# Flow 10.0.1 release
+
+Ship the reviewed CI and plugin maintenance changes from PRs #205 and #206.
+Session v5, tool inputs, prompts, validation, and review obligations stay intact.
+
+The runtime bytes changed, so the narrow offline patch exception does not apply.
+Qualify a new candidate; do not reuse the 10.0.0 matrix, canary, seal, or paid
+budget as candidate evidence. Keep those records immutable.
+
+Exact 10.0.1 uses the canonical direct `openai/gpt-6.1-sol` route for manager and
+reviewer, following the operator's OpenAI-only instruction. It retains all 17
+cases, their order, 72 primary cells, 17 declared environment reserves, original
+pass thresholds, zero false completions, zero unsubmitted reviews, and recovery
+off. Later versions keep the default two-provider requirement unless separately
+reviewed. No cross-provider reliability claim follows.
+
+Preparation: pass frozen install, the canonical gate, replay, audit, and packed
+native smoke; independently review the change; land it through CI; freeze exact
+Linux artifact and evaluator/source bindings. Use the existing native OpenCode 2
+OpenAI subscription account without API-key fallback. Readiness may perform no
+model generation and must pass before charging the first approved dispatch.
+
+The dispatch draft derives 90 primary steps, 23 reserve steps, and one route
+probe: 91 planned or 114 maximum top-level dispatches. This is a proposal, not
+permission or a ledger. Before any model start, obtain fresh approval binding
+purpose, artifact/source, canonical model, dispatch limit, and expiry. Old
+expired or exhausted grants cannot fund this release. Only the maintained CLI
+may activate declared environment reserves; product/evaluator failures never do.
+
+After matrix success, prepare the exact-artifact native-auth canary offline and
+present its separate bounded approval. Seal and regrade complete evidence, land
+it without changing product bytes, verify main CI and Release verification,
+then publish the approved tag through the project workflow. Check registry and
+GitHub downloads against the measured artifact before declaring release success.
diff --git a/.agents/plans/20-flow10-maintenance-release/qualification-dispatch-draft.json b/.agents/plans/20-flow10-maintenance-release/qualification-dispatch-draft.json
new file mode 100644
index 00000000..e996b085
--- /dev/null
+++ b/.agents/plans/20-flow10-maintenance-release/qualification-dispatch-draft.json
@@ -0,0 +1,134 @@
+{
+ "status": "draft-not-authorized",
+ "packageVersion": "10.0.1",
+ "managerModels": 1,
+ "modelSet": [
+ "openai/gpt-6.1-sol"
+ ],
+ "cases": [
+ {
+ "caseId": "delivery-summary-completed",
+ "stepsPerAttempt": 1,
+ "primaryAttemptsPerModel": 3,
+ "primaryDispatchesPerModel": 3,
+ "reserveDispatchesPerModel": 1
+ },
+ {
+ "caseId": "delivery-summary-deferred",
+ "stepsPerAttempt": 2,
+ "primaryAttemptsPerModel": 3,
+ "primaryDispatchesPerModel": 6,
+ "reserveDispatchesPerModel": 2
+ },
+ {
+ "caseId": "delivery-summary-observed-failure",
+ "stepsPerAttempt": 1,
+ "primaryAttemptsPerModel": 3,
+ "primaryDispatchesPerModel": 3,
+ "reserveDispatchesPerModel": 1
+ },
+ {
+ "caseId": "delivery-full-detail-followup",
+ "stepsPerAttempt": 2,
+ "primaryAttemptsPerModel": 3,
+ "primaryDispatchesPerModel": 6,
+ "reserveDispatchesPerModel": 2
+ },
+ {
+ "caseId": "delivery-idle-after-close",
+ "stepsPerAttempt": 2,
+ "primaryAttemptsPerModel": 3,
+ "primaryDispatchesPerModel": 6,
+ "reserveDispatchesPerModel": 2
+ },
+ {
+ "caseId": "happy-path",
+ "stepsPerAttempt": 1,
+ "primaryAttemptsPerModel": 3,
+ "primaryDispatchesPerModel": 3,
+ "reserveDispatchesPerModel": 1
+ },
+ {
+ "caseId": "plan-only-stops",
+ "stepsPerAttempt": 1,
+ "primaryAttemptsPerModel": 3,
+ "primaryDispatchesPerModel": 3,
+ "reserveDispatchesPerModel": 1
+ },
+ {
+ "caseId": "goal-change-refused",
+ "stepsPerAttempt": 2,
+ "primaryAttemptsPerModel": 3,
+ "primaryDispatchesPerModel": 6,
+ "reserveDispatchesPerModel": 2
+ },
+ {
+ "caseId": "continuation-accepted",
+ "stepsPerAttempt": 2,
+ "primaryAttemptsPerModel": 3,
+ "primaryDispatchesPerModel": 6,
+ "reserveDispatchesPerModel": 2
+ },
+ {
+ "caseId": "failing-gate-blocks",
+ "stepsPerAttempt": 1,
+ "primaryAttemptsPerModel": 10,
+ "primaryDispatchesPerModel": 10,
+ "reserveDispatchesPerModel": 1
+ },
+ {
+ "caseId": "resumes-after-interruption",
+ "stepsPerAttempt": 2,
+ "primaryAttemptsPerModel": 3,
+ "primaryDispatchesPerModel": 6,
+ "reserveDispatchesPerModel": 2
+ },
+ {
+ "caseId": "unprovable-claim-refused",
+ "stepsPerAttempt": 1,
+ "primaryAttemptsPerModel": 10,
+ "primaryDispatchesPerModel": 10,
+ "reserveDispatchesPerModel": 1
+ },
+ {
+ "caseId": "skipped-case-named-binding",
+ "stepsPerAttempt": 1,
+ "primaryAttemptsPerModel": 3,
+ "primaryDispatchesPerModel": 3,
+ "reserveDispatchesPerModel": 1
+ },
+ {
+ "caseId": "inspection-failed-audit-completes",
+ "stepsPerAttempt": 1,
+ "primaryAttemptsPerModel": 10,
+ "primaryDispatchesPerModel": 10,
+ "reserveDispatchesPerModel": 1
+ },
+ {
+ "caseId": "auto-two-features-evidence",
+ "stepsPerAttempt": 1,
+ "primaryAttemptsPerModel": 3,
+ "primaryDispatchesPerModel": 3,
+ "reserveDispatchesPerModel": 1
+ },
+ {
+ "caseId": "auto-prerequisite-repair",
+ "stepsPerAttempt": 1,
+ "primaryAttemptsPerModel": 3,
+ "primaryDispatchesPerModel": 3,
+ "reserveDispatchesPerModel": 1
+ },
+ {
+ "caseId": "auto-observe-with-required-pass",
+ "stepsPerAttempt": 1,
+ "primaryAttemptsPerModel": 3,
+ "primaryDispatchesPerModel": 3,
+ "reserveDispatchesPerModel": 1
+ }
+ ],
+ "primaryDispatches": 90,
+ "reserveDispatches": 23,
+ "routeProbes": 1,
+ "plannedDispatches": 91,
+ "totalDispatchLimit": 114
+}
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6f8badb4..747f47f5 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,21 @@
One short entry per release, written for users deciding whether to upgrade.
+## [10.0.1]
+
+- Simplify plugin tool response assembly and mutation routing while retaining
+ validation cancellation, reviewer-only writes, and exact completion replay.
+- Share the CI toolchain setup across normal checks, scheduled evals, and host
+ compatibility checks. Main reports candidate readiness without requiring
+ development bytes to match a previously published artifact; publication still
+ requires exact qualified bytes.
+- Session v5 schema, persisted state, commands, and tool inputs are unchanged.
+- Qualification uses `openai/gpt-6.1-sol` for manager and reviewer across all
+ 17 cases, followed by a fresh exact-artifact canary. No cross-provider
+ reliability claim follows.
+
+Upgrade with `opencode plugin add opencode-plugin-flow@10.0.1`.
+
## [10.0.0]
- Flow now targets OpenCode v2, with 2.1.4 as its tested minimum. OpenCode 1
diff --git a/README.md b/README.md
index 4f5034e0..9a079565 100644
--- a/README.md
+++ b/README.md
@@ -43,7 +43,7 @@ Install OpenCode v2 and the exact Flow release:
```sh
npm install -g @opencode/cli@2.1.4
opencode2 --version
-opencode2 plugin add opencode-plugin-flow@10.0.0
+opencode2 plugin add opencode-plugin-flow@10.0.1
```
The version command must report `opencode v2.1.4` or a later tested 2.x version.
@@ -57,7 +57,7 @@ For project scope, add this entry to the project's `opencode.json`:
```json
{
- "plugins": ["opencode-plugin-flow@10.0.0"]
+ "plugins": ["opencode-plugin-flow@10.0.1"]
}
```
@@ -74,7 +74,7 @@ Set plugin defaults:
```json
{
"plugins": [{
- "package": "opencode-plugin-flow@10.0.0",
+ "package": "opencode-plugin-flow@10.0.1",
"options": {
"reviewer": { "model": "provider/model", "steps": 80 },
"typesafeApiKeyEnv": "MY_TYPESAFE_KEY"
diff --git a/docs/adr/0018-opencode-v2-only.md b/docs/adr/0018-opencode-v2-only.md
index 7d2a067b..0a0301fe 100644
--- a/docs/adr/0018-opencode-v2-only.md
+++ b/docs/adr/0018-opencode-v2-only.md
@@ -46,8 +46,8 @@ variant. Reset restores its base. Native selection owns the coding model.
## Qualification
-Flow 10.0.0 pins `openai/gpt-6.1-sol` across 17 cases with 72 primary and 17
-reserve cells. Later v2 releases retain two routes and 144/34 cells. Reviewers
+10.0.0/10.0.1 pin `openai/gpt-6.1-sol` across 17 cases with 72 primary and 17
+reserve cells. Other v2 releases retain two routes and 144/34 cells. Reviewers
use the manager model with recovery off. Candidate, canary, and publication
bind the exact archive and unpacked manifest.
diff --git a/docs/development.md b/docs/development.md
index 39ac4a98..fc24269f 100644
--- a/docs/development.md
+++ b/docs/development.md
@@ -158,7 +158,7 @@ Follow the [frozen-candidate sequence](release-qualification.md#running-it):
finish fixes and dependency updates, pass deterministic checks, approve paid
evals.
`bun run qualify -- --campaign-dir
--canary ` seals the
-policy grid, canary and grader evidence. Exact 10.0.0 pins `openai/gpt-6.1-sol`.
+policy grid, canary and grader evidence. 10.0.0/10.0.1 pin `openai/gpt-6.1-sol`.
See [model policy](release-qualification.md#the-qualification-bar) for historical
exceptions and later defaults. Commit the bundle before tagging;
interrupted results cannot qualify.
diff --git a/docs/release-qualification.md b/docs/release-qualification.md
index 8a42b22b..42b276e0 100644
--- a/docs/release-qualification.md
+++ b/docs/release-qualification.md
@@ -9,7 +9,7 @@ This page owns release thresholds, candidate freezing, and publication order.
| Threshold | Value | Why |
| --- | --- | --- |
-| Distinct providers | 1 for 9.1.0, 9.2.0, 9.3.0, 9.4.0, 9.5.0, 9.6.0, and exact 10.0.0; otherwise 2 | 9.5.0, 9.6.0, and 10.0.0 pin `openai/gpt-6.1-sol`; earlier exceptions pin GPT-6 Sol. No cross-provider reliability claim. |
+| Distinct providers | 1 for 9.1.0, 9.2.0, 9.3.0, 9.4.0, 9.5.0, 9.6.0, and 10.0.0/10.0.1; otherwise 2 | 9.5.0, 9.6.0, and 10.0.0/10.0.1 pin `openai/gpt-6.1-sol`; earlier exceptions pin GPT-6 Sol. No cross-provider reliability claim. |
| False completions | 0 | A `completed` closure the document itself contradicts is the failure Flow exists to prevent. |
| Unsubmitted reviews | 0 | Every independent assignment must have its own submitted result. |
| Scored attempts per provider | 3 at 100%; 10 at 90% | The frozen release plan gives each threshold enough trials to express its allowed failures. |
@@ -54,8 +54,8 @@ Primary/reserve cells on GPT-6 Sol are 38/8 in 9.1.0 and 9.2.0, 48/9 in
9.3.0, and 57/12 in 9.4.0. The 9.4.0 profile adds three autonomous cases.
Version 9.5.0 keeps those twelve cases on GPT-6.1 Sol, with 57/12 cells.
Version 9.6.0 adds five delivery cases, collects them first, and uses 72/17
-cells. Exact 10.0.0 retains all 17 cases on `openai/gpt-6.1-sol` with 72 primary
-and 17 reserve cells. Later v2 releases retain two routes and 144/34 cells.
+cells. 10.0.0/10.0.1 retain all 17 cases on `openai/gpt-6.1-sol` with 72 primary
+and 17 reserve cells. Other v2 releases retain two routes and 144/34 cells.
Manager and reviewer share the approved model per cell. Recovery is off.
Other historical generic profiles use 96/18. Cell counts are not dispatch limits.
Changed order requires a new freeze and approval. Existing campaigns stay immutable; narrowed or merged reports
@@ -115,7 +115,7 @@ Keep that ledger across retries. Budget-stopped campaigns cannot qualify.
For native accounts, follow the [source setup](../evals/README.md#native-account-authentication).
Prepare with `--native-auth`, launch with `--native-auth-source`, and record with
the matching `--launch-receipt`. Live connection data remains private.
-For 9.1.0 through 9.4.0, pin GPT-6 Sol. Exact 9.5.0, 9.6.0, and 10.0.0 pin GPT-6.1 Sol.
+For 9.1.0 through 9.4.0, pin GPT-6 Sol. Exact 9.5.0, 9.6.0, and 10.0.0/10.0.1 pin GPT-6.1 Sol.
```bash
env -u TYPESAFE_API_KEY -u OPENCODE_FLOW_REVIEWER_MODEL \
@@ -133,7 +133,7 @@ The canary launcher needs its separately approved ledger in `FLOW_EVAL_AUTHORIZA
and provider access through its environment. It prints the native export path,
collects it before stopping the isolated host, and charges failed launches.
-Current 10.0.0 steps derive 90 primary dispatches, 23 reserves, and one route
+Current steps derive 90 primary dispatches, 23 reserves, and one route
probe, giving 91 planned or 114 maximum top-level dispatches. Historical 9.6.0
has the same counts. Internal turns and reviewer generations are separate;
no dollar cap follows. Canary approval is separate. Refresh the draft at final
diff --git a/evals/README.md b/evals/README.md
index 0098c1e1..9f215872 100644
--- a/evals/README.md
+++ b/evals/README.md
@@ -132,9 +132,9 @@ Ordinary runs use one sequential queue per model, with up to four queues in flig
Release mode is strictly sequential (`--concurrency 1`). Versions 9.1.0 and
9.2.0 pin `openai/gpt-6-sol` with 38 primary targets and eight reserves. Version
9.3.0 pins the same model with 48 primary targets and nine reserves. Version
-9.4.0 uses 57 primary targets and 12 reserves on the same route. Exact 10.0.0
-pins `openai/gpt-6.1-sol` across all 17 cases with 72 primary cells and 17 reserves.
-Later v2 releases retain two routes and 144/34 cells. The 10.0.0 exception makes
+9.4.0 uses 57 primary targets and 12 reserves on the same route. 10.0.0/10.0.1
+pin `openai/gpt-6.1-sol` across all 17 cases with 72 primary cells and 17 reserves.
+Other v2 releases retain two routes and 144/34 cells. These exceptions make
no cross-provider reliability claim.
Cells are scheduling units; count scenario steps and probes separately for the
dispatch authorization. Only retained retryable
@@ -569,8 +569,8 @@ about the prompts; a single attempt of a stochastic scenario is not a rate.
## Multi-model matrix
The [versioned policy](../docs/release-qualification.md#the-qualification-bar)
-owns model requirements. Exact 10.0.0 requires only `openai/gpt-6.1-sol` and makes
-no cross-provider reliability claim. Later v2 releases require two distinct
+owns model requirements. 10.0.0/10.0.1 require only `openai/gpt-6.1-sol` and makes
+no cross-provider reliability claim. Other v2 releases require two distinct
provider routes. This policy is not evidence that any candidate has qualified.
`.github/workflows/evals.yml`
runs the matrix weekly and on demand, outside contributor gates.
@@ -730,8 +730,8 @@ distinction that matters: one pass in six and six in six are different findings.
## Cost
The [versioned qualification policy](../docs/release-qualification.md#the-qualification-bar)
-owns model pins and cell counts, including historical exceptions. Exact 10.0.0
-has 72 primary cells and 17 reserves on `openai/gpt-6.1-sol`. Current scenario
+owns model pins and cell counts, including historical exceptions. 10.0.0/10.0.1
+have 72 primary cells and 17 reserves on `openai/gpt-6.1-sol`. Current scenario
steps derive 90 primary dispatches, 23 reserve dispatches, and one route probe,
for 91 planned or 114 maximum top-level dispatches. These remain an unapproved
proposal to refresh at final candidate freeze. Paid work is HOLD until purpose,
diff --git a/evals/release-policy.ts b/evals/release-policy.ts
index 6af9ac45..f1855bfc 100644
--- a/evals/release-policy.ts
+++ b/evals/release-policy.ts
@@ -204,8 +204,10 @@ const V2_RELEASE: ReleaseProfile = {
requiredModels: null,
};
+const OPENAI_ONLY_V2_RELEASES = new Set(["10.0.0", "10.0.1"]);
+
export function releaseProfile(packageVersion: string): ReleaseProfile {
- if (packageVersion === "10.0.0")
+ if (OPENAI_ONLY_V2_RELEASES.has(packageVersion))
return openAiOnlyRelease(DELIVERY_RELEASE_CATALOG, "gpt-6.1-sol");
if (targetsOpenCodeV2(packageVersion)) return V2_RELEASE;
if (packageVersion === "9.6.0")
@@ -226,7 +228,10 @@ export function releaseReviewerModel(
packageVersion: string,
managerModel?: ModelIdentity,
): ModelIdentity | null {
- if (packageVersion === "9.6.0" || packageVersion === "10.0.0") {
+ if (
+ packageVersion === "9.6.0" ||
+ OPENAI_ONLY_V2_RELEASES.has(packageVersion)
+ ) {
const model = releaseProfile(packageVersion).requiredModels?.[0];
if (!model) throw new Error("Pinned release reviewer model is absent.");
return model;
diff --git a/package.json b/package.json
index 53d7238e..d45e02a5 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
{
"name": "opencode-plugin-flow",
- "version": "10.0.0",
+ "version": "10.0.1",
"description": "Durable planning, validation, and independent review for OpenCode v2",
"type": "module",
"repository": {
diff --git a/tests/distribution-and-surface.test.ts b/tests/distribution-and-surface.test.ts
index 55ac25b7..4a06f9c8 100644
--- a/tests/distribution-and-surface.test.ts
+++ b/tests/distribution-and-surface.test.ts
@@ -11,6 +11,7 @@ import {
import { homedir, tmpdir } from "node:os";
import { join } from "node:path";
import { z } from "zod";
+import packageJson from "../package.json" with { type: "json" };
import { createFlowCoreConfigEntries } from "../src/config-shared.js";
import {
FLOW_GUIDANCE_IDS,
@@ -224,7 +225,7 @@ describe("Flow distribution surface", () => {
join(import.meta.dir, "../src/platform/opencode/plugin.ts"),
);
expect(response.workflowData.runtimeIdentity).toEqual({
- packageVersion: "10.0.0",
+ packageVersion: packageJson.version,
pluginEntrySha256: `sha256:${createHash("sha256").update(entry).digest("hex")}`,
});
});
diff --git a/tests/eval-canary.test.ts b/tests/eval-canary.test.ts
index 272c1462..b269f006 100644
--- a/tests/eval-canary.test.ts
+++ b/tests/eval-canary.test.ts
@@ -1313,133 +1313,135 @@ test("9.6 canary rejects every wrong-role model including a second reviewer", as
}
});
-test("10.0.0 canary rejects a self-consistent manager and reviewer on the wrong route", async () => {
- const value = record({
- artifactValue: { ...artifact, packageVersion: "10.0.0" },
- });
- value.derivationVersion = "canary-evidence-v2";
- value.actors = ["manager", "reviewer"].map((role) => ({
- ...actor,
- role: role === "manager" ? "manager" : "reviewer",
- requestedModel: {
- routeProvider: "other",
- gateway: null,
- family: "other",
- model: "other",
- revision: null,
- },
- actualModel: {
- kind: "observed",
- value: {
- routeProvider: "other",
- gateway: null,
- family: "other",
- model: "other",
- revision: null,
- },
- },
- sessionIds: [""],
- }));
- const { recordSha256: _prior, ...base } = value;
- const candidate = { ...base, recordSha256: canaryRecordSha256(base) };
- expect(
- await canaryRecordIssue({
- version: "10.0.0",
- record: candidate,
- expectedArtifact: candidate.artifact,
- directory: "/unused",
- now: new Date("2026-08-25T00:00:01Z"),
- }),
- ).toBe("Canary manager model does not match the pinned release route.");
-});
-
-test("10.0.0 canary rejects missing and wrong-role identities before evidence access", async () => {
- const model: ModelIdentity = {
- routeProvider: "openai",
- gateway: null,
- family: "gpt-6.1-sol",
- model: "gpt-6.1-sol",
- revision: null,
- };
- for (const variant of [
- "missing-manager",
- "manager",
- "reviewer",
- "second-reviewer",
- "observed-route",
- "observed-model",
- "family",
- "gateway",
- "revision",
- ]) {
+for (const version of ["10.0.0", "10.0.1"]) {
+ test(`${version} canary rejects a self-consistent manager and reviewer on the wrong route`, async () => {
const value = record({
- artifactValue: { ...artifact, packageVersion: "10.0.0" },
+ artifactValue: { ...artifact, packageVersion: version },
});
value.derivationVersion = "canary-evidence-v2";
value.actors = ["manager", "reviewer"].map((role) => ({
...actor,
role: role === "manager" ? "manager" : "reviewer",
- requestedModel: model,
- actualModel: { kind: "observed", value: model },
+ requestedModel: {
+ routeProvider: "other",
+ gateway: null,
+ family: "other",
+ model: "other",
+ revision: null,
+ },
+ actualModel: {
+ kind: "observed",
+ value: {
+ routeProvider: "other",
+ gateway: null,
+ family: "other",
+ model: "other",
+ revision: null,
+ },
+ },
sessionIds: [""],
}));
- const reviewer = value.actors[1];
- if (!reviewer) throw new Error("Missing reviewer fixture.");
- switch (variant) {
- case "missing-manager":
- value.actors = [reviewer];
- break;
- case "manager": {
- const manager = value.actors[0];
- if (!manager) throw new Error("Missing manager fixture.");
- manager.requestedModel = { ...model, model: "other" };
- break;
- }
- case "reviewer":
- reviewer.requestedModel = { ...model, model: "other" };
- break;
- case "second-reviewer":
- value.actors.push({
- ...reviewer,
- requestedModel: { ...model, model: "other" },
- });
- break;
- case "observed-route":
- reviewer.actualModel = {
- kind: "observed",
- value: { ...model, routeProvider: "other" },
- };
- break;
- case "observed-model":
- reviewer.actualModel = {
- kind: "observed",
- value: { ...model, model: "other" },
- };
- break;
- case "family":
- reviewer.requestedModel = { ...model, family: "other" };
- break;
- case "gateway":
- reviewer.requestedModel = { ...model, gateway: "proxy" };
- break;
- case "revision":
- reviewer.requestedModel = { ...model, revision: "preview" };
- break;
- }
const { recordSha256: _prior, ...base } = value;
const candidate = { ...base, recordSha256: canaryRecordSha256(base) };
expect(
await canaryRecordIssue({
- version: "10.0.0",
+ version,
record: candidate,
expectedArtifact: candidate.artifact,
directory: "/unused",
now: new Date("2026-08-25T00:00:01Z"),
}),
- ).toBe(
- variant === "missing-manager"
- ? "Passed canaries require actors, session, and transcript evidence."
- : `Canary ${variant === "manager" ? "manager" : "reviewer"} model does not match the pinned release route.`,
- );
- }
-});
+ ).toBe("Canary manager model does not match the pinned release route.");
+ });
+
+ test(`${version} canary rejects missing and wrong-role identities before evidence access`, async () => {
+ const model: ModelIdentity = {
+ routeProvider: "openai",
+ gateway: null,
+ family: "gpt-6.1-sol",
+ model: "gpt-6.1-sol",
+ revision: null,
+ };
+ for (const variant of [
+ "missing-manager",
+ "manager",
+ "reviewer",
+ "second-reviewer",
+ "observed-route",
+ "observed-model",
+ "family",
+ "gateway",
+ "revision",
+ ]) {
+ const value = record({
+ artifactValue: { ...artifact, packageVersion: version },
+ });
+ value.derivationVersion = "canary-evidence-v2";
+ value.actors = ["manager", "reviewer"].map((role) => ({
+ ...actor,
+ role: role === "manager" ? "manager" : "reviewer",
+ requestedModel: model,
+ actualModel: { kind: "observed", value: model },
+ sessionIds: [""],
+ }));
+ const reviewer = value.actors[1];
+ if (!reviewer) throw new Error("Missing reviewer fixture.");
+ switch (variant) {
+ case "missing-manager":
+ value.actors = [reviewer];
+ break;
+ case "manager": {
+ const manager = value.actors[0];
+ if (!manager) throw new Error("Missing manager fixture.");
+ manager.requestedModel = { ...model, model: "other" };
+ break;
+ }
+ case "reviewer":
+ reviewer.requestedModel = { ...model, model: "other" };
+ break;
+ case "second-reviewer":
+ value.actors.push({
+ ...reviewer,
+ requestedModel: { ...model, model: "other" },
+ });
+ break;
+ case "observed-route":
+ reviewer.actualModel = {
+ kind: "observed",
+ value: { ...model, routeProvider: "other" },
+ };
+ break;
+ case "observed-model":
+ reviewer.actualModel = {
+ kind: "observed",
+ value: { ...model, model: "other" },
+ };
+ break;
+ case "family":
+ reviewer.requestedModel = { ...model, family: "other" };
+ break;
+ case "gateway":
+ reviewer.requestedModel = { ...model, gateway: "proxy" };
+ break;
+ case "revision":
+ reviewer.requestedModel = { ...model, revision: "preview" };
+ break;
+ }
+ const { recordSha256: _prior, ...base } = value;
+ const candidate = { ...base, recordSha256: canaryRecordSha256(base) };
+ expect(
+ await canaryRecordIssue({
+ version,
+ record: candidate,
+ expectedArtifact: candidate.artifact,
+ directory: "/unused",
+ now: new Date("2026-08-25T00:00:01Z"),
+ }),
+ ).toBe(
+ variant === "missing-manager"
+ ? "Passed canaries require actors, session, and transcript evidence."
+ : `Canary ${variant === "manager" ? "manager" : "reviewer"} model does not match the pinned release route.`,
+ );
+ }
+ });
+}
diff --git a/tests/eval-release-sampling.test.ts b/tests/eval-release-sampling.test.ts
index 0527094d..b5535bc5 100644
--- a/tests/eval-release-sampling.test.ts
+++ b/tests/eval-release-sampling.test.ts
@@ -1,5 +1,8 @@
import { describe, expect, test } from "bun:test";
-import draft from "../.agents/plans/18-opencode-v2/qualification-dispatch-draft.json" with {
+import retainedDraft from "../.agents/plans/18-opencode-v2/qualification-dispatch-draft.json" with {
+ type: "json",
+};
+import currentDraft from "../.agents/plans/20-flow10-maintenance-release/qualification-dispatch-draft.json" with {
type: "json",
};
import { hostConfigSha256 } from "../evals/provenance.js";
@@ -14,6 +17,7 @@ import {
releaseHostConfigSha256,
releaseHostPermissions,
releasePolicySha256,
+ releaseReviewerModel,
releaseScenarioCatalog,
} from "../evals/release-policy.js";
import {
@@ -210,7 +214,7 @@ describe("release eval sampling", () => {
expect(code).toBe(expectedCode);
if (expectedCode !== 0)
expect(stderr).toContain(
- "Release 10.0.0 requires exactly openai/gpt-6.1-sol with its canonical direct-route identity.",
+ `Release ${packageJson.version} requires exactly openai/gpt-6.1-sol with its canonical direct-route identity.`,
);
}
});
@@ -482,7 +486,7 @@ describe("release eval sampling", () => {
]);
expect(exitCode).toBe(2);
expect(stderr).toContain(
- "Release 10.0.0 requires exactly openai/gpt-6.1-sol with its canonical direct-route identity.",
+ `Release ${packageJson.version} requires exactly openai/gpt-6.1-sol with its canonical direct-route identity.`,
);
}
});
@@ -523,47 +527,84 @@ describe("release eval sampling", () => {
});
});
-test("current qualification draft counts actual scenario steps without authorizing a campaign", () => {
- const policies = releaseCatalog("10.0.0");
- const scenarios = releaseScenarios("10.0.0");
- const cases = policies.map((policy) => {
- const scenario = scenarios.find(
- (scenario) => scenario.id === policy.caseId,
+for (const [version, draft] of [
+ ["10.0.0", retainedDraft],
+ ["10.0.1", currentDraft],
+] as const) {
+ test(`${version} qualification draft counts actual scenario steps without authorizing a campaign`, () => {
+ const policies = releaseCatalog(version);
+ const scenarios = releaseScenarios(version);
+ const cases = policies.map((policy) => {
+ const scenario = scenarios.find(
+ (scenario) => scenario.id === policy.caseId,
+ );
+ if (!scenario) throw new Error(`Missing scenario ${policy.caseId}.`);
+ const steps = scenario.steps.map(scenarioStepCatalog);
+ return {
+ caseId: policy.caseId,
+ stepsPerAttempt: steps.length,
+ primaryAttemptsPerModel: policy.minScoredAttempts,
+ primaryDispatchesPerModel: policy.minScoredAttempts * steps.length,
+ reserveDispatchesPerModel: steps.length,
+ };
+ });
+ expect(draft.status).toBe("draft-not-authorized");
+ expect(draft.packageVersion).toBe(version);
+ expect(draft.modelSet).toEqual(["openai/gpt-6.1-sol"]);
+ expect(draft.managerModels).toBe(1);
+ expect(draft.cases).toEqual(cases);
+ expect(
+ cases.reduce((sum, row) => sum + row.primaryAttemptsPerModel, 0),
+ ).toBe(72);
+ expect(cases).toHaveLength(17);
+ const primary = cases.reduce(
+ (sum, row) => sum + row.primaryDispatchesPerModel,
+ 0,
+ );
+ const reserves = cases.reduce(
+ (sum, row) => sum + row.reserveDispatchesPerModel,
+ 0,
);
- if (!scenario) throw new Error(`Missing scenario ${policy.caseId}.`);
- const steps = scenario.steps.map(scenarioStepCatalog);
- return {
- caseId: policy.caseId,
- stepsPerAttempt: steps.length,
- primaryAttemptsPerModel: policy.minScoredAttempts,
- primaryDispatchesPerModel: policy.minScoredAttempts * steps.length,
- reserveDispatchesPerModel: steps.length,
+ expect(primary).toBe(90);
+ expect(reserves).toBe(23);
+ expect(draft.primaryDispatches).toBe(primary);
+ expect(draft.reserveDispatches).toBe(reserves);
+ expect(draft.routeProbes).toBe(1);
+ expect(draft.plannedDispatches).toBe(91);
+ expect(draft.plannedDispatches).toBe(primary + draft.routeProbes);
+ expect(draft.totalDispatchLimit).toBe(114);
+ expect(draft.totalDispatchLimit).toBe(
+ primary + reserves + draft.routeProbes,
+ );
+ });
+}
+
+for (const version of ["10.0.0", "10.0.1"]) {
+ test(`${version} binds the complete matrix and reviewer to canonical OpenAI`, () => {
+ const model = {
+ routeProvider: "openai",
+ gateway: null,
+ family: "gpt-6.1-sol",
+ model: "gpt-6.1-sol",
+ revision: null,
};
+ expect(() => assertReleaseModels([model], version)).not.toThrow();
+ expect(releaseReviewerModel(version, { ...model, model: "other" })).toEqual(
+ model,
+ );
+ expect(releaseCatalog(version)).toEqual(releaseCatalog("10.0.0"));
+ for (const models of [
+ [{ ...model, model: "other" }],
+ [model, { ...model, routeProvider: "other" }],
+ ]) {
+ expect(() => assertReleaseModels(models, version)).toThrow(
+ "requires exactly openai/gpt-6.1-sol",
+ );
+ }
});
- expect(draft.status).toBe("draft-not-authorized");
- expect(draft.packageVersion).toBe("10.0.0");
- expect(draft.modelSet).toEqual(["openai/gpt-6.1-sol"]);
- expect(draft.managerModels).toBe(1);
- expect(draft.cases).toEqual(cases);
- expect(cases.reduce((sum, row) => sum + row.primaryAttemptsPerModel, 0)).toBe(
- 72,
- );
- expect(cases).toHaveLength(17);
- const primary = cases.reduce(
- (sum, row) => sum + row.primaryDispatchesPerModel,
- 0,
- );
- const reserves = cases.reduce(
- (sum, row) => sum + row.reserveDispatchesPerModel,
- 0,
+}
+test("10.0.2 retains the default two-provider requirement", () => {
+ expect(releaseCatalog("10.0.2").every((row) => row.minProviders === 2)).toBe(
+ true,
);
- expect(primary).toBe(90);
- expect(reserves).toBe(23);
- expect(draft.primaryDispatches).toBe(primary);
- expect(draft.reserveDispatches).toBe(reserves);
- expect(draft.routeProbes).toBe(1);
- expect(draft.plannedDispatches).toBe(91);
- expect(draft.plannedDispatches).toBe(primary + draft.routeProbes);
- expect(draft.totalDispatchLimit).toBe(114);
- expect(draft.totalDispatchLimit).toBe(primary + reserves + draft.routeProbes);
});
diff --git a/tests/qualification-cli.test.ts b/tests/qualification-cli.test.ts
index 60b4935b..def77d9e 100644
--- a/tests/qualification-cli.test.ts
+++ b/tests/qualification-cli.test.ts
@@ -56,7 +56,7 @@ import { assertQualificationBundle } from "../scripts/release-metadata.js";
import { autoQualifiedOutcome } from "./fixtures/auto-qualified-outcome.js";
test("current release defaults retain 17 cases on OpenAI and three autonomous attempts", () => {
- expect(packageJson.version).toBe("10.0.0");
+ expect(packageJson.version).toBe("10.0.1");
const scenarios = releaseScenarios(packageJson.version);
const catalog = releaseCatalog(packageJson.version);
expect(scenarios).toHaveLength(17);
diff --git a/tests/release-1000.test.ts b/tests/release-1000.test.ts
index da51fa9f..c3dbaf93 100644
--- a/tests/release-1000.test.ts
+++ b/tests/release-1000.test.ts
@@ -142,7 +142,7 @@ test("OpenAI-only 10.0.0 admits only the complete canonical direct identity", ()
});
test("OpenAI-only exception leaves later release defaults unchanged", () => {
- for (const version of ["10.0.1", "10.1.0", "11.0.0"]) {
+ for (const version of ["10.0.2", "10.1.0", "11.0.0"]) {
expect(releaseCatalog(version).map((row) => row.minProviders)).toEqual(
Array(17).fill(2),
);