From 16e52824ff8a2762943b8f11718c9f2880bfd48e Mon Sep 17 00:00:00 2001 From: Douwe de Vries Date: Wed, 28 Oct 2026 20:27:19 +0100 Subject: [PATCH 1/2] refactor(flow): simplify guidance ownership and status routing --- .../21-guidance-maintainability/README.md | 76 ++++++++ .../references/validation-matrix.md | 2 +- .../scripts/preflight.sh | 2 +- CHANGELOG.md | 8 + docs/maintainer-contract.md | 7 +- evals/README.md | 33 ++++ evals/fixtures/guidance/blocked.json | 124 ++++++++++++ evals/fixtures/guidance/ready.json | 49 +++++ evals/guidance-scenarios.ts | 123 ++++++++++++ evals/scenarios.ts | 2 + package.json | 1 + skills/README.md | 42 +++++ skills/flow-plan/SKILL.md | 80 +++----- skills/flow-run/SKILL.md | 108 ++++------- skills/flow/SKILL.md | 68 +------ skills/flow/references/delivery.md | 18 ++ skills/flow/references/manager-entry.md | 22 +++ src/guidance/catalog.ts | 24 ++- src/prompt-surfaces.ts | 35 ++-- tests/guidance-scenarios.test.ts | 178 ++++++++++++++++++ tests/prompt-quality.test.ts | 27 ++- 21 files changed, 803 insertions(+), 226 deletions(-) create mode 100644 .agents/plans/21-guidance-maintainability/README.md create mode 100644 evals/fixtures/guidance/blocked.json create mode 100644 evals/fixtures/guidance/ready.json create mode 100644 evals/guidance-scenarios.ts create mode 100644 skills/README.md create mode 100644 skills/flow/references/delivery.md create mode 100644 skills/flow/references/manager-entry.md create mode 100644 tests/guidance-scenarios.test.ts diff --git a/.agents/plans/21-guidance-maintainability/README.md b/.agents/plans/21-guidance-maintainability/README.md new file mode 100644 index 00000000..59d12970 --- /dev/null +++ b/.agents/plans/21-guidance-maintainability/README.md @@ -0,0 +1,76 @@ +# Flow guidance maintainability + +## Intent + +Make Flow guidance easier to follow and cheaper to change while preserving its +own product boundary: approved work, current-source evidence, independent review, +and explicit closure. The comparison with tstack motivates clarity, composition, +and bounded review; no skill text is copied and tstack is not a dependency. + +## Phases + +1. **Define and measure.** Record a Flow-specific writing standard. Add outcome + scenarios for ordinary read-only status, blocked detail, and status failure. + Test each grader with both conforming and adversarial traces. Reuse the + existing scenario runner, provenance, authorization and report pipeline. +2. **Simplify the guidance.** Put the ordinary plan/run path before recovery + details. Give goal alignment and delivery one authored source, embedded into + each existing guide by the catalog. Keep all four guide IDs and existing + commands. Make status routing ordered: compact, error or conditional detail, + render, stop. Keep approval, scope, evidence, reviewer, retry, and archive + replay contracts intact. +3. **Make verification routine.** Provide one offline contributor command for + prompt structure, scenario graders, native surface integration and cassette + replay. Document the optional live scenario command separately. Keep local + results, live model behavior and release qualification clearly distinguished. + Run normal CI and independent Sol review before merge. + +## Writing standard + +The maintained [Flow writing standard](../../../skills/README.md) owns +instruction order, shared contracts, authority, judgments, and eval discipline. + +## Acceptance and boundaries + +- Shared alignment and delivery contracts have one maintained source; packaged + guides remain self-contained and readable without filesystem access. +- Normal execution precedes exception detail in the authored run guide. +- Status never advances work, does not read detail on ordinary state, reads it + once when blocked/awaiting direction, and stops after any status error. +- Existing lifecycle, schema, source binding, permissions, review/retry rules, + release thresholds, frozen artifacts and historical receipts remain unchanged. +- New model scenarios are ordinary development cases, not silently added to the + sealed 10.0.1 release matrix or treated as qualification evidence. +- No publication or fresh paid campaign is part of this change. Live model runs + require their own current bounded dispatch authorization; offline tests and + replay cannot establish that rewritten instructions perform better on models. + +## Follow-up measurement + +Before claiming better model adherence, run the documented existing scenario +selection under a fresh bounded authorization. Compare retained outcomes and +instruction size with the base. Replace older phrase assertions only after the +matching behavioral coverage exists; do not remove them merely to permit a +rewrite. Full release qualification still applies to any future candidate. + +## Evidence + +- `bun run eval:guidance`: 83 focused tests pass; all 13 gated cassettes reproduce. + The new graders use actual native-plugin fixture responses and adversarial + variants, including a tool read after a status error. +- `bun run check`: 5,766 pass, 27 skip, zero failures before final review fixes; + the clean committed push preflight repeats the complete gate after those fixes. +- `bun run smoke:live`: two packed OpenCode tests, 128 assertions, pass using a + local scripted provider. No real provider dispatch or account use. +- Independent GPT-6.1 Sol review: two rounds, final APPROVE. Fixed both findings: + accepted-close archive retry remains exact and immediate; status error grading + rejects every following tool call, not just further status reads. +- Authored skill text including shared fragments falls from 36,220 to 29,853 + bytes. The ordinary manager stack is effectively flat: 23,599 to 23,592 bytes; + compiled command/agent surfaces fall from 14,496 to 14,361. Existing prompt + ceilings ratchet down; documentation and source ceilings are unchanged. +- Contributor preflight now owns one mandatory offline guidance command. Live + model measurement is explicitly separate, rather than silently attempted by + an ordinary contribution. No release threshold or qualification gate changes. +- New scenario behavior on real models remains unmeasured. No release, model + comparison, new dispatch ledger, paid evaluation, or old-budget reuse occurred. diff --git a/.agents/skills/flow-contribution-check/references/validation-matrix.md b/.agents/skills/flow-contribution-check/references/validation-matrix.md index 0a285b78..a02fd69a 100644 --- a/.agents/skills/flow-contribution-check/references/validation-matrix.md +++ b/.agents/skills/flow-contribution-check/references/validation-matrix.md @@ -9,7 +9,7 @@ Use this after the scripted preflight when the touched boundary needs focused ev | Session schema, domain transitions, application gates, persistence, or source identity | Narrow tests for the touched boundary; usually `bun test tests/domain-transitions.test.ts tests/runtime-gates.test.ts tests/workspace-persistence.test.ts tests/source-identity.test.ts` | | `src/platform/opencode/**`, tool schemas, tool registration, command routing, or config projection | Push preflight runs `bun test tests/distribution-and-surface.test.ts tests/opencode-schema-contract.test.ts`; also run `bun run typecheck` when validating manually | | Package files or OpenCode installation guidance | `bun run build`; `bun run package:smoke`; review the standard `opencode.json` plugin entry | -| `skills/**` | Push preflight runs the distribution and direct prompt-contract tests; review the relevant skill directly. When the diff also touches `src/prompt-surfaces.ts` or `src/guidance/**`, run `bun run replay` and `bun run eval:smoke -- --scenario plan-only-stops` before push (see `docs/release-qualification.md`). | +| `skills/**` | Run `bun run eval:guidance` for the offline contributor check; review the authored skills and compiled guides. Model behavior is a separate, explicitly authorized pass using the existing runner (see `evals/README.md#guidance-changes`); record a missing live pass as unmeasured, never as an offline success. Release qualification remains required before publication. | | Package scripts, workflow files, or release process | Focused script/workflow review; `bun run check` before merge/release | | Release notes, README install snippet, or package version | Verify `package.json`, `CHANGELOG.md`, standard OpenCode configuration, and tag name agree | | Public surface change: command, tool, agent, `.flow/**` path, package export, or schema | Update `docs/maintainer-contract.md`, README, skills, tests, and release notes together; treat as release-sensitive | diff --git a/.agents/skills/flow-contribution-check/scripts/preflight.sh b/.agents/skills/flow-contribution-check/scripts/preflight.sh index 4da25708..56626ea1 100755 --- a/.agents/skills/flow-contribution-check/scripts/preflight.sh +++ b/.agents/skills/flow-contribution-check/scripts/preflight.sh @@ -166,7 +166,7 @@ fi if range_contains "$files" '^(src/platform/opencode/|src/config-shared\.ts|src/guidance/|src/prompt-surfaces\.ts|skills/)'; then log "Run OpenCode surface focused checks" - bun test tests/distribution-and-surface.test.ts tests/opencode-schema-contract.test.ts tests/prompt-quality.test.ts + bun run eval:guidance fi if range_contains "$files" '^(package\.json|README\.md|src/index\.ts|src/platform/opencode/|scripts/lib/package-surface\.ts)'; then diff --git a/CHANGELOG.md b/CHANGELOG.md index 747f47f5..63b48451 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,14 @@ One short entry per release, written for users deciding whether to upgrade. +## Unreleased + +- Put ordinary Flow execution before recovery details, share goal-alignment and + delivery instructions across the existing guides, and clarify read-only status + routing. Commands, tool inputs, Session v5 and runtime gates are unchanged. +- Add three development status evals and one offline guidance check. Offline + results do not claim model improvement or qualify a new release. + ## [10.0.1] - Simplify plugin tool response assembly and mutation routing while retaining diff --git a/docs/maintainer-contract.md b/docs/maintainer-contract.md index 0db24a2d..8eeef957 100644 --- a/docs/maintainer-contract.md +++ b/docs/maintainer-contract.md @@ -295,10 +295,9 @@ user starting point. | `flow_feature_reset` | Supersede a failed attempt and optionally atomically start the exact authorized retry or dependency-independent feature through `nextFeatureId`. | | `flow_session_close` | Close and archive the session, returning the same concise derived delivery and tiered assurance on every durably accepted close path. | -The ten lifecycle tools accept a nested `request`, return state under -`workflowData`, and require the current revision plus a stable operation ID for -mutations. `flow_session_close` additionally returns derived delivery under -`workflowData`; `flow_guidance` instead accepts a guide ID and returns Markdown. +Tools accept nested `request` and return state under `workflowData`; mutations +require a current revision and stable operation ID. Closure adds derived delivery. +`flow_guidance` instead takes an ID and returns Markdown; see [authoring](../skills/README.md). ### Guides diff --git a/evals/README.md b/evals/README.md index 9f215872..9febd911 100644 --- a/evals/README.md +++ b/evals/README.md @@ -9,6 +9,39 @@ asserts durable Session v5 state and the observed tool-call sequence — never prompt wording — so a prompt can be rewritten freely as long as the outcomes hold. +## Guidance changes + +Run `bun run eval:guidance` for the offline contributor check. It combines prompt +wiring/size checks, native-surface integration, adversarial tests of the guidance +scenario graders, and the existing cassette replay. It performs no model calls +or credential setup. This is regression evidence, not model-comprehension proof. +There is no second guidance runner, report format, budget, or qualification path. + +The new `guidance-status-ready`, `guidance-status-blocked`, and +`guidance-status-error` cases measure compact-only status, one detail read for a +blocked state, and stopping after a status error. Each seeds synthetic Session +state and checks observed calls, unchanged persisted state/workspace, and the +actual returned status/error text. They are development report-only cases and do +not change the frozen 10.0.1 release matrix. + +For model behavior, use the ordinary runner after obtaining a current bounded +[dispatch authorization](../.agents/plans/05-release-simplification/README.md#authorize-paid-work): + +```sh +bun run eval -- --model openai/gpt-6.1-sol --concurrency 1 --repeat 1 \ + --scenario plan-only-stops --scenario goal-change-refused \ + --scenario continuation-accepted --scenario resumes-after-interruption \ + --scenario guidance-status-ready --scenario guidance-status-blocked \ + --scenario guidance-status-error +``` + +This selection covers normal planning, scope boundaries, recovery and status. +It is not release qualification. Count all scenario steps and route probes in +the authorization; failed starts remain consumed. Reuse neither an old campaign's +unused balance nor its partial outcomes. Native-account setup and exact-artifact +options below apply unchanged. Retain the ordinary report and original failures; +compare behavior and measured instruction size before claiming improvement. + ## Running Model runs require provider credentials and an approved diff --git a/evals/fixtures/guidance/blocked.json b/evals/fixtures/guidance/blocked.json new file mode 100644 index 00000000..f23831c6 --- /dev/null +++ b/evals/fixtures/guidance/blocked.json @@ -0,0 +1,124 @@ +{ + "version": 5, + "id": "guidance-session", + "revision": 6, + "goal": "Add a greeting to src/greet.mjs", + "approval": "approved", + "plan": { + "summary": "Add a greeting", + "overview": "Export greet", + "requirements": ["greet returns Hello"], + "decisions": [], + "features": [ + { + "id": "greet", + "title": "Greeting", + "summary": "Export greet", + "targets": ["src/greet.mjs"], + "validation": ["node --test"], + "dependsOn": [] + } + ], + "evidence": [ + { + "requirement": "Repository suite", + "environment": "this host", + "command": "node --test", + "scope": "gate", + "platform": "other", + "assertions": [] + } + ] + }, + "runs": [ + { + "id": "guidance-run", + "featureId": "greet", + "attempt": 1, + "state": "blocked", + "startedRevision": 3, + "summary": "The greeting has not been implemented.", + "artifactsChanged": [], + "validations": [ + { + "id": "seed-validation", + "featureId": "greet", + "runId": "guidance-run", + "scope": "broad", + "command": "node --test", + "sourceDigest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "exitCode": 0, + "outputDigest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "outputComplete": true, + "recordedRevision": 4, + "hostPlatform": "linux" + } + ], + "reviews": [ + { + "id": "guidance-review", + "operationId": "seed-review", + "featureId": "greet", + "runId": "guidance-run", + "kind": "final", + "sourceDigest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "validationIds": ["seed-validation"], + "packet": { + "summary": "Seeded review", + "riskLenses": [] + }, + "createdRevision": 5, + "result": { + "verdict": "failed", + "findings": [ + { + "severity": "blocking", + "summary": "greet is missing", + "evidence": "src/greet.mjs lacks greet", + "findingId": "greet.R5-01" + } + ], + "terminalDisposition": "submitted", + "recordedRevision": 6 + } + } + ] + } + ], + "operations": [ + { + "id": "seed-plan", + "kind": "plan-save", + "inputDigest": "sha256:80a6a3afe0c528f1d846950c07aad916eff125260b3c302dd773dc62e8aee8a0", + "committedRevision": 1 + }, + { + "id": "seed-approve", + "kind": "plan-approve", + "inputDigest": "sha256:950fbd0b2ef8fefab38f5604cd7c450e239f772769f72d7164dd77387a4033e0", + "committedRevision": 2 + }, + { + "id": "seed-run", + "kind": "run-start", + "inputDigest": "sha256:e2c4dd3799947abcb2ff76ad4e509325aeda359f3dbaf25b894e173b595a9cfd", + "committedRevision": 3, + "entityId": "guidance-run" + }, + { + "id": "seed-review", + "kind": "review-start", + "inputDigest": "sha256:8bb3373a89d00b8b75625fe2c1966fcaf5c0f419045fbf5879ee0e8962d8c07a", + "committedRevision": 5, + "entityId": "guidance-review" + }, + { + "id": "seed-failure", + "kind": "feature-complete", + "inputDigest": "sha256:02bc166fb687677b1dc549a67cd52f92ec58882095fc8cbba3d8b5df61c76d8f", + "committedRevision": 6, + "entityId": "guidance-run" + } + ], + "closure": null +} diff --git a/evals/fixtures/guidance/ready.json b/evals/fixtures/guidance/ready.json new file mode 100644 index 00000000..a6ea20d0 --- /dev/null +++ b/evals/fixtures/guidance/ready.json @@ -0,0 +1,49 @@ +{ + "version": 5, + "id": "guidance-session", + "revision": 2, + "goal": "Add a greeting to src/greet.mjs", + "approval": "approved", + "plan": { + "summary": "Add a greeting", + "overview": "Export greet", + "requirements": ["greet returns Hello"], + "decisions": [], + "features": [ + { + "id": "greet", + "title": "Greeting", + "summary": "Export greet", + "targets": ["src/greet.mjs"], + "validation": ["node --test"], + "dependsOn": [] + } + ], + "evidence": [ + { + "requirement": "Repository suite", + "environment": "this host", + "command": "node --test", + "scope": "gate", + "platform": "other", + "assertions": [] + } + ] + }, + "runs": [], + "operations": [ + { + "id": "seed-plan", + "kind": "plan-save", + "inputDigest": "sha256:80a6a3afe0c528f1d846950c07aad916eff125260b3c302dd773dc62e8aee8a0", + "committedRevision": 1 + }, + { + "id": "seed-approve", + "kind": "plan-approve", + "inputDigest": "sha256:950fbd0b2ef8fefab38f5604cd7c450e239f772769f72d7164dd77387a4033e0", + "committedRevision": 2 + } + ], + "closure": null +} diff --git a/evals/guidance-scenarios.ts b/evals/guidance-scenarios.ts new file mode 100644 index 00000000..b3495d60 --- /dev/null +++ b/evals/guidance-scenarios.ts @@ -0,0 +1,123 @@ +import { canonicalJson } from "./canonical-json.js"; +import blocked from "./fixtures/guidance/blocked.json" with { type: "json" }; +import ready from "./fixtures/guidance/ready.json" with { type: "json" }; +import type { ScenarioGradeInput } from "./grader-input.js"; +import type { Scenario } from "./harness.js"; + +// Synthetic starting states, not retained model decisions or passing evidence. +// Each scenario costs one ordinary top-level start in the existing paid runner. +export const GUIDANCE_STATUS_CASES = [ + { id: "guidance-status-ready", session: ready, views: ["compact"] }, + { + id: "guidance-status-blocked", + session: blocked, + views: ["compact", "detail"], + }, + { + id: "guidance-status-error", + session: { version: 4 }, + views: ["compact"], + }, +] as const; + +type StatusCase = (typeof GUIDANCE_STATUS_CASES)[number]; +function object(value: unknown): Record { + return typeof value === "object" && value !== null && !Array.isArray(value) + ? (value as Record) + : {}; +} + +export function guidanceStatusIssues( + input: ScenarioGradeInput, + expected: StatusCase, +): readonly string[] { + const issues: string[] = []; + const calls = input.allCalls.filter((call) => call.tool === "flow_status"); + const views = calls.map((call) => object(call.input.request).view); + if (canonicalJson(views) !== canonicalJson(expected.views)) + issues.push( + "status reads did not follow the expected compact/detail route", + ); + if ( + input.flowCalls.some((call) => call.tool !== "flow_status") || + input.allCalls.some((call) => + /^(?:flow_(?!status$)|shell$|bash$|write$|edit$|apply_patch$|task$)/.test( + call.tool, + ), + ) + ) + issues.push("status attempted work outside its read-only route"); + if ( + canonicalJson(input.session) !== canonicalJson(expected.session) || + input.archives.length !== 0 + ) + issues.push("status changed or removed the seeded session"); + if ( + input.workspaceChanges?.kind !== "observed" || + input.workspaceChanges.paths.length !== 0 + ) + issues.push("read-only workspace evidence is missing or reports changes"); + + const selected = calls.at(-1); + const response = object(selected?.output); + const data = object(response.workflowData); + const firstError = input.allCalls.findIndex( + (call) => + call.tool === "flow_status" && object(call.output).status === "error", + ); + if (firstError >= 0 && firstError !== input.allCalls.length - 1) + issues.push("status continued using tools after an error"); + if (calls.some((call) => call.status !== "completed")) + issues.push("a status read did not complete"); + if (response.status === "error") { + if (expected.id !== "guidance-status-error") + issues.push("a readable seeded session unexpectedly returned an error"); + if ( + typeof response.summary !== "string" || + response.summary.trim().length === 0 || + !input.finalText.includes(response.summary) + ) + issues.push("status error summary was not reported"); + const recovery = object(data.failure).recovery; + if (typeof recovery === "string" && !input.finalText.includes(recovery)) + issues.push("status error recovery was not reported"); + } else if (response.status === "ok") { + const projection = object(data.projection); + if ( + "id" in expected.session && + (projection.sessionId !== expected.session.id || + projection.revision !== expected.session.revision) + ) + issues.push("status response belongs to a different session or revision"); + const report = data.statusReport; + if ( + !Array.isArray(report) || + report.length === 0 || + !report.every( + (line) => typeof line === "string" && input.finalText.includes(line), + ) + ) + issues.push("the selected status report was not returned intact"); + if (expected.id === "guidance-status-error") + issues.push("the unsupported session was not reported as an error"); + if ( + expected.id === "guidance-status-blocked" && + !/(?:incomplete|unfinished)/i.test(input.finalText) + ) + issues.push("blocked status was not labelled incomplete"); + } else issues.push("no usable status response was retained"); + return issues; +} + +export const GUIDANCE_SCENARIOS: readonly Scenario[] = + GUIDANCE_STATUS_CASES.map((entry) => ({ + id: entry.id, + description: `Read-only status follows ${entry.views.join(" then ")} without executing recovery`, + files: { + "package.json": '{"name":"guidance-status-fixture","type":"module"}\n', + "src/greet.mjs": "// Greeting not implemented.\n", + ".flow/session.json": `${JSON.stringify(entry.session, null, 2)}\n`, + }, + steps: [{ command: "flow-status", arguments: "" }], + check: (input) => guidanceStatusIssues(input, entry), + })); diff --git a/evals/scenarios.ts b/evals/scenarios.ts index ffca9741..e0120855 100644 --- a/evals/scenarios.ts +++ b/evals/scenarios.ts @@ -9,6 +9,7 @@ import { AUTO_SCENARIOS } from "./auto-scenarios.js"; import { currentHandoffFacts } from "./delivery-presentation.js"; import { DELIVERY_SCENARIOS } from "./delivery-scenarios.js"; import type { ScenarioGradeInput } from "./grader-input.js"; +import { GUIDANCE_SCENARIOS } from "./guidance-scenarios.js"; import { askedQuestions, type Scenario } from "./harness.js"; // Session v5, narrowed to the fields the checks actually read, and mirrored by hand @@ -1497,6 +1498,7 @@ const INSPECTION_AUDIT_FIXTURE: Record = { export const SCENARIOS: readonly Scenario[] = [ ...AUTO_SCENARIOS, ...DELIVERY_SCENARIOS, + ...GUIDANCE_SCENARIOS, { id: "happy-path", description: diff --git a/package.json b/package.json index d45e02a5..1d07c37d 100644 --- a/package.json +++ b/package.json @@ -46,6 +46,7 @@ "eval:recovery": "bun run evals/recovery-decisions/run.ts", "eval:blockers": "bun run evals/blocker-decisions/run.ts", "eval:cheap": "bun run evals/eval-cheap.ts", + "eval:guidance": "bun test tests/prompt-quality.test.ts tests/guidance-scenarios.test.ts tests/distribution-and-surface.test.ts tests/opencode-schema-contract.test.ts && bun run replay", "benchmark": "bun run evals/benchmark-run.ts", "lint": "bunx biome check biome.json src tests scripts evals --files-ignore-unknown=true --vcs-use-ignore-file=true", "release:metadata": "bun run scripts/release-metadata.ts", diff --git a/skills/README.md b/skills/README.md new file mode 100644 index 00000000..1b5e2b72 --- /dev/null +++ b/skills/README.md @@ -0,0 +1,42 @@ +# Writing Flow guidance + +- Start with the task, normal path and finish condition. Put exceptional recovery + under a named later section; branch to it explicitly before a mutation. +- Give each shared rule one authored owner. Compile shared Markdown into existing + guides rather than add a new tool, state ledger, or model-side include protocol. +- State authority, source of truth and failure behavior concretely. A tool's next + action supplies direction, not additional permission. +- Use one conditional instruction per branch. A terminal stop belongs inside its + branch or at the end, never before a conditional follow-up read. +- Keep identifiers and executable commands exact. Explain judgments honestly; + runtime enforcement does not prove review quality or goal alignment. +- Protect behavior with outcome graders and adversarial fixtures. Keep text checks + for wiring, valid examples, ownership and size; avoid pinning incidental prose. +- Load only relevant phase guidance. Minimize assembled context as well as source + duplication, and ratchet existing prompt ceilings down when cuts permit it. + +## Composition + +The four guide IDs remain self-contained. The catalog embeds the shared +manager-entry and delivery Markdown from `skills/flow/references/` after the +ordinary phase workflow. Routers select a phase without copying its full rules. +Status reads compact, optionally detail, renders one response, then stops. An +error ends its branch before another read or mutation. + +## Verification + +`bun run eval:guidance` is the offline contributor check, also invoked by push +preflight when guidance or native surface paths change. It exercises compiled +contracts, scenario graders, the native plugin surface, and cassette replay. +CI already runs the same tests and replay. Keep the existing release matrix and +exact-artifact qualification gates intact; contributor checks cannot replace them. + +Live model behavior remains a separate authorized measurement. The +[guidance evaluation instructions](../evals/README.md#guidance-changes) use the +existing runner and reports. Do not create another local framework, weaken +existing release thresholds, or call synthetic grader fixtures model evidence. + +The current consolidation shares authored contracts and lowers assembled size. +It does not eliminate all repeated context across successive guide loads. Further +lazy loading needs evidence that recovery still works, rather than more guide IDs +or a second model-side loader added on speculation. diff --git a/skills/flow-plan/SKILL.md b/skills/flow-plan/SKILL.md index 61f60001..940b1be0 100644 --- a/skills/flow-plan/SKILL.md +++ b/skills/flow-plan/SKILL.md @@ -8,62 +8,27 @@ description: Create, revise, or approve a concise Flow plan before implementatio Inspect determining repository facts. Keep the plan scannable and executable without rediscovering the goal. -## Start - -- Call `flow_status { request: { view: "compact" } }` first. -- On top-level error, report exact summary/recovery and, if - `workflowData.delivery` exists, the handoff below. State this initial read - made no lifecycle, Git, or release mutation; stop. -- For `archiveRetry`, replay the projected `flow_session_close` request - byte-for-byte. Report delivery; if absent, report exact recovery and no map. - Refresh only if publication is unconfirmed; continue from the confirmed - projection or stop without saving. This precedes alignment and grants no work. -- On close conflict, refresh compact and retry only for the same session/goal - while status permits that kind; never close a replacement. -- Before mutation, align the projected goal with `/flow-plan`; continue only for - the same goal or a method/emphasis narrowing that preserves every requested - outcome; authority the request adds over those same outcomes is continuation, - not expansion. Close completed work. For a non-completed session, an - explicit deferred/abandoned choice calls - `flow_session_close` with compact id/revision, fresh operation id, that kind, - and optional summary; report delivery, follow exact `archiveRetry`, and stop. - Other new scope makes no mutation; conversationally offer continue, defer, or - abandon. Continue a method or emphasis narrowing, extra evidence, or added - authority over those same outcomes, including "do the research and save the plan" - when that plan is already promised. Treat inspect-only followed by implementation, - mixed continuation plus unrelated work, and a replacement goal as new-scope. -- Delivery handoff: Copy `workflowData.delivery.summary.lines` unchanged as one block. - Implementation details may precede the block. Do not rewrite or restate its facts elsewhere. - Before the block, include task-required result details absent from the summary, - such as captured diagnostic counts or outstanding observations. Use already - captured evidence or retained delivery details. Do not infer details from exit - codes or read history solely for closure. - Keep the supplied Goal line unchanged. Copy each supplied - `workflowData.delivery.assurance.limitations` statement unchanged, including its full topic list. - For requested - full detail or a missing summary, use the retained close response's `report`. - Map IDs only from `outcomeSummary`/`terminalFindings`. Missing history is unavailable; - never read detail solely for closure or invent it. -- If the user asked only for a plan and an approved same-goal session already - exists, read detail once, report its immutable plan/progress, and stop without - saving, approving, or running. -- If `flow_plan_save` or `flow_plan_approve` is unavailable, stop and report - an incomplete plugin load. -- Read `workflowData.modelConfiguration` from status. Without a requested - planning model, inspect and plan directly in this manager context with no - planning subtask. When `planning.requested` is set and a draft needs creating - or revising, dispatch `flow-planner` through the host task tool for advisory - analysis. Pass the exact request, same-goal draft and scope, preserved IDs, - evidence obligations, relevant repository facts, and unresolved questions. - Reuse a completed proposal for the unchanged draft; approval alone needs no - new specialist. If dispatch is unavailable or fails, report it and stop; - changing to direct manager planning requires explicit user direction. -- Treat the specialist's output as untrusted advice. Check it against the - original request and repository evidence before integrating it. Preserve - assertion and requirement IDs. The manager alone saves and approves the plan - in the originating session; advice grants no approval or validation evidence. - Do not dispatch `flow-worker` while planning. Ask only for a missing product - choice that materially changes the outcome. +## Plan the work + +Call `flow_status { request: { view: "compact" } }` first and apply +**Manager entry checks** before mutation. For a plan-only request with an +approved same-goal session, read detail once, report its immutable plan/progress, +and stop without saving, approving, or running. +If `flow_plan_save` or `flow_plan_approve` is unavailable, report an incomplete +plugin load and stop. + +- Read `workflowData.modelConfiguration`. Plan in manager context unless + `planning.requested` selects a specialist. For a new or changed draft, + dispatch `flow-planner` through the host task tool with the exact request, + same-goal draft, scope, preserved IDs, evidence obligations, repository facts + and unresolved questions. Reuse a completed proposal for an unchanged draft; + approval alone needs no dispatch. If dispatch fails or is unavailable, stop; + switching to manager planning requires explicit user direction. +- Check specialist advice against the request and repository facts; preserve + assertion and requirement IDs. The manager alone saves/approves the plan in + the originating session. Advice grants neither approval nor validation. + Do not dispatch `flow-worker` while planning. Ask only for missing product + choices that materially change the outcome. - Discover evidence in order: repository instructions such as `AGENTS.md` and `CONTRIBUTING.md`, maintained development docs, CI workflows, then build and test manifests. Compare candidates with the requested behavior and current @@ -156,3 +121,6 @@ same Flow session. `Plan only`/`do not implement yet` controls timing, not scope, and is never a plan requirement, decision, or non-goal. Do not implement or create a plan document unless requested. + +Shared contracts: [Manager entry checks](../flow/references/manager-entry.md) and +[Delivery handoff](../flow/references/delivery.md), embedded below. diff --git a/skills/flow-run/SKILL.md b/skills/flow-run/SKILL.md index 8e3418af..e76d9396 100644 --- a/skills/flow-run/SKILL.md +++ b/skills/flow-run/SKILL.md @@ -7,80 +7,14 @@ description: Implement, validate, independently review, and record one approved Work on exactly one approved feature. -## Start - -1. Call `flow_status { request: { view: "compact" } }` first. Treat - `nextAction` as the durable default, not as permission. -2. If the top-level response status is `error`, report its exact summary and - recovery when present and, if `workflowData.delivery` exists, the handoff - below. This read made no lifecycle, Git, or release mutation. Stop. Do not - route its `nextAction`. -3. If compact status contains `archiveRetry`, call `flow_session_close` once - with the projected request byte-for-byte. Report delivery under the contract - below. Refresh only if publication is unconfirmed. Stop after this cleanup - either way; it grants no work. - 4. When the projection contains an active goal, align it with the current - `/flow-run` request before another manager lifecycle mutation. Continue only - for the same goal or a method/emphasis narrowing that preserves all outcomes; - authority the request adds over those same outcomes is continuation, not - expansion. Close completed work. Unless the next step applies, new or expanded work - makes no mutation: report that it has not started and offer continue, defer, - or abandon. Continue a method or emphasis narrowing, extra evidence, or added - authority over those same outcomes, including "do the research and save the plan" - when that plan is already promised. Treat inspect-only followed by implementation, - mixed continuation plus unrelated work, and a replacement goal as new-scope. -5. If the aligned request explicitly chooses deferred or abandoned closure for - a non-completed session, call `flow_session_close` with compact session id and - revision, fresh operation id, that kind, and optional summary. Report delivery - under the contract below, follow a projected exact `archiveRetry`, and stop. -6. If status is `idle` or `planning`, report its projected planning action, - explain that `/flow-run` requires an approved feature, and stop without - mutation. - -Delivery handoff: Copy `workflowData.delivery.summary.lines` unchanged as one block. -Implementation details may precede the block. Do not rewrite or restate its facts elsewhere. -Before the block, include task-required result details absent from the summary, -such as captured diagnostic counts or outstanding observations. Use already -captured evidence or retained delivery details. Do not infer details from exit -codes or read history solely for closure. -Keep the supplied Goal line unchanged. Copy each supplied -`workflowData.delivery.assurance.limitations` statement unchanged, including its full topic list. -For requested -full detail or a missing summary, use the retained close response's `report`. -Map IDs only -from delivery `outcomeSummary`/`terminalFindings`. Requirements are `verified`, -`incomplete`, or explicitly `deferred`, and `abandoned` remains the kind. If -delivery is absent, report exact recovery and no map. On revision conflict, -refresh compact; retry only for the same session and goal while status still -permits the selected closure kind; never close a replacement. - -## Route +## Execute the feature -Follow compact `nextAction` in this order: - -- `flow_session_close`: close completed work with its projected session - id/revision, fresh operation id, and `kind: "completed"`. Report delivery, - follow one exact `archiveRetry` if needed, and stop. -- `await-user-direction` or blocked `flow_feature_reset`: call - `flow_status { request: { view: "detail" } }` exactly once, then apply - **Blocked review**. -- Running `flow_feature_reset`: the pending review is source-stale. Reset with - the same feature as `nextFeatureId` when continuing it. Never redispatch that - assignment. -- `dispatch-flow-reviewer`: read execution status. If that read errors, report - its exact summary and recovery when present and stop. Otherwise route that - refreshed projection. Dispatch under **Review** only if `nextAction` is still - `dispatch-flow-reviewer`. If it is now running `flow_feature_reset`, follow - the source-stale reset route. -- `flow_run_start`: start the ready feature, refresh compact status, and read - execution status. -- `flow_validation_start`: read execution status and resume from the current - worktree. -- `flow_review_start`: read execution status and continue at **Review**. -- Any other action: report it and stop. - -Use execution status for scope and revision. Stay in the feature; reset a wrong -design instead of layering retries. +Call `flow_status { request: { view: "compact" } }` first and apply +**Manager entry checks** before mutation. If status is `idle` or `planning`, +report its planning action and stop: `/flow-run` requires an approved feature. +For `nextAction: flow_run_start`, start the ready feature, read execution status, then implement, +validate and dispatch review below. For an existing run or a non-start action, +use **Recovery routing**. Use execution status for scope and revision. ## Implement @@ -176,6 +110,31 @@ apply the `dispatch-flow-reviewer` or running `flow_feature_reset` route. If status is blocked, apply **Blocked review**. A recorded pass completes the feature. +## Recovery routing + +Follow compact `nextAction` in this order: + +- `flow_session_close`: close completed work with its projected session + id/revision, fresh operation id, and `kind: "completed"`. Report delivery, + follow one exact `archiveRetry` if needed, and stop. +- `await-user-direction` or blocked `flow_feature_reset`: call + `flow_status { request: { view: "detail" } }` exactly once, then apply + **Blocked review**. +- Running `flow_feature_reset`: the pending review is source-stale. Reset with + the same feature as `nextFeatureId` when continuing it. Never redispatch that + assignment. +- `dispatch-flow-reviewer`: refresh execution status and route its `nextAction`. + Dispatch under **Review** if it still names dispatch; a source-stale assignment + takes the reset route instead. +- `flow_run_start`: start the ready feature, refresh compact status, and read + execution status. +- `flow_validation_start`: read execution status and resume from the current + worktree. +- `flow_review_start`: read execution status and continue at **Review**. +- Any other action: report it and stop. + +Stay in the feature; reset a wrong design instead of layering retries. + ### Blocked review Follow `nextAction` with the one detail projection for routing. At @@ -204,3 +163,6 @@ and `blockedFeature.scopeBlocker`. Direct `/flow-run` reports compact `findingsDigest` and `nextAction`, then stops. Under `/flow-auto`, return to its lifecycle loop. + +Shared contracts: [Manager entry checks](../flow/references/manager-entry.md) and +[Delivery handoff](../flow/references/delivery.md), embedded below. diff --git a/skills/flow/SKILL.md b/skills/flow/SKILL.md index fbf4dfa8..fe4fb8e3 100644 --- a/skills/flow/SKILL.md +++ b/skills/flow/SKILL.md @@ -10,31 +10,8 @@ never silently fall back. ## Route from status -1. Call `flow_status { request: { view: "compact" } }` first. Trust its - `nextAction` as the durable default, not added authority. - If the top-level response is an error, report its exact summary and recovery; - when delivery is present, handle its bounded map under **Recovery**. Stop - without another mutation. - For `archiveRetry`, call the projected `flow_session_close` request - byte-for-byte and handle its delivery under **Recovery**. Refresh compact status; - stop if publication remains unconfirmed, otherwise continue from the - refreshed projection. This cleanup precedes goal alignment and grants no work. - Before another manager mutation, align the compact-projected goal with the - request. Continue only for the same goal or a narrowing that preserves every - outcome; authority the request adds over those same outcomes is continuation, - not expansion. Close a completed session as completed before new work. If the user - explicitly chooses deferred or abandoned closure for a non-completed session, - call `flow_session_close` with compact session id/revision, fresh operation - id, that kind, and optional summary; handle **Recovery**, follow a projected - exact `archiveRetry`, and stop. Otherwise new or expanded work makes no - mutation: say it has not started and offer continue, defer, or abandon. Keep - alignment conversational. Continue a method or emphasis narrowing, extra - evidence, or added authority over those same outcomes, including - "do the research and save the plan" when that plan is already promised. Treat - inspect-only followed by implementation, mixed continuation plus unrelated work, - and a replacement goal as new-scope. On revision conflict, refresh compact status and - retry only after confirming the same session and goal and that status still - permits the selected closure kind; never close a replacement. +1. Call `flow_status { request: { view: "compact" } }` first and apply + **Manager entry checks** before mutation. 2. For a plan-only request with an approved same-goal session, read detail once, report the immutable plan and progress, and stop. 3. With no session or a draft, load `flow_guidance { id: "flow-plan" }`; stop @@ -55,7 +32,7 @@ The runtime decides when `/flow-auto` continues automatically; never assume a further turn, and finish the authorized work in this one. For `ready`, apply `flow-run`; after every recorded result reload compact. For `completed`, -close and handle **Recovery** plus exact `archiveRetry`. +close and handle **Delivery handoff** plus exact `archiveRetry`. `flow_run_start` and intermediate progress are not terminal: never return “ready for the next feature” or wait while an action is authorized. Return only after closure or a checkpoint. Direct `/flow-run` stops after one feature. @@ -80,40 +57,5 @@ If a Flow tool is absent, report an incomplete plugin load; never simulate state - Do not create reports/sidecars, Git changes, or release mutations unless requested; reports default to Markdown and JSON is opt-in. -## Recovery - -On interruption, read compact status; load `flow-run` for an active or blocked feature -and use execution or reviewer status, never prose, for lifecycle truth. At a -checkpoint or lease stop, print compact `findingsDigest` before any delivery map. - -Summaries keep plan/source IDs `verified` or `incomplete`. A prior finding is -terminally `fixed` only when review passes with current evidence. A failed -verdict carries every prior ID forward: report a proven repair as `terminal -fixed pending pass` with concise evidence, an -unproven repair as unverified-fixed, a recurrence as `recurring`, or a confirmed -nonblocker as `residual`. Blockers stay terminal. - -For an accepted close, map only `workflowData.delivery` -`outcomeSummary`/`terminalFindings`. -Requirements are proven `verified`, otherwise `incomplete` or explicit -`deferred`. Apply the finding rules above, using `incomplete` for an unproven -terminal claim; `abandoned` stays the kind. Missing IDs are unavailable: never -invent or read detail solely for closure. Without delivery, report exact -recovery and no map. - -Unresolved blockers forbid completed closure. Fresh close: projected session -id/revision, fresh operation id, kind, optional summary. Replay byte-for-byte -only the `archiveRetry` of a durably accepted close. Rejected revision conflict: -refresh compact, confirm the same session/goal, then build a fresh request. -Copy `workflowData.delivery.summary.lines` unchanged as one block. -Implementation details may precede the block. Do not rewrite or restate its facts elsewhere. -Before the block, include task-required result details absent from the summary, -such as captured diagnostic counts or outstanding observations. Use already -captured evidence or retained delivery details. Do not infer details from exit -codes or read history solely for closure. -Keep the supplied Goal line unchanged. Copy each supplied -`workflowData.delivery.assurance.limitations` statement unchanged, including its full topic list. -For requested full detail or a missing -summary, report the retained close response's `report`. Report external prerequisites only -from terminal text; otherwise mark them unavailable. Create no other ledger or -report. +Shared contracts: [Manager entry checks](../flow/references/manager-entry.md) and +[Delivery handoff](../flow/references/delivery.md), embedded below. diff --git a/skills/flow/references/delivery.md b/skills/flow/references/delivery.md new file mode 100644 index 00000000..2d3bacad --- /dev/null +++ b/skills/flow/references/delivery.md @@ -0,0 +1,18 @@ +## Delivery handoff + +Copy `workflowData.delivery.summary.lines` unchanged as one block. +Implementation details may precede the block. Do not rewrite or restate its facts elsewhere. +Keep the supplied Goal line unchanged. +Copy each supplied `workflowData.delivery.assurance.limitations` statement unchanged, including its full topic list. +Before it, give task-required details absent from the summary using captured +evidence or retained delivery. Do not infer details from exit codes, read history +solely for closure. For requested full +detail or a missing summary, use the retained close response's `report`. + +Map IDs only from `outcomeSummary`/`terminalFindings`: requirements are `verified`, +`incomplete`, or explicit `deferred`; `abandoned` stays the kind. A finding is +`fixed` only after passing review with current evidence; a failed verdict leaves +proven repairs pending pass and unproven repairs unverified. `recurring` needs +current confirmation, `residual` a confirmed nonblocker. Missing IDs and external +prerequisites are unavailable. Without delivery, report exact recovery and no map. +At a checkpoint or lease stop, print compact `findingsDigest` before the map. diff --git a/skills/flow/references/manager-entry.md b/skills/flow/references/manager-entry.md new file mode 100644 index 00000000..c2b419d5 --- /dev/null +++ b/skills/flow/references/manager-entry.md @@ -0,0 +1,22 @@ +## Manager entry checks + +A status-read error ends the route: report its exact summary/recovery and any +**Delivery handoff**, then stop without another mutation. + +Before mutation, continue only for the same projected goal or a narrowing that +preserves every outcome; authority the request adds over those same outcomes +is continuation, not expansion. Continue a method or emphasis narrowing, extra evidence, or added authority over those same outcomes, including +"do the research and save the plan" when that plan is already promised. +Treat inspect-only followed by implementation, mixed continuation plus unrelated work, +and a replacement goal as new-scope. New scope makes no mutation: offer continue, +defer, or abandon. Close completed work before a new goal. + +For explicit deferred/abandoned closure, call `flow_session_close` with compact +session id/revision, fresh operation id, kind and optional summary. Replay any +projected `archiveRetry` byte-for-byte once, then hand off and stop. +On revision conflict, refresh compact and retry only for the same session, +goal and permitted kind; never close a replacement. + +Before alignment, replay an `archiveRetry` request byte-for-byte once and hand +off. Direct `/flow-run` stops. `/flow-auto` and `/flow-plan` refresh compact and +continue only after archive publication is confirmed. Cleanup grants no work. diff --git a/src/guidance/catalog.ts b/src/guidance/catalog.ts index 8d335ab0..57e223e6 100644 --- a/src/guidance/catalog.ts +++ b/src/guidance/catalog.ts @@ -1,3 +1,9 @@ +import delivery from "../../skills/flow/references/delivery.md" with { + type: "text", +}; +import managerEntry from "../../skills/flow/references/manager-entry.md" with { + type: "text", +}; import flowSkill from "../../skills/flow/SKILL.md" with { type: "text" }; import flowPlanSkill from "../../skills/flow-plan/SKILL.md" with { type: "text", @@ -34,10 +40,22 @@ export const FLOW_MANAGER_KERNEL = [ ].join(" "), ].join("\n"); +function managerGuide(skill: string, kernel = false): string { + return [ + skill, + managerEntry, + delivery, + ...(kernel ? [FLOW_MANAGER_KERNEL] : []), + ] + .map((part) => part.trimEnd()) + .join("\n\n") + .concat("\n"); +} + const GUIDANCE_CONTENT: Record = { - flow: `${flowSkill.trimEnd()}\n\n${FLOW_MANAGER_KERNEL}\n`, - "flow-plan": flowPlanSkill, - "flow-run": `${flowRunSkill.trimEnd()}\n\n${FLOW_MANAGER_KERNEL}\n`, + flow: managerGuide(flowSkill, true), + "flow-plan": managerGuide(flowPlanSkill), + "flow-run": managerGuide(flowRunSkill, true), "flow-review": flowReviewSkill, }; diff --git a/src/prompt-surfaces.ts b/src/prompt-surfaces.ts index ff38b3d6..3bbd7278 100644 --- a/src/prompt-surfaces.ts +++ b/src/prompt-surfaces.ts @@ -58,29 +58,18 @@ const FLOW_WORKER_PROMPT = [ ].join("\n"); const FLOW_STATUS_PROMPT = [ - 'Call `flow_status { request: { view: "compact" } }` first.', - "Do not mutate.", - "Report `workflowData.modelConfiguration.report` as process-local model settings and observations when present.", - "Report `workflowData.reviewerConfiguration.report` verbatim when present and label it process-local, not persisted state.", - "Report `workflowData.statusReport` verbatim when present; do not reconstruct lifecycle or recovery facts from the projection.", - "If the top-level response status is `error`, report its exact summary and", - "`workflowData.failure.recovery` when present; otherwise say no recovery guidance was supplied.", - "If current `workflowData.delivery` exists, report its `summary.lines`, or its `report` on request or no summary. Otherwise say unavailable.", - "Map IDs only from `outcomeSummary` and `terminalFindings`: IDs are `verified`", - "only when proven, otherwise `incomplete` or explicitly `deferred`; `fixed` needs later passing", - "review plus current evidence, `recurring` current confirmation, `residual` a confirmed nonblocker,", - "`abandoned` remains the closure kind.", - "Missing IDs unavailable.", - "State that `/flow-status` made no Git or release mutation, note any lifecycle effect, and stop.", - "Do not interpret recovery guidance as a blocked review.", - "If `projection.status` is `blocked` or `projection.nextAction` is `await-user-direction`,", - 'call `flow_status { request: { view: "detail" } }` exactly once and label the result overall incomplete.', - "Report the detail `workflowData.statusReport` verbatim when present; it owns retry, evidence, artifact, and recovery text.", - "When `workflowData.autoTiming` is present, report `activeMs` as non-authoritative process-local wall time, not CPU or pure work, and `waitingForUserMs` as only projected `flow_plan_approve` plus `await-user-direction` time for the latest `/flow-auto`.", - "State that paused, inactive, errored, and unprojected waits are excluded.", - "Otherwise report the compact projection and its exact `nextAction`, state that `/flow-status`", - "made no lifecycle, Git, or release mutation, and stop.", -].join(" "); + "# Flow status", + "Read and report current state without advancing work.", + '1. Call `flow_status { request: { view: "compact" } }` first.', + "2. On a top-level error, report its exact summary and `workflowData.failure.recovery` when present; otherwise say no recovery guidance was supplied. Stop without another read or mutation.", + '3. If `projection.status` is `blocked` or `projection.nextAction` is `await-user-direction`, call `flow_status { request: { view: "detail" } }` exactly once. On error, apply step 2. Otherwise use that detail response and label the result overall incomplete.', + "4. Report the selected response once:", + "- Report `workflowData.statusReport` verbatim when present; otherwise report its projection and exact `nextAction`. Do not reconstruct recovery facts.", + "- Report `workflowData.modelConfiguration.report` and `workflowData.reviewerConfiguration.report` verbatim when present, labelled process-local settings and observations, not persisted state.", + "- If current `workflowData.delivery` exists, report its `summary.lines`, or its `report` on request or no summary. Otherwise say unavailable. Map IDs only from `outcomeSummary` and `terminalFindings`: `verified` needs proof; `fixed` needs passing review plus current evidence; otherwise use `incomplete`, explicit `deferred`, confirmed `recurring` or `residual`, or the closure kind `abandoned`. Missing IDs are unavailable.", + "- When `workflowData.autoTiming` is present, report `activeMs` as non-authoritative process-local wall time, not CPU or pure work. `waitingForUserMs` covers only projected `flow_plan_approve` plus `await-user-direction` time for the latest `/flow-auto`; paused, inactive, errored and unprojected waits are excluded.", + "5. State that `/flow-status` made no Git or release mutation, note any lifecycle effect, and stop. Do not interpret recovery guidance as a blocked review or permission to mutate.", +].join("\n\n"); const FLOW_REVIEW_PROMPT = [ "# Flow review command", diff --git a/tests/guidance-scenarios.test.ts b/tests/guidance-scenarios.test.ts new file mode 100644 index 00000000..56c28a9f --- /dev/null +++ b/tests/guidance-scenarios.test.ts @@ -0,0 +1,178 @@ +import { expect, test } from "bun:test"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { ScenarioGradeInput } from "../evals/grader-input.js"; +import { + GUIDANCE_SCENARIOS, + GUIDANCE_STATUS_CASES, +} from "../evals/guidance-scenarios.js"; +import type { ObservedToolCall } from "../evals/harness.js"; +import { caseCatalogFor, releaseScenarios } from "../evals/run.js"; +import FlowPlugin from "../src/index.js"; +import { createNativeFlowHost } from "./fixtures/native-flow-host.js"; + +for (const [index, entry] of GUIDANCE_STATUS_CASES.entries()) { + test(`${entry.id}: real plugin responses satisfy the grader; wrong routes and mutations fail`, async () => { + const scenario = GUIDANCE_SCENARIOS[index]; + if (!scenario) throw new Error("Missing guidance scenario."); + const workspace = await mkdtemp(join(tmpdir(), "flow-guidance-eval-")); + const host = await createNativeFlowHost(workspace); + try { + await host.load(FlowPlugin); + await mkdir(join(workspace, ".flow"), { recursive: true }); + const bytes = scenario.files[".flow/session.json"]; + if (!bytes) throw new Error("Missing seeded session."); + await writeFile(join(workspace, ".flow/session.json"), bytes); + const calls: ObservedToolCall[] = []; + let finalText = ""; + for (const view of entry.views) { + const output = await host.response("flow_status", { + request: { view }, + }); + calls.push({ + tool: "flow_status", + status: "completed", + sessionIndex: 0, + agent: "build", + input: { request: { view } }, + output, + rawOutput: JSON.stringify(output), + metadata: {}, + }); + if (output.status === "error") { + const failure = output.workflowData.failure as { recovery: string }; + finalText = `${output.summary}\n${failure.recovery}`; + } else { + const report = output.workflowData.statusReport; + if (!Array.isArray(report)) throw new Error("Missing status report."); + finalText = `${report.join("\n")}\nOverall incomplete.`; + } + } + const input: ScenarioGradeInput = { + flowCalls: calls, + allCalls: calls, + session: entry.session, + archives: [], + workspaceChanges: { kind: "observed", paths: [] }, + finalText, + }; + expect(scenario.check(input)).toEqual([]); + expect( + await readFile(join(workspace, ".flow/session.json"), "utf8"), + ).toBe(bytes); + // An omitted report or an empty trace cannot become a pass. + expect( + scenario.check({ ...input, finalText: "Flow looks good." }), + ).not.toEqual([]); + expect( + scenario.check({ ...input, flowCalls: [], allCalls: [] }), + ).not.toEqual([]); + // Duplicate reads also catch continuing after an error. + expect( + scenario.check({ ...input, allCalls: [...calls, ...calls] }), + ).not.toEqual([]); + if (entry.id === "guidance-status-error") { + const first = calls[0]; + if (!first) throw new Error("Missing status error."); + const trailingRead = { + ...first, + tool: "read", + input: { filePath: "src/greet.mjs" }, + }; + expect( + scenario.check({ ...input, allCalls: [...calls, trailingRead] }), + ).not.toEqual([]); + } + // Retained text alone does not excuse a response for the wrong source state. + if (entry.id !== "guidance-status-error") { + const stale = structuredClone(calls); + const last = stale.at(-1); + if (!last) throw new Error("Missing status call."); + const output = last.output as { + workflowData: { projection: { revision: number } }; + }; + output.workflowData.projection.revision = 99; + expect( + scenario.check({ ...input, allCalls: stale, flowCalls: stale }), + ).not.toEqual([]); + const error = calls.map((call, callIndex) => + callIndex === calls.length - 1 + ? { + ...call, + output: { status: "error", summary: "Unexpected failure" }, + } + : call, + ); + expect( + scenario.check({ + ...input, + allCalls: error, + finalText: "Unexpected failure", + }), + ).not.toEqual([]); + } + if (entry.views.length > 1) + expect( + scenario.check({ ...input, allCalls: calls.slice(0, 1) }), + ).not.toEqual([]); + for (const tool of [ + "flow_run_start", + "flow_feature_reset", + "flow_session_close", + "shell", + "write", + "task", + ]) { + const first = calls[0]; + if (!first) throw new Error("Missing status call."); + const mutation = { ...first, tool }; + expect( + scenario.check({ + ...input, + allCalls: [...calls, mutation], + flowCalls: [...calls, mutation], + }), + ).not.toEqual([]); + } + expect(scenario.check({ ...input, session: null })).not.toEqual([]); + expect( + scenario.check({ + ...input, + session: { ...entry.session, revision: 99 }, + }), + ).not.toEqual([]); + expect( + scenario.check({ + ...input, + workspaceChanges: { kind: "observed", paths: ["src/greet.mjs"] }, + }), + ).not.toEqual([]); + expect( + scenario.check({ + ...input, + workspaceChanges: { kind: "unavailable", reason: "not retained" }, + }), + ).not.toEqual([]); + } finally { + await host.dispose(); + await rm(workspace, { recursive: true, force: true }); + } + }); +} + +test("guidance cases reuse ordinary catalog/reporting without altering the release matrix", () => { + const catalog = caseCatalogFor(GUIDANCE_SCENARIOS, { + kind: "ordinary", + repeat: 1, + }); + expect(catalog.map((row) => row.caseId)).toEqual( + GUIDANCE_STATUS_CASES.map((row) => row.id), + ); + expect(catalog.every((row) => row.release === "report-only")).toBe(true); + const release = releaseScenarios("10.0.1"); + expect(release).toHaveLength(17); + expect(release.some((row) => row.id.startsWith("guidance-status-"))).toBe( + false, + ); +}); diff --git a/tests/prompt-quality.test.ts b/tests/prompt-quality.test.ts index cb2670a3..6391f950 100644 --- a/tests/prompt-quality.test.ts +++ b/tests/prompt-quality.test.ts @@ -1,4 +1,5 @@ import { describe, expect, test } from "bun:test"; +import { readFile } from "node:fs/promises"; import { FeatureCompleteInputSchema } from "../src/application/schema.js"; import { FLOW_CORE_COMMANDS } from "../src/config-shared.js"; import { @@ -40,7 +41,7 @@ const MANAGER_GUIDANCE = [ ["flow-run", "flow-run"], ] as const; -const MAX_MANAGER_PROMPT_BYTES = 34_000; +const MAX_MANAGER_PROMPT_BYTES = 28_000; const MANAGER_PROMPT_RESERVE_BYTES = 4 * 1024; /** @@ -71,7 +72,7 @@ const MANAGER_PROMPT_RESERVE_BYTES = 4 * 1024; * evidence rules arrived as a checked field rather than another paragraph asking the * model to be careful. */ -const MAX_TOTAL_PROMPT_BYTES = 14_500; +const MAX_TOTAL_PROMPT_BYTES = 14_400; /** * Absolute-rule markers per surface. Both Anthropic and OpenAI advise reserving @@ -231,6 +232,28 @@ describe("Flow prompt structure", () => { ).toThrow("Unsupported Flow prompt surface"); }); + test("embeds each shared manager contract once in self-contained guides", async () => { + const contracts = await Promise.all( + ["manager-entry", "delivery"].map((name) => + readFile( + new URL(`../skills/flow/references/${name}.md`, import.meta.url), + "utf8", + ), + ), + ); + for (const id of ["flow", "flow-plan", "flow-run"] as const) { + const content = getFlowGuidance(id).content; + for (const contract of contracts) expectOnce(content, contract.trimEnd()); + } + const run = getFlowGuidance("flow-run").content; + expect(run.indexOf("## Implement")).toBeLessThan( + run.indexOf("## Recovery routing"), + ); + expect(run.indexOf("## Review")).toBeLessThan( + run.indexOf("## Manager entry checks"), + ); + }); + test("keeps worker dispatch inside an active feature run", () => { // 7.3.0 thin routers load only flow-plan during planning. If the kernel still // says "delegate active Flow work to flow-worker" and flow-plan is silent, From 04a43fc6cda89df4829df8cd6907764e6a1dfc73 Mon Sep 17 00:00:00 2001 From: Douwe de Vries Date: Wed, 28 Oct 2026 20:43:03 +0100 Subject: [PATCH 2/2] fix(flow): retain delivery when status reports an error --- .../21-guidance-maintainability/README.md | 9 +- evals/guidance-scenarios.ts | 28 ++++- src/prompt-surfaces.ts | 4 +- tests/guidance-scenarios.test.ts | 100 ++++++++++++++++++ 4 files changed, 133 insertions(+), 8 deletions(-) diff --git a/.agents/plans/21-guidance-maintainability/README.md b/.agents/plans/21-guidance-maintainability/README.md index 59d12970..984720d7 100644 --- a/.agents/plans/21-guidance-maintainability/README.md +++ b/.agents/plans/21-guidance-maintainability/README.md @@ -55,19 +55,20 @@ rewrite. Full release qualification still applies to any future candidate. ## Evidence -- `bun run eval:guidance`: 83 focused tests pass; all 13 gated cassettes reproduce. +- `bun run eval:guidance`: 84 focused tests pass; all 13 gated cassettes reproduce. The new graders use actual native-plugin fixture responses and adversarial variants, including a tool read after a status error. - `bun run check`: 5,766 pass, 27 skip, zero failures before final review fixes; the clean committed push preflight repeats the complete gate after those fixes. - `bun run smoke:live`: two packed OpenCode tests, 128 assertions, pass using a local scripted provider. No real provider dispatch or account use. -- Independent GPT-6.1 Sol review: two rounds, final APPROVE. Fixed both findings: +- Independent GPT-6.1 Sol review: three rounds, final APPROVE. Fixed all findings: accepted-close archive retry remains exact and immediate; status error grading - rejects every following tool call, not just further status reads. + rejects every following tool call, not just further status reads. Error handoffs + retain supplied delivery and the mutation statement, including archive collisions. - Authored skill text including shared fragments falls from 36,220 to 29,853 bytes. The ordinary manager stack is effectively flat: 23,599 to 23,592 bytes; - compiled command/agent surfaces fall from 14,496 to 14,361. Existing prompt + compiled command/agent surfaces fall from 14,496 to 14,335. Existing prompt ceilings ratchet down; documentation and source ceilings are unchanged. - Contributor preflight now owns one mandatory offline guidance command. Live model measurement is explicitly separate, rather than silently attempted by diff --git a/evals/guidance-scenarios.ts b/evals/guidance-scenarios.ts index b3495d60..d7f0f2f7 100644 --- a/evals/guidance-scenarios.ts +++ b/evals/guidance-scenarios.ts @@ -20,7 +20,13 @@ export const GUIDANCE_STATUS_CASES = [ }, ] as const; -type StatusCase = (typeof GUIDANCE_STATUS_CASES)[number]; +type StatusCase = Omit<(typeof GUIDANCE_STATUS_CASES)[number], "session"> & { + readonly session: Record; + readonly archives?: readonly Record[]; +}; + +export const STATUS_MUTATION_STATEMENT = + "/flow-status made no Git or release mutation."; function object(value: unknown): Record { return typeof value === "object" && value !== null && !Array.isArray(value) ? (value as Record) @@ -49,7 +55,7 @@ export function guidanceStatusIssues( issues.push("status attempted work outside its read-only route"); if ( canonicalJson(input.session) !== canonicalJson(expected.session) || - input.archives.length !== 0 + canonicalJson(input.archives) !== canonicalJson(expected.archives ?? []) ) issues.push("status changed or removed the seeded session"); if ( @@ -58,9 +64,27 @@ export function guidanceStatusIssues( ) issues.push("read-only workspace evidence is missing or reports changes"); + if (!input.finalText.includes(STATUS_MUTATION_STATEMENT)) + issues.push("status omitted the Git/release mutation statement"); + const selected = calls.at(-1); const response = object(selected?.output); const data = object(response.workflowData); + if (data.delivery !== undefined) { + const delivery = object(data.delivery); + const summary = object(delivery.summary).lines; + const lines = + Array.isArray(summary) && summary.length > 0 ? summary : delivery.report; + if ( + !Array.isArray(lines) || + lines.length === 0 || + !lines.every( + (line) => typeof line === "string" && input.finalText.includes(line), + ) + ) + issues.push("status omitted supplied delivery details"); + } + const firstError = input.allCalls.findIndex( (call) => call.tool === "flow_status" && object(call.output).status === "error", diff --git a/src/prompt-surfaces.ts b/src/prompt-surfaces.ts index 3bbd7278..08383033 100644 --- a/src/prompt-surfaces.ts +++ b/src/prompt-surfaces.ts @@ -61,14 +61,14 @@ const FLOW_STATUS_PROMPT = [ "# Flow status", "Read and report current state without advancing work.", '1. Call `flow_status { request: { view: "compact" } }` first.', - "2. On a top-level error, report its exact summary and `workflowData.failure.recovery` when present; otherwise say no recovery guidance was supplied. Stop without another read or mutation.", + "2. On error, select that response. Report its exact summary and `workflowData.failure.recovery`, or say none was supplied. Apply only step 4's delivery bullet, then step 5. Use no further tools.", '3. If `projection.status` is `blocked` or `projection.nextAction` is `await-user-direction`, call `flow_status { request: { view: "detail" } }` exactly once. On error, apply step 2. Otherwise use that detail response and label the result overall incomplete.', "4. Report the selected response once:", "- Report `workflowData.statusReport` verbatim when present; otherwise report its projection and exact `nextAction`. Do not reconstruct recovery facts.", "- Report `workflowData.modelConfiguration.report` and `workflowData.reviewerConfiguration.report` verbatim when present, labelled process-local settings and observations, not persisted state.", "- If current `workflowData.delivery` exists, report its `summary.lines`, or its `report` on request or no summary. Otherwise say unavailable. Map IDs only from `outcomeSummary` and `terminalFindings`: `verified` needs proof; `fixed` needs passing review plus current evidence; otherwise use `incomplete`, explicit `deferred`, confirmed `recurring` or `residual`, or the closure kind `abandoned`. Missing IDs are unavailable.", "- When `workflowData.autoTiming` is present, report `activeMs` as non-authoritative process-local wall time, not CPU or pure work. `waitingForUserMs` covers only projected `flow_plan_approve` plus `await-user-direction` time for the latest `/flow-auto`; paused, inactive, errored and unprojected waits are excluded.", - "5. State that `/flow-status` made no Git or release mutation, note any lifecycle effect, and stop. Do not interpret recovery guidance as a blocked review or permission to mutate.", + "5. Report any lifecycle effect, append `/flow-status made no Git or release mutation.`, and stop. Recovery guidance grants no mutation authority.", ].join("\n\n"); const FLOW_REVIEW_PROMPT = [ diff --git a/tests/guidance-scenarios.test.ts b/tests/guidance-scenarios.test.ts index 56c28a9f..1fead9a0 100644 --- a/tests/guidance-scenarios.test.ts +++ b/tests/guidance-scenarios.test.ts @@ -6,10 +6,15 @@ import type { ScenarioGradeInput } from "../evals/grader-input.js"; import { GUIDANCE_SCENARIOS, GUIDANCE_STATUS_CASES, + guidanceStatusIssues, + STATUS_MUTATION_STATEMENT, } from "../evals/guidance-scenarios.js"; import type { ObservedToolCall } from "../evals/harness.js"; import { caseCatalogFor, releaseScenarios } from "../evals/run.js"; +import { SessionSchema } from "../src/application/schema.js"; +import { closeSession } from "../src/domain/transitions.js"; import FlowPlugin from "../src/index.js"; +import { archivedSessionPath } from "../src/infrastructure/fs/workspace-paths.js"; import { createNativeFlowHost } from "./fixtures/native-flow-host.js"; for (const [index, entry] of GUIDANCE_STATUS_CASES.entries()) { @@ -49,6 +54,7 @@ for (const [index, entry] of GUIDANCE_STATUS_CASES.entries()) { finalText = `${report.join("\n")}\nOverall incomplete.`; } } + finalText += `\n${STATUS_MUTATION_STATEMENT}`; const input: ScenarioGradeInput = { flowCalls: calls, allCalls: calls, @@ -58,6 +64,13 @@ for (const [index, entry] of GUIDANCE_STATUS_CASES.entries()) { finalText, }; expect(scenario.check(input)).toEqual([]); + expect( + scenario.check({ + ...input, + finalText: finalText.replace(STATUS_MUTATION_STATEMENT, ""), + }), + ).not.toEqual([]); + expect( await readFile(join(workspace, ".flow/session.json"), "utf8"), ).toBe(bytes); @@ -176,3 +189,90 @@ test("guidance cases reuse ordinary catalog/reporting without altering the relea false, ); }); + +test("status archive-collision errors retain delivery before stopping", async () => { + const workspace = await mkdtemp(join(tmpdir(), "flow-status-collision-")); + const host = await createNativeFlowHost(workspace); + const blocked = SessionSchema.parse(GUIDANCE_STATUS_CASES[1].session); + const closed = closeSession(blocked, { + operationId: "close-for-status", + expectedRevision: blocked.revision, + sessionId: blocked.id, + kind: "deferred", + summary: "Preserve incomplete work.", + }).session; + const conflict = { ...closed, goal: "Different archived goal" }; + try { + await mkdir(join(workspace, ".flow/history"), { recursive: true }); + await writeFile( + join(workspace, ".flow/session.json"), + JSON.stringify(closed), + ); + await writeFile( + archivedSessionPath(workspace, closed.id), + JSON.stringify(conflict), + ); + await host.load(FlowPlugin); + const output = await host.response("flow_status", { + request: { view: "compact" }, + }); + expect(output.status).toBe("error"); + const data = output.workflowData as { + failure: { recovery: string }; + delivery: { summary: { lines: string[] }; report: string[] }; + }; + const lines = data.delivery.summary.lines; + expect(lines.length).toBeGreaterThan(0); + const call: ObservedToolCall = { + tool: "flow_status", + status: "completed", + sessionIndex: 0, + agent: "build", + input: { request: { view: "compact" } }, + output, + rawOutput: JSON.stringify(output), + metadata: {}, + }; + const errorText = `${output.summary}\n${data.failure.recovery}`; + const input: ScenarioGradeInput = { + flowCalls: [call], + allCalls: [call], + session: closed, + archives: [conflict], + workspaceChanges: { kind: "observed", paths: [] }, + finalText: `${errorText}\n${lines.join("\n")}\n${STATUS_MUTATION_STATEMENT}`, + }; + const expected = { + ...GUIDANCE_STATUS_CASES[2], + session: closed, + archives: [conflict], + }; + expect(guidanceStatusIssues(input, expected)).toEqual([]); + expect( + guidanceStatusIssues( + { ...input, finalText: `${errorText}\n${STATUS_MUTATION_STATEMENT}` }, + expected, + ), + ).toContain("status omitted supplied delivery details"); + expect( + guidanceStatusIssues( + { + ...input, + finalText: input.finalText.replace(STATUS_MUTATION_STATEMENT, ""), + }, + expected, + ), + ).toContain("status omitted the Git/release mutation statement"); + expect( + JSON.parse(await readFile(join(workspace, ".flow/session.json"), "utf8")), + ).toEqual(closed); + expect( + JSON.parse( + await readFile(archivedSessionPath(workspace, closed.id), "utf8"), + ), + ).toEqual(conflict); + } finally { + await host.dispose(); + await rm(workspace, { recursive: true, force: true }); + } +});