From be9885e146d36ca346e5704b3e345a2da5b8756d Mon Sep 17 00:00:00 2001 From: gggodlin <26342074+GGGODLIN@users.noreply.github.com> Date: Tue, 13 Oct 2026 16:15:35 +0800 Subject: [PATCH 1/2] fix(canary): an Oh My Pi long draft is read as its opaque paste chip OMP collapses a pasted draft of more than ten lines into a numbered chip, so the 14-long-multi draft case waited for words that never reach the screen and ended not-reached. The phone's contract for such a token is to show it and stand Take over down, so the adapter recognising it is the read. Agents that define draftCarriesSend keep the stricter match. Real OMP 18.10.1 run 20261013T081454Z: drafts 15/15. Co-Authored-By: Claude Opus 5.5 --- scripts/harness-canary/drive.test.ts | 23 ++++++++++++++++++++++- scripts/harness-canary/readers.ts | 1 + scripts/harness-canary/scenarios.ts | 20 +++++++++++++++++--- 3 files changed, 40 insertions(+), 4 deletions(-) diff --git a/scripts/harness-canary/drive.test.ts b/scripts/harness-canary/drive.test.ts index 6eee39746..da5260d1a 100644 --- a/scripts/harness-canary/drive.test.ts +++ b/scripts/harness-canary/drive.test.ts @@ -12,7 +12,8 @@ import { colorAnswers, unfinishedTail } from "./client"; import { cleanEnv } from "./herdr"; import { JOURNAL_MESSAGE, MESSAGES, README_TOKEN, NARROW_DRAFT_IDS, SEND_IDS, messageById } from "./messages"; import { lastPointedRow } from "./dialogs"; -import { answeredBelow } from "./scenarios"; +import { answeredBelow, opaqueTokenRead } from "./scenarios"; +import type { Adapter } from "./readers"; import { DEFAULT_SCENARIOS } from "./verdict"; const ESC = String.fromCodePoint(0x1b); @@ -175,6 +176,26 @@ describe("drafts and messages", () => { }); }); +describe("opaqueTokenRead", () => { + const chip = (draft: string) => /^๐Ÿ“„ #\d+$/.test(draft); + const omp: Adapter = { agent: "omp", extractInputDraft: () => null, draftIsOpaque: chip }; + + test("a token with no evidence of its content counts as the draft read back", () => { + expect(opaqueTokenRead(omp, "๐Ÿ“„ #1")).toBe(true); + }); + + test("real text, no draft or no adapter is not a token read", () => { + expect(opaqueTokenRead(omp, "Reply with only OK.")).toBe(false); + expect(opaqueTokenRead(omp, null)).toBe(false); + expect(opaqueTokenRead(undefined, "๐Ÿ“„ #1")).toBe(false); + }); + + test("a harness whose token can be matched to the send keeps that stricter check", () => { + const matched: Adapter = { ...omp, agent: "claude", draftCarriesSend: () => false }; + expect(opaqueTokenRead(matched, "๐Ÿ“„ #1")).toBe(false); + }); +}); + describe("answeredBelow", () => { const plain = messageById("01-plain"); const read = JOURNAL_MESSAGE; diff --git a/scripts/harness-canary/readers.ts b/scripts/harness-canary/readers.ts index 7da49293a..f346ac549 100644 --- a/scripts/harness-canary/readers.ts +++ b/scripts/harness-canary/readers.ts @@ -25,6 +25,7 @@ export interface Adapter { extractInputDraft(lines: Line[]): string | null; composerReady?(lines: Line[]): boolean; draftCarriesSend?(sent: string, draft: string): boolean; + draftIsOpaque?(draft: string): boolean; } export type ReplyOutcome = diff --git a/scripts/harness-canary/scenarios.ts b/scripts/harness-canary/scenarios.ts index 4842699e1..0b0d8b5bf 100644 --- a/scripts/harness-canary/scenarios.ts +++ b/scripts/harness-canary/scenarios.ts @@ -215,6 +215,18 @@ export function wordsOnScreen(texts: readonly string[], text: string): boolean { return probes(text).some((p) => flat.includes(p)); } +/** + * Whether the adapter read back its own opaque paste token, from a harness whose token says nothing + * of what it holds (OMP's `๐Ÿ“„ #1`). The typed words can never reach the screen there, and the phone's + * contract for such a token is to show it and stand "Take over" down (`draftIsOpaque`), so + * recognising it IS the read. A harness that defines `draftCarriesSend` keeps the stricter check: + * its token can be matched to this send, and accepting any token would hide a mismatch. + */ +export function opaqueTokenRead(adapter: Adapter | undefined, draft: string | null): boolean { + if (adapter === undefined || draft === null || adapter.draftCarriesSend !== undefined) return false; + return adapter.draftIsOpaque?.(draft) === true; +} + /** The bullet glyphs an agent puts before its reply row. */ const REPLY_BULLET = "(?:[โบโ€ขโ—โ–ฃ>*-]\\s*)?"; @@ -434,7 +446,7 @@ export class Driver { let problems: string[] = ["not judged"]; let landed = false; for (;;) { - landed = wordsOnScreen(s.texts.slice(-COMPOSER_ROWS), m.text); + landed = wordsOnScreen(s.texts.slice(-COMPOSER_ROWS), m.text) || opaqueTokenRead(adapter, s.draft); problems = this.draftProblems(s, m.text, adapter); if ((landed && problems.length === 0) || Date.now() >= deadline) break; await Bun.sleep(POLL_MS); @@ -444,7 +456,7 @@ export class Driver { const result = !landed ? notReachedCase(m.id, "the typed words never showed on screen") : problems.length === 0 - ? passCase(m.id) + ? passCase(m.id, opaqueTokenRead(adapter, s.draft) ? `read as the opaque token ${JSON.stringify(s.draft)}` : "") : failCase(m.id, problems.join(", ")); const cleared = await this.clearDraft(m.text); return { result, cleared }; @@ -459,7 +471,9 @@ export class Driver { } if (adapter.composerReady && s.composer !== true) problems.push("composer not found"); const carried = - this.ctx.readers.draftCarriesSend(sent, s.draft) || (s.draft !== null && adapter.draftCarriesSend?.(sent, s.draft) === true); + this.ctx.readers.draftCarriesSend(sent, s.draft) || + (s.draft !== null && adapter.draftCarriesSend?.(sent, s.draft) === true) || + opaqueTokenRead(adapter, s.draft); if (!carried) problems.push(`draft not read back (read ${JSON.stringify(s.draft?.slice(0, 40) ?? null)})`); return problems; } From b9b3d9b8fcdc3485605d9d1b08d53265f0d6746d Mon Sep 17 00:00:00 2001 From: gggodlin <26342074+GGGODLIN@users.noreply.github.com> Date: Thu, 15 Oct 2026 21:56:48 +0800 Subject: [PATCH 2/2] feat(canary): a bar scenario presses each harness-bar button through the phone's send path Each picker button of an agent's shipped harness bar is sent with the client's own sendGuardedReply, closed with Escape, and its screen saved as bar-.ansi. Compact and two-tap rows are skipped, so no model turn is made. After Escape the scenario waits for the agent to be idle at its box and clears whatever the agent left there, the way the drafts scenario does: Grok 1.0.52 closes its /model picker on Escape and keeps '/model' typed. An agent with no shipped bar is reported not reached. Run on 2026-10-15 it passed claude, codex, pi, omp and opencode, and failed grok (welcome card hides the input box), muse (argument hint and popup read as draft) and agy (slash popup read as an unread dialog); those three are reader fixes sent separately, and with them in, every agent passed. Co-Authored-By: Claude --- scripts/harness-canary/README.md | 23 ++++++-- scripts/harness-canary/args.ts | 2 +- scripts/harness-canary/drive.test.ts | 22 ++++++- scripts/harness-canary/readers.ts | 13 ++++ scripts/harness-canary/run.ts | 6 +- scripts/harness-canary/scenarios.ts | 88 +++++++++++++++++++++++++++- scripts/harness-canary/verdict.ts | 5 +- 7 files changed, 147 insertions(+), 12 deletions(-) diff --git a/scripts/harness-canary/README.md b/scripts/harness-canary/README.md index 18ac60762..136af89b1 100644 --- a/scripts/harness-canary/README.md +++ b/scripts/harness-canary/README.md @@ -5,7 +5,8 @@ names them, in a Herdr session of its own, types drafts and a few real sends, and checks with Collie's own code what the phone would read and whether each send lands. It then reads the session each agent WROTE, with the bridge's own journal adapter, and fails on a row or content type the reader does not recognise. With `--dialogs` it also opens real -dialogs and sends to a busy agent, and with `--scenario options` it starts each New-sheet option once. +dialogs and sends to a busy agent, with `--scenario options` it starts each New-sheet option once, +and with `--scenario bar` it presses each harness-bar button through the phone's own send path. Specs: M37/02, M37/03 and M41/05 in the workspace tracker. ```sh @@ -15,6 +16,7 @@ bun run canary --agent claude,codex # some agents bun run canary --scenario idle,drafts # some scenarios, no model turns bun run canary --keep # leave the session up for a look bun run canary --agent claude,codex --scenario options # each New-sheet option, started once +bun run canary --agent muse,agy --scenario bar # each harness-bar picker, opened and closed bun run canary --record # write clean agents into the ledger bun run canary --readers /tmp/c1131 # judge with another checkout's readers ``` @@ -41,8 +43,8 @@ command on PATH, not through the launcher: a launcher should change the provider installed agent, and the canary does not check that it does. A launcher is yours to write, outside this repo: it points the agent at another provider and ends -in `exec "$@"`, so Herdr sees the agent itself. It covers the six default scenarios only; -`dialogs`, `busy` and `options` start their own lines. +in `exec "$@"`, so Herdr sees the agent itself. It covers the six default scenarios and +`bar`; `dialogs`, `busy` and `options` start their own lines. A launcher that moves the agent's home (`GROK_HOME`, `PI_CODING_AGENT_DIR`, `HOME`) moves its log too. Point the journal at it with the bridge's own override for that agent, `COLLIE_GROK_ROOT`, @@ -50,7 +52,7 @@ too. Point the journal at it with the bridge's own override for that agent, `COL ## What it checks -Six scenarios per agent run by default; the seventh, `options`, runs only when `--scenario` names it, and +Six scenarios per agent run by default; `options` and `bar` run only when `--scenario` names them, and grok, hermes, agy, omp and muse are the five opt-in agents that run only when `--agent` names them. Only `sends` makes model turns: four per agent. @@ -114,6 +116,19 @@ its declared cancel key, on the edit. "Confirm" is never pressed: it would allow OpenCode restarts. Plan approval is read and never answered. A dialog the model does not open (it answers in words instead) is `not-reached`. The dialog screens are saved as `dialogs-.ansi`. +### The harness bar (`--scenario bar`) + +One pane per agent. Each button of the agent's shipped harness bar (`web/src/lib/harness-bar.ts`) is +sent through the client's own `sendGuardedReply`, which is how the phone's bar sends, then Escape +closes what it opened. Compact is left out because it spends a model turn and an empty session has +nothing to compact; a two-tap row is left out because the phone asks before it sends one. No model +turn is made. + +A case passes when the command is sent, it is gone from the input box, and Escape brings the empty +input box back, after clearing what the agent itself left there (Grok keeps `/model` typed when +Escape closes its picker). That does not prove a picker opened, since each agent paints its own: the screen +each press left is saved as `bar-.ansi` for a person to read. + ## Verdicts Whether a screen was reached is judged without Collie: Herdr's agent state and the screen's plain diff --git a/scripts/harness-canary/args.ts b/scripts/harness-canary/args.ts index e81189069..0df7d3f38 100644 --- a/scripts/harness-canary/args.ts +++ b/scripts/harness-canary/args.ts @@ -55,7 +55,7 @@ screen and send with Collie's own readers. See scripts/harness-canary/README.md. --readers PATH judge with the web/src/lib readers of another Collie checkout (a worktree) --out DIR where captures and summary.json go (default: ${DEFAULT_OUT}) --via DIR start an agent through DIR/ when that file is executable (another - provider, a moved home); the default scenarios only, not dialogs or options + provider, a moved home); the default scenarios and bar, not dialogs or options -h, --help print this`; /** Parse argv (without the runtime and script). Throws a one-line message on anything unknown. */ diff --git a/scripts/harness-canary/drive.test.ts b/scripts/harness-canary/drive.test.ts index da5260d1a..d9de1b434 100644 --- a/scripts/harness-canary/drive.test.ts +++ b/scripts/harness-canary/drive.test.ts @@ -12,7 +12,7 @@ import { colorAnswers, unfinishedTail } from "./client"; import { cleanEnv } from "./herdr"; import { JOURNAL_MESSAGE, MESSAGES, README_TOKEN, NARROW_DRAFT_IDS, SEND_IDS, messageById } from "./messages"; import { lastPointedRow } from "./dialogs"; -import { answeredBelow, opaqueTokenRead } from "./scenarios"; +import { answeredBelow, opaqueTokenRead, pressableBar } from "./scenarios"; import type { Adapter } from "./readers"; import { DEFAULT_SCENARIOS } from "./verdict"; @@ -49,6 +49,13 @@ describe("parseArgs", () => { expect(DEFAULT_SCENARIOS).not.toContain("options"); }); + test("the harness bar is pressed only when asked for", () => { + const o = parseArgs(["--agent", "muse", "--scenario", "bar"], "/repo"); + if (o === "help") throw new Error("help"); + expect(o.scenarios).toEqual(["bar"]); + expect(DEFAULT_SCENARIOS).not.toContain("bar"); + }); + test("--agent, --scenario, --cols, --readers", () => { const o = parseArgs(["--agent", "claude,codex", "--scenario", "idle,drafts", "--cols", "80", "--readers", "/tmp/c1131"], "/repo"); if (o === "help") throw new Error("unexpected help"); @@ -258,3 +265,16 @@ describe("lastPointedRow", () => { expect(lastPointedRow(["no pointer here"], "โฏ")).toBeNull(); }); }); + +describe("pressableBar", () => { + test("presses the picker buttons in bar order, never Compact and never a two-tap row", () => { + const bar = [ + { id: "model", command: "/model" }, + { id: "compact", command: "/compact" }, + { id: "wipe", command: "/clear", confirm: true }, + { id: "resume", command: "/resume" }, + ]; + expect(pressableBar(bar).map((i) => i.id)).toEqual(["model", "resume"]); + expect(pressableBar([])).toEqual([]); + }); +}); diff --git a/scripts/harness-canary/readers.ts b/scripts/harness-canary/readers.ts index f346ac549..16d3976de 100644 --- a/scripts/harness-canary/readers.ts +++ b/scripts/harness-canary/readers.ts @@ -41,6 +41,8 @@ export interface Readers { adapterFor(agent: string): Adapter | undefined; draftCarriesSend(sent: string, draft: string | null): boolean; sendGuardedReply(paneId: string, text: string, agent: string): Promise; + /** The shipped harness bar for an agent. Absent when the checkout predates `harness-bar.ts`. */ + readonly barFor?: (agent: string) => readonly BarItem[]; /** * Stamp a successful read into the readers' own lib/liveness.ts (M46). Absent when the checkout * predates it (`--readers` at v1.13.1), where the reply action has no liveness gate to satisfy. @@ -48,6 +50,15 @@ export interface Readers { readonly markLive?: (paneId: string) => void; } +/** One harness-bar button, as `web/src/lib/harness-bar.ts` ships it. */ +export interface BarItem { + readonly id: string; + readonly command: string; + readonly confirm?: boolean; +} +interface BarModule { + barFor(agent: string): readonly BarItem[]; +} interface LivenessModule { markLive(paneId: string): void; } @@ -80,6 +91,7 @@ export async function loadReaders(root: string): Promise { const harness: HarnessModule = await import(join(lib, "harness", "index.ts")); const reply: ReplyModule = await import(join(lib, "reply-action.ts")); const liveness: LivenessModule | undefined = await import(join(lib, "liveness.ts")).catch(() => undefined); + const bar: BarModule | undefined = await import(join(lib, "harness-bar.ts")).catch(() => undefined); return { root, parse: (text) => blocks.splitLines(ansi.parseAnsi(text)), @@ -89,5 +101,6 @@ export async function loadReaders(root: string): Promise { draftCarriesSend: (sent, draft) => reply.draftCarriesSend(sent, draft), sendGuardedReply: (paneId, text, agent) => reply.sendGuardedReply({ paneId, text, agent }), markLive: liveness === undefined ? undefined : (paneId) => liveness.markLive(paneId), + barFor: bar === undefined ? undefined : (agent) => bar.barFor(agent), }; } diff --git a/scripts/harness-canary/run.ts b/scripts/harness-canary/run.ts index 2d6d5c9aa..ef2aadc57 100644 --- a/scripts/harness-canary/run.ts +++ b/scripts/harness-canary/run.ts @@ -136,10 +136,10 @@ async function main(options: CanaryOptions): Promise { // of the canary has always been this checkout's (README ยง "Proof against 1.13.1"). const journals = canaryJournals(); adapters = new Map(options.agents.map((a) => [a, readers.adapterFor(a) !== undefined])); - const usesProfileLaunch = options.scenarios.some((s) => DEFAULT_SCENARIOS.includes(s)); + const usesProfileLaunch = options.scenarios.some((s) => DEFAULT_SCENARIOS.includes(s) || s === "bar"); for (const agent of options.agents) { - // Only the default scenarios start through the profile; `dialogs`, `busy` and `options` type - // their own lines, so a launcher is neither used nor credited when none of those run. + // Only the default scenarios and `bar` start through the profile; `dialogs`, `busy` and + // `options` type their own lines, so a launcher is neither used nor credited when none of those run. const launcher = usesProfileLaunch ? viaLauncher(options.via, agent) : null; // The version comes from the profile's own command on PATH, never through the launcher; the // canary does not check that a launcher runs that same binary. diff --git a/scripts/harness-canary/scenarios.ts b/scripts/harness-canary/scenarios.ts index 0b0d8b5bf..0c532b667 100644 --- a/scripts/harness-canary/scenarios.ts +++ b/scripts/harness-canary/scenarios.ts @@ -20,7 +20,7 @@ import type { JournalAdapter } from "../../bridge/journal/types"; import type { CanarySession } from "./herdr"; import { judgeJournal, loadOwnSession } from "./journal"; import { MESSAGES, NARROW_DRAFT_IDS, SEND_IDS, messageById, type CanaryMessage } from "./messages"; -import type { Adapter, Block, Line, Readers } from "./readers"; +import type { Adapter, BarItem, Block, Line, Readers } from "./readers"; import type { Transport } from "./transport"; import { failCase, notReachedCase, passCase, scenarioResult, type CaseResult, type ScenarioId, type ScenarioResult } from "./verdict"; @@ -41,6 +41,8 @@ const TURN_TIMEOUT_MS = 120_000; */ const TOOL_TURN_TIMEOUT_MS = 300_000; const SEND_TIMEOUT_MS = 60_000; +/** How long a pressed bar button has to paint its picker before the screen is kept. */ +const BAR_SETTLE_MS = 2_000; const EXIT_TIMEOUT_MS = 12_000; /** How long the exit window keeps sampling once the shell is back. */ const EXIT_TAIL_MS = 1_500; @@ -162,6 +164,7 @@ export async function runAgent(ctx: AgentContext): Promise { if (want("start-exit")) results.push(scenarioResult(agent, "start-exit", startExit)); if (want("options")) results.push(scenarioResult(agent, "options", await launchOptions(ctx))); + if (want("bar")) results.push(scenarioResult(agent, "bar", await pressBar(ctx))); return results; } @@ -194,6 +197,47 @@ async function launchOptions(ctx: AgentContext): Promise { return cases; } +/** + * The bar buttons the `bar` scenario presses, in bar order. Compact is left out because it spends a + * model turn and an empty session has nothing to compact; a two-tap row is left out because the + * phone asks before it sends one. + */ +export function pressableBar(items: readonly BarItem[]): BarItem[] { + return items.filter((item) => item.id !== "compact" && item.confirm !== true); +} + +/** + * Every pressable button of the agent's shipped harness bar, sent through the client's own + * `sendGuardedReply` on a live pane, the path the phone's bar sends on. A pass is the command sent + * and gone from the input box, and Escape bringing the empty input box back. It does not prove a + * picker opened, since what one looks like is each agent's own: the screen each press left is saved + * under `bar-` for a person to read. No model turn is made. + */ +async function pressBar(ctx: AgentContext): Promise { + const agent = ctx.profile.agent; + const items = pressableBar(ctx.readers.barFor?.(agent) ?? []); + if (items.length === 0) return [notReachedCase("bar", `no shipped harness bar for ${agent}`)]; + const d = await Driver.open(ctx, `canary-${agent}-bar`, ctx.options.cols, "bar"); + try { + const ready = await d.launch([]); + if (ready === null) return items.map((item) => notStarted(`bar-${item.id}`)); + const cases: CaseResult[] = []; + let live = true; + for (const item of items) { + if (!live) { + cases.push(notReachedCase(`bar-${item.id}`, "the previous button left the input box unreachable")); + continue; + } + const r = await d.press(item); + cases.push(r.result); + live = r.closed; + } + return cases; + } finally { + d.close(); + } +} + function notStarted(id: string): CaseResult { return notReachedCase(id, "the agent never came up idle"); } @@ -387,6 +431,48 @@ export class Driver { return null; } + /** One `bar` case: the button's command sent the way the phone's bar sends it, then Escape. */ + async press(item: BarItem): Promise<{ result: CaseResult; closed: boolean }> { + const id = `bar-${item.id}`; + const outcome = await Promise.race([ + this.ctx.readers.sendGuardedReply(this.paneId, item.command, this.agent), + Bun.sleep(SEND_TIMEOUT_MS).then(() => ({ status: "error" as const, error: "sendGuardedReply did not return in 60 s" })), + ]); + if (outcome.status !== "sent") { + const file = this.save(id, await this.screen()); + const closed = await this.clearDraft(item.command); + return { result: failCase(id, `${item.command}: outcome ${outcome.status}: ${outcome.error} (${file})`), closed }; + } + await Bun.sleep(BAR_SETTLE_MS); + const s = await this.screen(); + const file = this.save(id, s); + const stuck = s.draft !== null && this.ctx.readers.draftCarriesSend(item.command, s.draft); + const closed = await this.escapeToComposer(); + if (stuck) return { result: failCase(id, `${item.command} stayed in the input box (${file})`), closed }; + if (!closed) return { result: failCase(id, `${item.command}: Escape did not bring the empty input box back (${file})`), closed }; + return { result: passCase(id, `${item.command} sent, closed with Escape (${file})`), closed }; + } + + /** + * Escape until the agent is idle at its input box again, at most three times, then clear whatever + * the agent itself left in the box: Grok closes its /model picker on Escape and keeps `/model` + * typed (live, 1.0.52). + */ + private async escapeToComposer(): Promise { + for (let attempt = 0; attempt < 3; attempt++) { + this.keys(["Escape"]); + const deadline = Date.now() + CLEAR_TIMEOUT_MS; + while (Date.now() < deadline) { + await Bun.sleep(POLL_MS * 2); + const s = await this.screen(); + if (s.composer === false || !this.herdrIdle()) continue; + if (s.draft === null || s.draft.trim() === "") return true; + return this.clearDraft(s.draft); + } + } + return false; + } + /** One `options` case: `command` brings the agent up, idle or asking first. */ async started(id: string, command: string): Promise { this.ctx.session.sendText(this.paneId, command); diff --git a/scripts/harness-canary/verdict.ts b/scripts/harness-canary/verdict.ts index 3ceca1c28..f9a103cbb 100644 --- a/scripts/harness-canary/verdict.ts +++ b/scripts/harness-canary/verdict.ts @@ -13,12 +13,13 @@ export type Verdict = CaseVerdict | "known-gap"; /** * Every scenario, by the ids the table, the captures and known-gaps.json use: the five of spec * M37/02, then `journal` (M41/05), then the two of M37/03 (`dialogs`, `busy`), which make more model - * turns and run only when asked for (`--dialogs` or `--scenario`). + * turns and run only when asked for (`--dialogs` or `--scenario`). `options` and `bar` also run only + * when `--scenario` names them, and make no model turn. * * `journal` sits next to `sends` because it reads what those sends wrote: it makes no model turn of * its own and it has nothing to read unless `sends` ran in the same pane. */ -export const SCENARIOS = ["idle", "drafts", "sends", "journal", "narrow", "start-exit", "dialogs", "busy", "options"] as const; +export const SCENARIOS = ["idle", "drafts", "sends", "journal", "narrow", "start-exit", "dialogs", "busy", "options", "bar"] as const; export type ScenarioId = (typeof SCENARIOS)[number]; /** What a run without `--scenario` does: the five screens and sends of M37/02, plus M41/05's read of