From 760b849c1534e35f0429744d029e16b28f66a773 Mon Sep 17 00:00:00 2001 From: Roomote Date: Sun, 25 Oct 2026 22:41:22 +0000 Subject: [PATCH] feat(evals): show the Zoo Code panel in the Solheim smoke video Add the sidebar-runner.cjs extension-tests module so the smoke recording shows the Zoo Code chat panel, hold teardown three seconds after the completion message, and bound the display dependency install at five minutes. No production extension changes. --- .github/workflows/solheim-provider-smoke.yml | 1 + scripts/agent-evals/README.md | 5 ++ scripts/agent-evals/isolation.ts | 11 ++- scripts/agent-evals/launcher.ts | 4 + scripts/agent-evals/runner.test.ts | 79 ++++++++++++++++++++ scripts/agent-evals/runner.ts | 9 +++ scripts/solheim-smoke-workflow.test.mjs | 6 ++ scripts/solheim-smoke/README.md | 10 ++- scripts/solheim-smoke/config.test.ts | 12 +++ scripts/solheim-smoke/config.ts | 3 + scripts/solheim-smoke/driver.mts | 8 +- scripts/solheim-smoke/run.test.ts | 7 +- scripts/solheim-smoke/run.ts | 3 +- scripts/solheim-smoke/sidebar-runner.cjs | 39 ++++++++++ 14 files changed, 191 insertions(+), 6 deletions(-) create mode 100644 scripts/solheim-smoke/sidebar-runner.cjs diff --git a/.github/workflows/solheim-provider-smoke.yml b/.github/workflows/solheim-provider-smoke.yml index 7e0298818c..fb9dc2a7d7 100644 --- a/.github/workflows/solheim-provider-smoke.yml +++ b/.github/workflows/solheim-provider-smoke.yml @@ -41,6 +41,7 @@ jobs: pnpm --filter @roo-code/vscode-webview build - name: Install display dependencies + timeout-minutes: 5 run: sudo apt-get update && sudo apt-get install -y xvfb ffmpeg libasound2t64 - name: Run Solheim provider smoke diff --git a/scripts/agent-evals/README.md b/scripts/agent-evals/README.md index bd24b6cfba..6eef1d6b0b 100644 --- a/scripts/agent-evals/README.md +++ b/scripts/agent-evals/README.md @@ -30,6 +30,11 @@ Execution limits are enforced, not decorative. The controller applies the activation, task start, task run, usage drain, and teardown step limits, and the trace records the measured durations. +Two launch options are off by default. `testsPath` adds `--extensionTestsPath` +to the VS Code launch; that module's `run()` must never settle, because VS Code +exits once it does. `holdBeforeTeardownMs` holds the finished task on screen +for that window after the completion message, before teardown starts. + ## Structured trace Each run writes `trace.json` with `traceSchemaVersion: 1`. The trace is diff --git a/scripts/agent-evals/isolation.ts b/scripts/agent-evals/isolation.ts index 06ab950ef6..17af4a9ae3 100644 --- a/scripts/agent-evals/isolation.ts +++ b/scripts/agent-evals/isolation.ts @@ -19,12 +19,21 @@ export function buildChildEnvironment( } } -export function buildLaunchArgs(workspace: string, userData: string, extensions: string, extension: string): string[] { +export function buildLaunchArgs( + workspace: string, + userData: string, + extensions: string, + extension: string, + testsPath?: string, +): string[] { return [ workspace, `--user-data-dir=${userData}`, `--extensions-dir=${extensions}`, `--extensionDevelopmentPath=${extension}`, + // Off by default: an --extensionTestsPath module settles into a VS Code + // exit, so only a caller that owns a never-settling module may set it. + ...(testsPath ? [`--extensionTestsPath=${testsPath}`] : []), "--no-sandbox", "--disable-gpu-sandbox", "--password-store=basic", diff --git a/scripts/agent-evals/launcher.ts b/scripts/agent-evals/launcher.ts index 1edd2917d2..3cbf5b321f 100644 --- a/scripts/agent-evals/launcher.ts +++ b/scripts/agent-evals/launcher.ts @@ -17,6 +17,9 @@ export interface HostRuntimeOptions { artifactsDir: string now?: () => number pause?: (ms: number) => Promise + // Optional --extensionTestsPath module. The module's run() must never + // settle, because VS Code exits when it does. Off by default. + testsPath?: string } const readCommit = async (root: string): Promise => { @@ -84,6 +87,7 @@ export function createHostRuntime(options: HostRuntimeOptions): AgentRuntime { path.join(temp, "userdata"), path.join(temp, "extensions"), extension, + options.testsPath, ) const child = spawn( process.env.DISPLAY ? executable : "xvfb-run", diff --git a/scripts/agent-evals/runner.test.ts b/scripts/agent-evals/runner.test.ts index d65e56a144..5f66c0ac63 100644 --- a/scripts/agent-evals/runner.test.ts +++ b/scripts/agent-evals/runner.test.ts @@ -1050,4 +1050,83 @@ describe("agent fixture controller", () => { assert.equal(result.execution.sessionLost, false) assert.equal(result.teardown.status, "ok") }) + + it("holds teardown after the completion message for the requested window", async () => { + registerTestFixture("controller-hold") + const f = harness("no-events") + f.emit({ + eventName: RooCodeEventName.Message, + payload: [ + { + taskId: "root", + action: "created", + message: { ts: 1, type: "say", say: "completion_result", text: "synthetic final answer" }, + }, + ], + }) + f.emit({ + eventName: RooCodeEventName.TaskCompleted, + payload: [ + "root", + { totalTokensIn: 7, totalTokensOut: 3, totalCost: 0, contextTokens: 9 }, + {}, + { + isSubtask: false, + }, + ], + }) + const holdPauses: number[] = [] + let closeTaskSeenDuringHold = false + const pause = f.runtime.pause + f.runtime.pause = async (ms) => { + if (ms === 3_000) { + holdPauses.push(ms) + closeTaskSeenDuringHold = f.commands.some( + (command) => command.commandName === TaskCommandName.CloseTask, + ) + } + await pause(ms) + } + const result = await runAgentFixture(f.runtime, { + fixtureName: "controller-hold", + credential: CREDENTIAL, + observers: { shouldStop: () => true }, + holdBeforeTeardownMs: 3_000, + }) + assert.equal(result.execution.code, "COMPLETED") + assert.equal(result.execution.completionMessageSeen, true) + assert.deepEqual(holdPauses, [3_000]) + // The hold runs before teardown, so CloseTask is sent only after it. + assert.equal(closeTaskSeenDuringHold, false) + assert.equal(f.commands.at(-1)?.commandName, TaskCommandName.CloseTask) + assert.equal(result.teardown.status, "ok") + }) + + it("skips the teardown hold without a completion message", async () => { + registerTestFixture("controller-hold-skipped", { + limits: { + activationTimeoutMs: 120_000, + taskStartTimeoutMs: 35_000, + taskRunTimeoutMs: 200, + usageDrainMs: 100, + teardownStepMs: 100, + }, + }) + const f = harness("no-events") + const holdPauses: number[] = [] + const pause = f.runtime.pause + f.runtime.pause = async (ms) => { + if (ms === 3_000) holdPauses.push(ms) + await pause(ms) + } + const result = await runAgentFixture(f.runtime, { + fixtureName: "controller-hold-skipped", + credential: CREDENTIAL, + holdBeforeTeardownMs: 3_000, + }) + assert.equal(result.execution.code, "TASK_RUN_TIMEOUT") + assert.equal(result.execution.completionMessageSeen, false) + assert.deepEqual(holdPauses, []) + assert.equal(result.teardown.status, "ok") + }) }) diff --git a/scripts/agent-evals/runner.ts b/scripts/agent-evals/runner.ts index 5c12961db3..c671dabc83 100644 --- a/scripts/agent-evals/runner.ts +++ b/scripts/agent-evals/runner.ts @@ -57,6 +57,10 @@ export interface AgentRunOptions { fixtureName: string credential: string observers?: AgentRunObservers + // Optional hold before teardown after the completion message, so a + // recording can capture the finished task. Off by default; skipped when + // no completion message was seen or the host is already gone. + holdBeforeTeardownMs?: number } export interface AgentRunResult { @@ -325,6 +329,11 @@ export async function runAgentFixture(runtime: AgentRuntime, options: AgentRunOp // Never expose thrown provider, filesystem or host messages. } finally { waiter?.onExit() + // A recording needs the finished task on screen before teardown + // clears it. The hold is opt-in and only applies after the + // completion message, while a live host can still show it. + if (completionMessageSeen && options.holdBeforeTeardownMs && host && !host.exited) + await runtime.pause(options.holdBeforeTeardownMs) const teardownStart = runtime.now() const attempt = async (step: string, operation: () => unknown) => { try { diff --git a/scripts/solheim-smoke-workflow.test.mjs b/scripts/solheim-smoke-workflow.test.mjs index b6656d6aef..f6394234d2 100644 --- a/scripts/solheim-smoke-workflow.test.mjs +++ b/scripts/solheim-smoke-workflow.test.mjs @@ -34,6 +34,12 @@ describe("Solheim provider smoke workflow policy", () => { assert.equal(checkouts[0].with["persist-credentials"], false) }) + it("bounds the display dependency install to five minutes", () => { + const apt = steps.find((step) => step.name === "Install display dependencies") + assert.ok(apt, "missing display dependency step") + assert.equal(apt["timeout-minutes"], 5) + }) + it("exposes the provider credential only to the bounded provider step", () => { const provider = steps.find((step) => step.id === "provider") assert.ok(provider, "missing provider step") diff --git a/scripts/solheim-smoke/README.md b/scripts/solheim-smoke/README.md index 30b331ae90..6ac5f30911 100644 --- a/scripts/solheim-smoke/README.md +++ b/scripts/solheim-smoke/README.md @@ -38,7 +38,10 @@ Activation and completion each have a 120-second limit; task-start acknowledgmen in the controller. The extension awaits startup and configuration persistence without racing shared-profile writes against a timer. If the controller times out, it tears down its isolated VS Code process rather than trying to roll back shared settings or remove a possibly unrelated task. The live CI step has an eight-minute outer timeout. -Teardown escalates through close, terminate, and kill within independent five-second windows. A non-empty +Teardown escalates through close, terminate, and kill within independent five-second windows. After the +completion message, the driver first holds the finished task on screen for three seconds, so the recording ends +on the completed chat before teardown clears it. The display dependency install step in the workflow is bounded +at five minutes, so a slow apt mirror fails the job instead of consuming it. A non-empty completion and actual output-token usage must belong to the accepted task. Activation alone cannot pass. The driver never approves dialogs or tool execution; it closes the task during teardown. @@ -53,7 +56,10 @@ It does not contradict the `OK` verdict. See The action uploads `verdict.json` (fixed failure code, boolean gates, duration and provider/model names), `trace.json` (versioned metadata-only structured trace from the shared substrate; see [`scripts/agent-evals/README.md`](../agent-evals/README.md) for its accounting limits), and `smoke.mp4`, -a 1280×720, 12 fps screen recording of the fresh CI virtual display. It records up to +a 1280×720, 12 fps screen recording of the fresh CI virtual display. The recording shows the Zoo Code chat +panel: `sidebar-runner.cjs` runs inside the extension host through `--extensionTestsPath`, focuses the Zoo Code +sidebar, and then keeps its `run()` pending for the whole smoke, because VS Code exits once that `run()` settles. +It records up to eight minutes / approximately 120 MiB, with bounded shutdown and seven-day artifact retention. Video contains the visible smoke UI, potentially including the fixed prompt and model answer; unlike the verdict and the trace, it is not text-redacted. No settings screen is opened, and no host logs, storage, task diff --git a/scripts/solheim-smoke/config.test.ts b/scripts/solheim-smoke/config.test.ts index 345062f0d3..d770ca6155 100644 --- a/scripts/solheim-smoke/config.test.ts +++ b/scripts/solheim-smoke/config.test.ts @@ -62,4 +62,16 @@ describe("Solheim provider smoke configuration", () => { assert.ok(args.includes("--extensionDevelopmentPath=/extension")) assert.ok(args.includes("--user-data-dir=/data")) }) + it("adds the extension tests path only when one is supplied", () => { + const plain = buildLaunchArgs("/empty-workspace", "/data", "/extensions", "/extension") + assert.ok(!plain.some((arg) => arg.startsWith("--extensionTestsPath="))) + const withTests = buildLaunchArgs( + "/empty-workspace", + "/data", + "/extensions", + "/extension", + "/runner/sidebar-runner.cjs", + ) + assert.ok(withTests.includes("--extensionTestsPath=/runner/sidebar-runner.cjs")) + }) }) diff --git a/scripts/solheim-smoke/config.ts b/scripts/solheim-smoke/config.ts index b394d12cf6..215a37d588 100644 --- a/scripts/solheim-smoke/config.ts +++ b/scripts/solheim-smoke/config.ts @@ -5,6 +5,9 @@ export { buildChildEnvironment, buildLaunchArgs } from "../agent-evals/isolation export const SOLHEIM_BASE_URL = "https://api.solheim.ai/v1" export const SOLHEIM_MODEL = "qwen3.8-27b" export const SMOKE_TIMEOUT_MS = 120_000 +// Keeps the finished task on screen before teardown clears it, so the +// recording ends on the completed chat. +export const SMOKE_TEARDOWN_HOLD_MS = 3_000 export const SMOKE_PROMPT = "This is a provider connectivity smoke test, not a code review. Call attempt_completion once with the result: Provider smoke completed. Do not read files, use other tools, delegate, or judge code." diff --git a/scripts/solheim-smoke/driver.mts b/scripts/solheim-smoke/driver.mts index df10cd5566..d08a97c104 100644 --- a/scripts/solheim-smoke/driver.mts +++ b/scripts/solheim-smoke/driver.mts @@ -8,7 +8,13 @@ const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..") const out = process.env.SOLHEIM_SMOKE_OUT_DIR if (!out) throw new Error("Set SOLHEIM_SMOKE_OUT_DIR to a scratch directory") await fs.mkdir(out, { recursive: true }) -const runtime = createHostRuntime({ root, artifactsDir: out }) +const runtime = createHostRuntime({ + root, + artifactsDir: out, + // The runner module focuses the Zoo Code sidebar so the recording shows + // the chat panel. Its run() never settles; the driver owns teardown. + testsPath: path.join(root, "scripts", "solheim-smoke", "sidebar-runner.cjs"), +}) try { const verdict = await runSmoke(runtime, process.env.SOLHEIM_API_KEY ?? "") console.log(`Solheim provider smoke: ${verdict.code}`) diff --git a/scripts/solheim-smoke/run.test.ts b/scripts/solheim-smoke/run.test.ts index 9be74b3d75..d27a0bb229 100644 --- a/scripts/solheim-smoke/run.test.ts +++ b/scripts/solheim-smoke/run.test.ts @@ -1,6 +1,7 @@ import assert from "node:assert/strict" import { describe, it } from "node:test" import { RooCodeEventName, TaskCommandName, type TaskCommand } from "@roo-code/types" +import { SMOKE_TEARDOWN_HOLD_MS } from "./config.ts" import { runSmoke, smokeExitCode, type SmokeRuntime, type SmokeVerdict } from "./run.ts" type Scenario = @@ -164,6 +165,9 @@ describe("provider smoke orchestration", () => { const verdict = await runSmoke(f.runtime, "private-test-key") assert.equal(verdict.code, "OK") assert.equal(smokeExitCode(verdict), 0) + // The teardown hold runs once after the completion message; no other + // pause fires in this scenario, so the duration is exactly the hold. + assert.equal(verdict.durationMs, SMOKE_TEARDOWN_HOLD_MS) assert.deepEqual(verdict.gates, { extensionActivated: true, ipcConnected: true, @@ -210,7 +214,8 @@ describe("provider smoke orchestration", () => { const f = fixture("success", { stuck: true }) const verdict = await runSmoke(f.runtime, "private-test-key") assert.equal(verdict.code, "TEARDOWN_FAILED") - assert.equal(verdict.durationMs, 15_000) + // The 3 second completion hold plus three 5 second termination windows. + assert.equal(verdict.durationMs, 15_000 + SMOKE_TEARDOWN_HOLD_MS) assert.deepEqual(f.signals, ["SIGTERM", "SIGKILL"]) assert.equal(verdict.gates.childExited, false) assert.equal(f.cleaned, false) diff --git a/scripts/solheim-smoke/run.ts b/scripts/solheim-smoke/run.ts index 8cbf1e9fb5..b2aed5c0e9 100644 --- a/scripts/solheim-smoke/run.ts +++ b/scripts/solheim-smoke/run.ts @@ -1,6 +1,6 @@ import { runAgentFixture, type AgentRuntime } from "../agent-evals/runner.ts" import { getFixture } from "../agent-evals/registry.ts" -import { SOLHEIM_BASE_URL, SOLHEIM_MODEL } from "./config.ts" +import { SOLHEIM_BASE_URL, SOLHEIM_MODEL, SMOKE_TEARDOWN_HOLD_MS } from "./config.ts" import { collectEvidence, smokeCode, type SmokeCode, type SmokeEvidence } from "./evidence.ts" import { SMOKE_FIXTURE_NAME, ensureSmokeFixtureRegistered } from "./fixture.ts" @@ -45,6 +45,7 @@ export async function runSmoke(runtime: SmokeRuntime, apiKey: string): Promise new Promise((resolve) => setTimeout(resolve, ms)) + +async function focusSidebarWhenReady() { + const extension = vscode.extensions.getExtension(EXTENSION_ID) + if (!extension) throw new Error("Zoo Code extension not found") + const api = extension.isActive ? extension.exports : await extension.activate() + await vscode.commands.executeCommand(FOCUS_COMMAND) + const deadline = Date.now() + READY_TIMEOUT_MS + while (!api.isReady()) { + if (Date.now() >= deadline) throw new Error("Zoo Code webview did not launch in time") + await delay(POLL_INTERVAL_MS) + } +} + +exports.run = async () => { + try { + await focusSidebarWhenReady() + } catch (error) { + // Focus is best effort for the recording. A rejected run() would end + // the test host and cut the smoke short, so log and stay pending. + console.warn("[solheim-smoke] sidebar focus failed:", error && error.message ? error.message : error) + } + await new Promise(() => {}) +}