diff --git a/.github/workflows/solheim-provider-smoke.yml b/.github/workflows/solheim-provider-smoke.yml index 7e0298818c..fb9dc2a7d7 100644 --- a/.github/workflows/solheim-provider-smoke.yml +++ b/.github/workflows/solheim-provider-smoke.yml @@ -41,6 +41,7 @@ jobs: pnpm --filter @roo-code/vscode-webview build - name: Install display dependencies + timeout-minutes: 5 run: sudo apt-get update && sudo apt-get install -y xvfb ffmpeg libasound2t64 - name: Run Solheim provider smoke diff --git a/scripts/agent-evals/README.md b/scripts/agent-evals/README.md index bd24b6cfba..6eef1d6b0b 100644 --- a/scripts/agent-evals/README.md +++ b/scripts/agent-evals/README.md @@ -30,6 +30,11 @@ Execution limits are enforced, not decorative. The controller applies the activation, task start, task run, usage drain, and teardown step limits, and the trace records the measured durations. +Two launch options are off by default. `testsPath` adds `--extensionTestsPath` +to the VS Code launch; that module's `run()` must never settle, because VS Code +exits once it does. `holdBeforeTeardownMs` holds the finished task on screen +for that window after the completion message, before teardown starts. + ## Structured trace Each run writes `trace.json` with `traceSchemaVersion: 1`. The trace is diff --git a/scripts/agent-evals/isolation.ts b/scripts/agent-evals/isolation.ts index 06ab950ef6..17af4a9ae3 100644 --- a/scripts/agent-evals/isolation.ts +++ b/scripts/agent-evals/isolation.ts @@ -19,12 +19,21 @@ export function buildChildEnvironment( } } -export function buildLaunchArgs(workspace: string, userData: string, extensions: string, extension: string): string[] { +export function buildLaunchArgs( + workspace: string, + userData: string, + extensions: string, + extension: string, + testsPath?: string, +): string[] { return [ workspace, `--user-data-dir=${userData}`, `--extensions-dir=${extensions}`, `--extensionDevelopmentPath=${extension}`, + // Off by default: an --extensionTestsPath module settles into a VS Code + // exit, so only a caller that owns a never-settling module may set it. + ...(testsPath ? [`--extensionTestsPath=${testsPath}`] : []), "--no-sandbox", "--disable-gpu-sandbox", "--password-store=basic", diff --git a/scripts/agent-evals/launcher.ts b/scripts/agent-evals/launcher.ts index 1edd2917d2..3cbf5b321f 100644 --- a/scripts/agent-evals/launcher.ts +++ b/scripts/agent-evals/launcher.ts @@ -17,6 +17,9 @@ export interface HostRuntimeOptions { artifactsDir: string now?: () => number pause?: (ms: number) => Promise + // Optional --extensionTestsPath module. The module's run() must never + // settle, because VS Code exits when it does. Off by default. + testsPath?: string } const readCommit = async (root: string): Promise => { @@ -84,6 +87,7 @@ export function createHostRuntime(options: HostRuntimeOptions): AgentRuntime { path.join(temp, "userdata"), path.join(temp, "extensions"), extension, + options.testsPath, ) const child = spawn( process.env.DISPLAY ? executable : "xvfb-run", diff --git a/scripts/agent-evals/runner.test.ts b/scripts/agent-evals/runner.test.ts index d65e56a144..5f66c0ac63 100644 --- a/scripts/agent-evals/runner.test.ts +++ b/scripts/agent-evals/runner.test.ts @@ -1050,4 +1050,83 @@ describe("agent fixture controller", () => { assert.equal(result.execution.sessionLost, false) assert.equal(result.teardown.status, "ok") }) + + it("holds teardown after the completion message for the requested window", async () => { + registerTestFixture("controller-hold") + const f = harness("no-events") + f.emit({ + eventName: RooCodeEventName.Message, + payload: [ + { + taskId: "root", + action: "created", + message: { ts: 1, type: "say", say: "completion_result", text: "synthetic final answer" }, + }, + ], + }) + f.emit({ + eventName: RooCodeEventName.TaskCompleted, + payload: [ + "root", + { totalTokensIn: 7, totalTokensOut: 3, totalCost: 0, contextTokens: 9 }, + {}, + { + isSubtask: false, + }, + ], + }) + const holdPauses: number[] = [] + let closeTaskSeenDuringHold = false + const pause = f.runtime.pause + f.runtime.pause = async (ms) => { + if (ms === 3_000) { + holdPauses.push(ms) + closeTaskSeenDuringHold = f.commands.some( + (command) => command.commandName === TaskCommandName.CloseTask, + ) + } + await pause(ms) + } + const result = await runAgentFixture(f.runtime, { + fixtureName: "controller-hold", + credential: CREDENTIAL, + observers: { shouldStop: () => true }, + holdBeforeTeardownMs: 3_000, + }) + assert.equal(result.execution.code, "COMPLETED") + assert.equal(result.execution.completionMessageSeen, true) + assert.deepEqual(holdPauses, [3_000]) + // The hold runs before teardown, so CloseTask is sent only after it. + assert.equal(closeTaskSeenDuringHold, false) + assert.equal(f.commands.at(-1)?.commandName, TaskCommandName.CloseTask) + assert.equal(result.teardown.status, "ok") + }) + + it("skips the teardown hold without a completion message", async () => { + registerTestFixture("controller-hold-skipped", { + limits: { + activationTimeoutMs: 120_000, + taskStartTimeoutMs: 35_000, + taskRunTimeoutMs: 200, + usageDrainMs: 100, + teardownStepMs: 100, + }, + }) + const f = harness("no-events") + const holdPauses: number[] = [] + const pause = f.runtime.pause + f.runtime.pause = async (ms) => { + if (ms === 3_000) holdPauses.push(ms) + await pause(ms) + } + const result = await runAgentFixture(f.runtime, { + fixtureName: "controller-hold-skipped", + credential: CREDENTIAL, + holdBeforeTeardownMs: 3_000, + }) + assert.equal(result.execution.code, "TASK_RUN_TIMEOUT") + assert.equal(result.execution.completionMessageSeen, false) + assert.deepEqual(holdPauses, []) + assert.equal(result.teardown.status, "ok") + }) }) diff --git a/scripts/agent-evals/runner.ts b/scripts/agent-evals/runner.ts index 5c12961db3..c671dabc83 100644 --- a/scripts/agent-evals/runner.ts +++ b/scripts/agent-evals/runner.ts @@ -57,6 +57,10 @@ export interface AgentRunOptions { fixtureName: string credential: string observers?: AgentRunObservers + // Optional hold before teardown after the completion message, so a + // recording can capture the finished task. Off by default; skipped when + // no completion message was seen or the host is already gone. + holdBeforeTeardownMs?: number } export interface AgentRunResult { @@ -325,6 +329,11 @@ export async function runAgentFixture(runtime: AgentRuntime, options: AgentRunOp // Never expose thrown provider, filesystem or host messages. } finally { waiter?.onExit() + // A recording needs the finished task on screen before teardown + // clears it. The hold is opt-in and only applies after the + // completion message, while a live host can still show it. + if (completionMessageSeen && options.holdBeforeTeardownMs && host && !host.exited) + await runtime.pause(options.holdBeforeTeardownMs) const teardownStart = runtime.now() const attempt = async (step: string, operation: () => unknown) => { try { diff --git a/scripts/solheim-smoke-workflow.test.mjs b/scripts/solheim-smoke-workflow.test.mjs index b6656d6aef..f6394234d2 100644 --- a/scripts/solheim-smoke-workflow.test.mjs +++ b/scripts/solheim-smoke-workflow.test.mjs @@ -34,6 +34,12 @@ describe("Solheim provider smoke workflow policy", () => { assert.equal(checkouts[0].with["persist-credentials"], false) }) + it("bounds the display dependency install to five minutes", () => { + const apt = steps.find((step) => step.name === "Install display dependencies") + assert.ok(apt, "missing display dependency step") + assert.equal(apt["timeout-minutes"], 5) + }) + it("exposes the provider credential only to the bounded provider step", () => { const provider = steps.find((step) => step.id === "provider") assert.ok(provider, "missing provider step") diff --git a/scripts/solheim-smoke/README.md b/scripts/solheim-smoke/README.md index 30b331ae90..6ac5f30911 100644 --- a/scripts/solheim-smoke/README.md +++ b/scripts/solheim-smoke/README.md @@ -38,7 +38,10 @@ Activation and completion each have a 120-second limit; task-start acknowledgmen in the controller. The extension awaits startup and configuration persistence without racing shared-profile writes against a timer. If the controller times out, it tears down its isolated VS Code process rather than trying to roll back shared settings or remove a possibly unrelated task. The live CI step has an eight-minute outer timeout. -Teardown escalates through close, terminate, and kill within independent five-second windows. A non-empty +Teardown escalates through close, terminate, and kill within independent five-second windows. After the +completion message, the driver first holds the finished task on screen for three seconds, so the recording ends +on the completed chat before teardown clears it. The display dependency install step in the workflow is bounded +at five minutes, so a slow apt mirror fails the job instead of consuming it. A non-empty completion and actual output-token usage must belong to the accepted task. Activation alone cannot pass. The driver never approves dialogs or tool execution; it closes the task during teardown. @@ -53,7 +56,10 @@ It does not contradict the `OK` verdict. See The action uploads `verdict.json` (fixed failure code, boolean gates, duration and provider/model names), `trace.json` (versioned metadata-only structured trace from the shared substrate; see [`scripts/agent-evals/README.md`](../agent-evals/README.md) for its accounting limits), and `smoke.mp4`, -a 1280×720, 12 fps screen recording of the fresh CI virtual display. It records up to +a 1280×720, 12 fps screen recording of the fresh CI virtual display. The recording shows the Zoo Code chat +panel: `sidebar-runner.cjs` runs inside the extension host through `--extensionTestsPath`, focuses the Zoo Code +sidebar, and then keeps its `run()` pending for the whole smoke, because VS Code exits once that `run()` settles. +It records up to eight minutes / approximately 120 MiB, with bounded shutdown and seven-day artifact retention. Video contains the visible smoke UI, potentially including the fixed prompt and model answer; unlike the verdict and the trace, it is not text-redacted. No settings screen is opened, and no host logs, storage, task diff --git a/scripts/solheim-smoke/config.test.ts b/scripts/solheim-smoke/config.test.ts index 345062f0d3..d770ca6155 100644 --- a/scripts/solheim-smoke/config.test.ts +++ b/scripts/solheim-smoke/config.test.ts @@ -62,4 +62,16 @@ describe("Solheim provider smoke configuration", () => { assert.ok(args.includes("--extensionDevelopmentPath=/extension")) assert.ok(args.includes("--user-data-dir=/data")) }) + it("adds the extension tests path only when one is supplied", () => { + const plain = buildLaunchArgs("/empty-workspace", "/data", "/extensions", "/extension") + assert.ok(!plain.some((arg) => arg.startsWith("--extensionTestsPath="))) + const withTests = buildLaunchArgs( + "/empty-workspace", + "/data", + "/extensions", + "/extension", + "/runner/sidebar-runner.cjs", + ) + assert.ok(withTests.includes("--extensionTestsPath=/runner/sidebar-runner.cjs")) + }) }) diff --git a/scripts/solheim-smoke/config.ts b/scripts/solheim-smoke/config.ts index b394d12cf6..215a37d588 100644 --- a/scripts/solheim-smoke/config.ts +++ b/scripts/solheim-smoke/config.ts @@ -5,6 +5,9 @@ export { buildChildEnvironment, buildLaunchArgs } from "../agent-evals/isolation export const SOLHEIM_BASE_URL = "https://api.solheim.ai/v1" export const SOLHEIM_MODEL = "qwen3.8-27b" export const SMOKE_TIMEOUT_MS = 120_000 +// Keeps the finished task on screen before teardown clears it, so the +// recording ends on the completed chat. +export const SMOKE_TEARDOWN_HOLD_MS = 3_000 export const SMOKE_PROMPT = "This is a provider connectivity smoke test, not a code review. Call attempt_completion once with the result: Provider smoke completed. Do not read files, use other tools, delegate, or judge code." diff --git a/scripts/solheim-smoke/driver.mts b/scripts/solheim-smoke/driver.mts index df10cd5566..d08a97c104 100644 --- a/scripts/solheim-smoke/driver.mts +++ b/scripts/solheim-smoke/driver.mts @@ -8,7 +8,13 @@ const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..") const out = process.env.SOLHEIM_SMOKE_OUT_DIR if (!out) throw new Error("Set SOLHEIM_SMOKE_OUT_DIR to a scratch directory") await fs.mkdir(out, { recursive: true }) -const runtime = createHostRuntime({ root, artifactsDir: out }) +const runtime = createHostRuntime({ + root, + artifactsDir: out, + // The runner module focuses the Zoo Code sidebar so the recording shows + // the chat panel. Its run() never settles; the driver owns teardown. + testsPath: path.join(root, "scripts", "solheim-smoke", "sidebar-runner.cjs"), +}) try { const verdict = await runSmoke(runtime, process.env.SOLHEIM_API_KEY ?? "") console.log(`Solheim provider smoke: ${verdict.code}`) diff --git a/scripts/solheim-smoke/run.test.ts b/scripts/solheim-smoke/run.test.ts index 9be74b3d75..d27a0bb229 100644 --- a/scripts/solheim-smoke/run.test.ts +++ b/scripts/solheim-smoke/run.test.ts @@ -1,6 +1,7 @@ import assert from "node:assert/strict" import { describe, it } from "node:test" import { RooCodeEventName, TaskCommandName, type TaskCommand } from "@roo-code/types" +import { SMOKE_TEARDOWN_HOLD_MS } from "./config.ts" import { runSmoke, smokeExitCode, type SmokeRuntime, type SmokeVerdict } from "./run.ts" type Scenario = @@ -164,6 +165,9 @@ describe("provider smoke orchestration", () => { const verdict = await runSmoke(f.runtime, "private-test-key") assert.equal(verdict.code, "OK") assert.equal(smokeExitCode(verdict), 0) + // The teardown hold runs once after the completion message; no other + // pause fires in this scenario, so the duration is exactly the hold. + assert.equal(verdict.durationMs, SMOKE_TEARDOWN_HOLD_MS) assert.deepEqual(verdict.gates, { extensionActivated: true, ipcConnected: true, @@ -210,7 +214,8 @@ describe("provider smoke orchestration", () => { const f = fixture("success", { stuck: true }) const verdict = await runSmoke(f.runtime, "private-test-key") assert.equal(verdict.code, "TEARDOWN_FAILED") - assert.equal(verdict.durationMs, 15_000) + // The 3 second completion hold plus three 5 second termination windows. + assert.equal(verdict.durationMs, 15_000 + SMOKE_TEARDOWN_HOLD_MS) assert.deepEqual(f.signals, ["SIGTERM", "SIGKILL"]) assert.equal(verdict.gates.childExited, false) assert.equal(f.cleaned, false) diff --git a/scripts/solheim-smoke/run.ts b/scripts/solheim-smoke/run.ts index 8cbf1e9fb5..b2aed5c0e9 100644 --- a/scripts/solheim-smoke/run.ts +++ b/scripts/solheim-smoke/run.ts @@ -1,6 +1,6 @@ import { runAgentFixture, type AgentRuntime } from "../agent-evals/runner.ts" import { getFixture } from "../agent-evals/registry.ts" -import { SOLHEIM_BASE_URL, SOLHEIM_MODEL } from "./config.ts" +import { SOLHEIM_BASE_URL, SOLHEIM_MODEL, SMOKE_TEARDOWN_HOLD_MS } from "./config.ts" import { collectEvidence, smokeCode, type SmokeCode, type SmokeEvidence } from "./evidence.ts" import { SMOKE_FIXTURE_NAME, ensureSmokeFixtureRegistered } from "./fixture.ts" @@ -45,6 +45,7 @@ export async function runSmoke(runtime: SmokeRuntime, apiKey: string): Promise new Promise((resolve) => setTimeout(resolve, ms)) + +async function focusSidebarWhenReady() { + const extension = vscode.extensions.getExtension(EXTENSION_ID) + if (!extension) throw new Error("Zoo Code extension not found") + const api = extension.isActive ? extension.exports : await extension.activate() + await vscode.commands.executeCommand(FOCUS_COMMAND) + const deadline = Date.now() + READY_TIMEOUT_MS + while (!api.isReady()) { + if (Date.now() >= deadline) throw new Error("Zoo Code webview did not launch in time") + await delay(POLL_INTERVAL_MS) + } +} + +exports.run = async () => { + try { + await focusSidebarWhenReady() + } catch (error) { + // Focus is best effort for the recording. A rejected run() would end + // the test host and cut the smoke short, so log and stay pending. + console.warn("[solheim-smoke] sidebar focus failed:", error && error.message ? error.message : error) + } + await new Promise(() => {}) +}