diff --git a/evals/harness.ts b/evals/harness.ts index 5c74978a..acd84eb5 100644 --- a/evals/harness.ts +++ b/evals/harness.ts @@ -3155,16 +3155,22 @@ export class EvalHost { if (remaining === 0) continue; const observerDeadline = AbortSignal.timeout(remaining); let sessions: readonly OwnedProgressSession[]; + let active: Awaited>; + const observerSignal = AbortSignal.any([ + ...(this.signal ? [this.signal] : []), + observerDeadline, + ]); try { sessions = await this.ownedProgressSessions( sessionId, messages, - AbortSignal.any([ - ...(this.signal ? [this.signal] : []), - observerDeadline, - ]), + observerSignal, options.observation.metadata, ); + active = await this.nativeRequest( + (options) => this.client.session.active(options), + observerSignal, + ); } catch (error) { accountPollElapsed(); if (this.signal?.aborted && error === this.signal.reason) throw error; @@ -3215,6 +3221,7 @@ export class EvalHost { ); const busy = delivery?.kind === "pending" || + sessions.some((session) => Object.hasOwn(active, session.id)) || ownedPending.length > 0 || ownedMessages.some( (entry) => diff --git a/tests/eval-native-terminal.test.ts b/tests/eval-native-terminal.test.ts index 415f419e..4d8da908 100644 --- a/tests/eval-native-terminal.test.ts +++ b/tests/eval-native-terminal.test.ts @@ -29,6 +29,7 @@ import { executeStudy } from "../evals/study-runner.js"; import { fixtureModel, messagePage, + type NativeHandler, nativeAssistant, nativeClient, nativeHost, @@ -402,7 +403,9 @@ test("old V2 fields stay absent and strict new fields do not accept extra data", type FixtureOptions = { messages: (poll: number) => SessionMessageInfo[]; prior?: SessionMessageInfo[]; - child?: "failed" | "running" | "read-failure"; + child?: "failed" | "running" | "read-failure" | "empty"; + active?: (poll: number) => Record; + timeoutMs?: number; pendingPost?: boolean; cancel?: "before" | "after"; outcomeOnly?: boolean; @@ -422,7 +425,7 @@ async function observeWait(options: FixtureOptions) { parentID: "ses_root", agent: "flow-reviewer", }); - const client = nativeClient(async ({ url, init }) => { + const handler: NativeHandler = async ({ url, init }) => { const path = url.pathname; if (path.endsWith("/model")) return new Response(null, { status: 204 }); if (path.endsWith("/command") || path.endsWith("/prompt")) { @@ -453,23 +456,26 @@ async function observeWait(options: FixtureOptions) { if (options.child === "read-failure") throw new Error("Fixture child read failed"); return messagePage( - options.child === "failed" - ? [idle("failed", "msg_child_idle")] - : [ - nativeAssistant("msg_child", { - content: [ - nativeTool("call_read", "read", { - status: "running", - input: {}, - metadata: {}, - }), - ], - }), - ], + options.child === "empty" + ? [] + : options.child === "failed" + ? [idle("failed", "msg_child_idle")] + : [ + nativeAssistant("msg_child", { + content: [ + nativeTool("call_read", "read", { + status: "running", + input: {}, + metadata: {}, + }), + ], + }), + ], ); } throw new Error("Unexpected fixture endpoint"); - }); + }; + const client = nativeClient(handler, () => options.active?.(poll) ?? {}); const clock = spyOn(Date, "now").mockImplementation(() => now); const sleep = spyOn(Bun, "sleep").mockImplementation(async () => { now += 2000; @@ -483,7 +489,7 @@ async function observeWait(options: FixtureOptions) { result = await evaluationPhase("host", "command-aborted", true, () => host.runCommand("ses_root", "flow-plan", "fixture", fixtureModel, { quietMs: 0, - timeoutMs: 8000, + timeoutMs: options.timeoutMs ?? 8000, stalledMs: 4000, }), ).catch((error: unknown) => error); @@ -1131,3 +1137,51 @@ test("study runner grades an allowed question stop instead of recording native f await rm(directory, { recursive: true, force: true }); } }); + +for (const scope of ["root", "child"] as const) + test(`native active ${scope} without an exported unfinished assistant cannot settle quiet`, async () => { + const result = await observeWait({ + messages: (poll) => (poll < 3 ? [] : [idle("succeeded")]), + ...(scope === "child" ? { child: "empty" as const } : {}), + active: (poll) => + poll < 4 + ? { + [scope === "root" ? "ses_root" : "ses_child"]: { + type: "running", + }, + } + : {}, + timeoutMs: 20000, + }); + expect(result.result).toBe("quiet"); + expect(result.now).toBeGreaterThanOrEqual(8000); + expect(result.aborts).toBe(0); + }); + +test("unrelated native activity does not hold the owned command open", async () => { + const result = await observeWait({ + messages: () => [idle("succeeded")], + active: () => ({ ses_other: { type: "running" } }), + }); + expect(result.result).toBe("quiet"); + expect(result.now).toBeLessThanOrEqual(6000); +}); + +test("unexported native activity still reaches the command deadline", async () => { + const result = await observeWait({ + messages: () => [], + active: () => ({ ses_root: { type: "running" } }), + }); + expect(result.result).toBeInstanceOf(EvaluationPhaseError); + expect(result.aborts).toBe(1); +}); + +test("native activity read failure cannot establish quiet", async () => { + const result = await observeWait({ + messages: () => [], + active: () => { + throw new Error("Native activity unavailable"); + }, + }); + expect(result.result).toBeInstanceOf(EvaluationPhaseError); +}); diff --git a/tests/fixtures/native-eval-client.ts b/tests/fixtures/native-eval-client.ts index a93b20c8..86646848 100644 --- a/tests/fixtures/native-eval-client.ts +++ b/tests/fixtures/native-eval-client.ts @@ -90,7 +90,10 @@ export function sessionReply(data: SessionInfo): Response { return Response.json({ data }); } -export function nativeClient(handler: NativeHandler): OpenCodeClient { +export function nativeClient( + handler: NativeHandler, + active: () => Record = () => ({}), +): OpenCodeClient { const transport: typeof fetch = Object.assign( async (input: Parameters[0], init?: RequestInit) => { const url = new URL(input instanceof Request ? input.url : String(input)); @@ -98,6 +101,8 @@ export function nativeClient(handler: NativeHandler): OpenCodeClient { throw new Error( `Fixture refused transport outside its origin ${url.origin}`, ); + if (url.pathname === "/api/session/active") + return Response.json({ data: active() }); return handler({ url, method: init?.method ?? "GET", init: init ?? {} }); }, { preconnect: globalThis.fetch.preconnect }, @@ -150,6 +155,8 @@ export function nativeHttpClient(handler: NativeHandler): { hostname: "127.0.0.1", port: 0, async fetch(request) { + if (new URL(request.url).pathname === "/api/session/active") + return Response.json({ data: {} }); const body = request.method === "GET" ? undefined : await request.text(); return handler({ url: new URL(request.url), diff --git a/tests/fixtures/opencode-v2-scripted-provider.ts b/tests/fixtures/opencode-v2-scripted-provider.ts index 5484bec2..01e41457 100644 --- a/tests/fixtures/opencode-v2-scripted-provider.ts +++ b/tests/fixtures/opencode-v2-scripted-provider.ts @@ -29,15 +29,17 @@ export function localScriptedProvider() { requests.push({ model: body.model, tools, - scenario: text?.includes("FLOW_SMOKE_RETRY") - ? "retry" - : text?.includes("FLOW_SMOKE_PROVIDER_ERROR") - ? "error" - : text?.includes("FLOW_SMOKE_DELAY") - ? "delay" - : text?.includes("FLOW_SMOKE_ACTIONS") - ? "actions" - : "plain", + scenario: text?.includes("FLOW_SMOKE_QUIET") + ? "quiet" + : text?.includes("FLOW_SMOKE_RETRY") + ? "retry" + : text?.includes("FLOW_SMOKE_PROVIDER_ERROR") + ? "error" + : text?.includes("FLOW_SMOKE_DELAY") + ? "delay" + : text?.includes("FLOW_SMOKE_ACTIONS") + ? "actions" + : "plain", }); if ( text?.includes("FLOW_SMOKE_RETRY") && @@ -47,6 +49,11 @@ export function localScriptedProvider() { { error: { message: "Local transient error" } }, { status: 503 }, ); + if ( + text?.includes("FLOW_SMOKE_QUIET") && + requests.filter((entry) => entry.scenario === "quiet").length === 2 + ) + await Bun.sleep(6000); if (text?.includes("FLOW_SMOKE_PROVIDER_ERROR")) return Response.json( { error: { message: "Local scripted provider error" } }, diff --git a/tests/live-opencode-smoke.test.ts b/tests/live-opencode-smoke.test.ts index 3faea1ee..205d95b4 100644 --- a/tests/live-opencode-smoke.test.ts +++ b/tests/live-opencode-smoke.test.ts @@ -22,8 +22,10 @@ import type { } from "@opencode/client"; import { currentBunToolchain } from "../evals/bun-toolchain.js"; import { + EvalHost, packPlugin, preparePackageCache, + runSessionRequest, terminateChildProcessTree, } from "../evals/harness.js"; import { @@ -546,6 +548,62 @@ test.skipIf(!LIVE)( ).toMatchObject({ status: "ok" }); } + const quietSession = await primaryClient.session.create( + { + title: "Native active wait control", + agent: "build", + model: { providerID: "flow-smoke", id: "catalog-model" }, + location: { directory: project }, + }, + request(), + ); + const quietHost = Reflect.construct(EvalHost, [project, root]); + Reflect.set(quietHost, "nativeClient", primaryClient); + const observation = await Reflect.get( + quietHost, + "commandObservation", + ).call(quietHost, quietSession.id); + await runSessionRequest({ + url: "local", + body: null, + onRejected() {}, + waitOwnsCancellation: true, + post: (_url, _body, options) => + primaryClient.session.prompt( + { + sessionID: quietSession.id, + text: `FLOW_SMOKE_QUIET FLOW_SMOKE_ACTIONS ${JSON.stringify([{ tool: "flow_status", input: { request: { view: "compact" } } }])}`, + }, + { signal: options.signal }, + ), + wait: (pending) => + Reflect.get(quietHost, "waitForQuiet").call( + quietHost, + quietSession.id, + { quietMs: 1500, timeoutMs: 30000, request: pending, observation }, + ), + }); + expect( + Object.hasOwn( + await primaryClient.session.active(request()), + quietSession.id, + ), + ).toBe(false); + const quietMessages = await listNativeMessages( + primaryClient, + quietSession.id, + ); + expect( + quietMessages.some( + (message) => + message.type === "idle" && message.outcome === "succeeded", + ), + ).toBe(true); + expect(JSON.stringify(quietMessages)).toContain("Local smoke complete"); + expect( + provider.requests.filter((entry) => entry.scenario === "quiet"), + ).toHaveLength(2); + try { const errorSession = await client.session.create( {