Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 11 additions & 4 deletions evals/harness.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3155,16 +3155,22 @@ export class EvalHost {
if (remaining === 0) continue;
const observerDeadline = AbortSignal.timeout(remaining);
let sessions: readonly OwnedProgressSession[];
let active: Awaited<ReturnType<OpenCodeClient["session"]["active"]>>;
const observerSignal = AbortSignal.any([
...(this.signal ? [this.signal] : []),
observerDeadline,
]);
try {
sessions = await this.ownedProgressSessions(
sessionId,
messages,
AbortSignal.any([
...(this.signal ? [this.signal] : []),
observerDeadline,
]),
observerSignal,
options.observation.metadata,
);
active = await this.nativeRequest(
(options) => this.client.session.active(options),
observerSignal,
);
} catch (error) {
accountPollElapsed();
if (this.signal?.aborted && error === this.signal.reason) throw error;
Expand Down Expand Up @@ -3215,6 +3221,7 @@ export class EvalHost {
);
const busy =
delivery?.kind === "pending" ||
sessions.some((session) => Object.hasOwn(active, session.id)) ||
ownedPending.length > 0 ||
ownedMessages.some(
(entry) =>
Expand Down
88 changes: 71 additions & 17 deletions tests/eval-native-terminal.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import { executeStudy } from "../evals/study-runner.js";
import {
fixtureModel,
messagePage,
type NativeHandler,
nativeAssistant,
nativeClient,
nativeHost,
Expand Down Expand Up @@ -402,7 +403,9 @@ test("old V2 fields stay absent and strict new fields do not accept extra data",
type FixtureOptions = {
messages: (poll: number) => SessionMessageInfo[];
prior?: SessionMessageInfo[];
child?: "failed" | "running" | "read-failure";
child?: "failed" | "running" | "read-failure" | "empty";
active?: (poll: number) => Record<string, { type: "running" }>;
timeoutMs?: number;
pendingPost?: boolean;
cancel?: "before" | "after";
outcomeOnly?: boolean;
Expand All @@ -422,7 +425,7 @@ async function observeWait(options: FixtureOptions) {
parentID: "ses_root",
agent: "flow-reviewer",
});
const client = nativeClient(async ({ url, init }) => {
const handler: NativeHandler = async ({ url, init }) => {
const path = url.pathname;
if (path.endsWith("/model")) return new Response(null, { status: 204 });
if (path.endsWith("/command") || path.endsWith("/prompt")) {
Expand Down Expand Up @@ -453,23 +456,26 @@ async function observeWait(options: FixtureOptions) {
if (options.child === "read-failure")
throw new Error("Fixture child read failed");
return messagePage(
options.child === "failed"
? [idle("failed", "msg_child_idle")]
: [
nativeAssistant("msg_child", {
content: [
nativeTool("call_read", "read", {
status: "running",
input: {},
metadata: {},
}),
],
}),
],
options.child === "empty"
? []
: options.child === "failed"
? [idle("failed", "msg_child_idle")]
: [
nativeAssistant("msg_child", {
content: [
nativeTool("call_read", "read", {
status: "running",
input: {},
metadata: {},
}),
],
}),
],
);
}
throw new Error("Unexpected fixture endpoint");
});
};
const client = nativeClient(handler, () => options.active?.(poll) ?? {});
const clock = spyOn(Date, "now").mockImplementation(() => now);
const sleep = spyOn(Bun, "sleep").mockImplementation(async () => {
now += 2000;
Expand All @@ -483,7 +489,7 @@ async function observeWait(options: FixtureOptions) {
result = await evaluationPhase("host", "command-aborted", true, () =>
host.runCommand("ses_root", "flow-plan", "fixture", fixtureModel, {
quietMs: 0,
timeoutMs: 8000,
timeoutMs: options.timeoutMs ?? 8000,
stalledMs: 4000,
}),
).catch((error: unknown) => error);
Expand Down Expand Up @@ -1131,3 +1137,51 @@ test("study runner grades an allowed question stop instead of recording native f
await rm(directory, { recursive: true, force: true });
}
});

for (const scope of ["root", "child"] as const)
test(`native active ${scope} without an exported unfinished assistant cannot settle quiet`, async () => {
const result = await observeWait({
messages: (poll) => (poll < 3 ? [] : [idle("succeeded")]),
...(scope === "child" ? { child: "empty" as const } : {}),
active: (poll) =>
poll < 4
? {
[scope === "root" ? "ses_root" : "ses_child"]: {
type: "running",
},
}
: {},
timeoutMs: 20000,
});
expect(result.result).toBe("quiet");
expect(result.now).toBeGreaterThanOrEqual(8000);
expect(result.aborts).toBe(0);
});

test("unrelated native activity does not hold the owned command open", async () => {
const result = await observeWait({
messages: () => [idle("succeeded")],
active: () => ({ ses_other: { type: "running" } }),
});
expect(result.result).toBe("quiet");
expect(result.now).toBeLessThanOrEqual(6000);
});

test("unexported native activity still reaches the command deadline", async () => {
const result = await observeWait({
messages: () => [],
active: () => ({ ses_root: { type: "running" } }),
});
expect(result.result).toBeInstanceOf(EvaluationPhaseError);
expect(result.aborts).toBe(1);
});

test("native activity read failure cannot establish quiet", async () => {
const result = await observeWait({
messages: () => [],
active: () => {
throw new Error("Native activity unavailable");
},
});
expect(result.result).toBeInstanceOf(EvaluationPhaseError);
});
9 changes: 8 additions & 1 deletion tests/fixtures/native-eval-client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -90,14 +90,19 @@ export function sessionReply(data: SessionInfo): Response {
return Response.json({ data });
}

export function nativeClient(handler: NativeHandler): OpenCodeClient {
export function nativeClient(
handler: NativeHandler,
active: () => Record<string, { type: "running" }> = () => ({}),
): OpenCodeClient {
const transport: typeof fetch = Object.assign(
async (input: Parameters<typeof fetch>[0], init?: RequestInit) => {
const url = new URL(input instanceof Request ? input.url : String(input));
if (url.origin !== "http://native-eval.invalid")
throw new Error(
`Fixture refused transport outside its origin ${url.origin}`,
);
if (url.pathname === "/api/session/active")
return Response.json({ data: active() });
return handler({ url, method: init?.method ?? "GET", init: init ?? {} });
},
{ preconnect: globalThis.fetch.preconnect },
Expand Down Expand Up @@ -150,6 +155,8 @@ export function nativeHttpClient(handler: NativeHandler): {
hostname: "127.0.0.1",
port: 0,
async fetch(request) {
if (new URL(request.url).pathname === "/api/session/active")
return Response.json({ data: {} });
const body = request.method === "GET" ? undefined : await request.text();
return handler({
url: new URL(request.url),
Expand Down
25 changes: 16 additions & 9 deletions tests/fixtures/opencode-v2-scripted-provider.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,15 +29,17 @@ export function localScriptedProvider() {
requests.push({
model: body.model,
tools,
scenario: text?.includes("FLOW_SMOKE_RETRY")
? "retry"
: text?.includes("FLOW_SMOKE_PROVIDER_ERROR")
? "error"
: text?.includes("FLOW_SMOKE_DELAY")
? "delay"
: text?.includes("FLOW_SMOKE_ACTIONS")
? "actions"
: "plain",
scenario: text?.includes("FLOW_SMOKE_QUIET")
? "quiet"
: text?.includes("FLOW_SMOKE_RETRY")
? "retry"
: text?.includes("FLOW_SMOKE_PROVIDER_ERROR")
? "error"
: text?.includes("FLOW_SMOKE_DELAY")
? "delay"
: text?.includes("FLOW_SMOKE_ACTIONS")
? "actions"
: "plain",
});
if (
text?.includes("FLOW_SMOKE_RETRY") &&
Expand All @@ -47,6 +49,11 @@ export function localScriptedProvider() {
{ error: { message: "Local transient error" } },
{ status: 503 },
);
if (
text?.includes("FLOW_SMOKE_QUIET") &&
requests.filter((entry) => entry.scenario === "quiet").length === 2
)
await Bun.sleep(6000);
if (text?.includes("FLOW_SMOKE_PROVIDER_ERROR"))
return Response.json(
{ error: { message: "Local scripted provider error" } },
Expand Down
58 changes: 58 additions & 0 deletions tests/live-opencode-smoke.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,10 @@ import type {
} from "@opencode/client";
import { currentBunToolchain } from "../evals/bun-toolchain.js";
import {
EvalHost,
packPlugin,
preparePackageCache,
runSessionRequest,
terminateChildProcessTree,
} from "../evals/harness.js";
import {
Expand Down Expand Up @@ -546,6 +548,62 @@ test.skipIf(!LIVE)(
).toMatchObject({ status: "ok" });
}

const quietSession = await primaryClient.session.create(
{
title: "Native active wait control",
agent: "build",
model: { providerID: "flow-smoke", id: "catalog-model" },
location: { directory: project },
},
request(),
);
const quietHost = Reflect.construct(EvalHost, [project, root]);
Reflect.set(quietHost, "nativeClient", primaryClient);
const observation = await Reflect.get(
quietHost,
"commandObservation",
).call(quietHost, quietSession.id);
await runSessionRequest({
url: "local",
body: null,
onRejected() {},
waitOwnsCancellation: true,
post: (_url, _body, options) =>
primaryClient.session.prompt(
{
sessionID: quietSession.id,
text: `FLOW_SMOKE_QUIET FLOW_SMOKE_ACTIONS ${JSON.stringify([{ tool: "flow_status", input: { request: { view: "compact" } } }])}`,
},
{ signal: options.signal },
),
wait: (pending) =>
Reflect.get(quietHost, "waitForQuiet").call(
quietHost,
quietSession.id,
{ quietMs: 1500, timeoutMs: 30000, request: pending, observation },
),
});
expect(
Object.hasOwn(
await primaryClient.session.active(request()),
quietSession.id,
),
).toBe(false);
const quietMessages = await listNativeMessages(
primaryClient,
quietSession.id,
);
expect(
quietMessages.some(
(message) =>
message.type === "idle" && message.outcome === "succeeded",
),
).toBe(true);
expect(JSON.stringify(quietMessages)).toContain("Local smoke complete");
expect(
provider.requests.filter((entry) => entry.scenario === "quiet"),
).toHaveLength(2);

try {
const errorSession = await client.session.create(
{
Expand Down
Loading