From 756e77ae5e297fb1e9ba7d83fc91af8d1ea7a6a6 Mon Sep 17 00:00:00 2001 From: Douwe de Vries Date: Sun, 25 Oct 2026 15:38:14 +0100 Subject: [PATCH 1/4] test(evals): expose native OAuth readiness gaps --- tests/native-auth-readiness.test.ts | 213 ++++++++++++++++++++++++++++ 1 file changed, 213 insertions(+) create mode 100644 tests/native-auth-readiness.test.ts diff --git a/tests/native-auth-readiness.test.ts b/tests/native-auth-readiness.test.ts new file mode 100644 index 00000000..35e83979 --- /dev/null +++ b/tests/native-auth-readiness.test.ts @@ -0,0 +1,213 @@ +import { afterEach, expect, test } from "bun:test"; +import { chmod, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { Context } from "@opencode/plugin/promise/plugin"; +import { EvaluationPhaseError } from "../evals/failure-origin.js"; +import { EvalHost } from "../evals/harness.js"; +import { registerNativeAccountClient } from "../evals/native-auth/client-plugin.js"; +import { PLACEHOLDER_REFRESH } from "../evals/native-auth/protocol.js"; + +const roots: string[] = []; +const servers: ReturnType[] = []; +const disposers: Array<() => Promise> = []; +afterEach(async () => { + for (const dispose of disposers.splice(0)) await dispose(); + for (const server of servers.splice(0)) await server.stop(true); + for (const root of roots.splice(0)) + await rm(root, { recursive: true, force: true }); +}); +const route = { + providerID: "openai", + integrationID: "openai", + credentialID: "synthetic-selected", + methodID: "chatgpt-browser", + label: "Synthetic", +}; +const metadata = { + accountID: "synthetic-account", + nested: { tier: "fixture" }, +}; +const value = { + type: "oauth", + methodID: route.methodID, + access: "synthetic-bearer", + refresh: PLACEHOLDER_REFRESH, + expires: 4102444800000, + metadata, +}; +const profile = { providerID: "openai", id: "gpt-6.1-sol" }; +async function fixture() { + const root = await mkdtemp(join(tmpdir(), "flow-readiness-")); + roots.push(root); + await chmod(root, 0o700); + const server = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + fetch(request) { + return Response.json( + new URL(request.url).pathname === "/resolve" ? { route, value } : {}, + ); + }, + }); + servers.push(server); + const configurationPath = join(root, "configuration.json"); + await writeFile( + configurationPath, + JSON.stringify({ + connection: { + version: 1, + url: server.url.href, + capability: "a".repeat(64), + accounts: [{ route, metadata }], + }, + lease: "b".repeat(64), + models: ["openai/gpt-6.1-sol"], + }), + { mode: 0o600 }, + ); + const hooks = new Map Promise>(); + const provider = { + id: "openai", + integrationID: "openai", + activation: "enabled", + package: "@opencode/ai/providers/openai", + settings: { + baseURL: "https://chatgpt.com/backend-api/codex", + transport: "http", + }, + headers: { "chatgpt-account-id": metadata.accountID }, + }; + const active = { + type: "credential", + id: route.credentialID, + label: route.label, + method: "oauth", + }; + const ctx = { + options: { configurationPath }, + provider: { + get: async () => ({ data: provider }), + list: async () => ({ data: [provider] }), + }, + model: { + list: async () => ({ + data: [ + { ...profile, modelID: profile.id, enabled: true, variants: [] }, + ], + }), + }, + integration: { + get: async () => ({ + data: { + id: "openai", + methods: [{ type: "oauth", id: route.methodID }], + }, + }), + connection: { active: async () => active, resolve: async () => value }, + transform: async (callback: (editor: unknown) => void) => { + callback({ + method: { + list: () => [{ type: "oauth", id: route.methodID }], + update: () => {}, + }, + }); + return { dispose: async () => {} }; + }, + }, + rpc: { register: async () => ({ dispose: async () => {} }) }, + session: { + hook: async ( + name: string, + callback: (input: unknown) => Promise, + ) => { + hooks.set(name, callback); + return { + dispose: async () => { + hooks.delete(name); + }, + }; + }, + }, + } as unknown as Context; + disposers.push(await registerNativeAccountClient(ctx)); + return { + async guard( + url: string, + account = metadata.accountID, + name = "http.request", + ) { + const hook = hooks.get(name); + if (!hook) throw new Error("Missing guard"); + await hook({ + model: profile, + request: new Request(url, { + headers: { "chatgpt-account-id": account }, + }), + }); + }, + }; +} +for (const [url, account] of [ + ["https://api.openai.com/v1/responses", metadata.accountID], + [ + "https://chatgpt.com/backend-api/codex-elsewhere/responses", + metadata.accountID, + ], + [ + "https://chatgpt.com/backend-api/codex/responses", + "synthetic-other-account", + ], +] as const) + test(`legacy final HTTP refuses mismatched target or account ${url} ${account}`, async () => { + const f = await fixture(); + await expect(f.guard(url, account)).rejects.toThrow( + "Native account authentication failed.", + ); + }); +test("legacy final HTTP permits Codex base descendants and refuses a late WebSocket route", async () => { + const f = await fixture(); + await f.guard("https://chatgpt.com/backend-api/codex/responses?synthetic=1"); + await expect( + f.guard( + "https://chatgpt.com/backend-api/codex/responses", + metadata.accountID, + "experimental.ws.handshake", + ), + ).rejects.toThrow("Native account authentication failed."); +}); +test("probe readiness fails as host infrastructure before creating a session or invoking dispatch", async () => { + const host = Object.create(EvalHost.prototype) as EvalHost; + const failure = new EvaluationPhaseError( + { + origin: "host", + code: "native-auth-not-ready", + detail: "Native account authentication is not ready (route-unsettled).", + retryable: false, + }, + undefined, + ); + let sessions = 0, + callbacks = 0; + Object.assign(host, { + project: "/synthetic/project", + verifyArtifacts: async () => {}, + nativeAuthHost: { + requireReady: async () => { + throw failure; + }, + }, + createSession: async () => { + sessions++; + throw failure; + }, + }); + await expect( + host.probeModel("openai/gpt-6.1-sol", { + onDispatch: () => { + callbacks++; + }, + }), + ).rejects.toBe(failure); + expect({ sessions, callbacks }).toEqual({ sessions: 0, callbacks: 0 }); +}); From 7b4d38f12f96691287cb0cd46ed37502485876e4 Mon Sep 17 00:00:00 2001 From: Douwe de Vries Date: Sun, 25 Oct 2026 15:51:10 +0100 Subject: [PATCH 2/4] fix(evals): wait for native OAuth routing before dispatch --- evals/harness.ts | 39 ++ evals/native-auth/client-plugin.ts | 416 +++++++++++++++- evals/native-auth/host.ts | 102 +++- evals/native-auth/protocol.ts | 168 +++++++ scripts/canary-run.ts | 7 + tests/canary-v2-native-auth.test.ts | 78 ++- tests/native-auth-readiness.test.ts | 723 +++++++++++++++++++++++++--- tests/native-auth.test.ts | 11 +- 8 files changed, 1466 insertions(+), 78 deletions(-) diff --git a/evals/harness.ts b/evals/harness.ts index 423e330b..5c74978a 100644 --- a/evals/harness.ts +++ b/evals/harness.ts @@ -2494,6 +2494,26 @@ export class EvalHost { ); } } + if (host.nativeAuthHost && options.nativeAuthentication) { + await host.nativeAuthHost.requireReady(host.client, { + directory: project, + models: [ + ...options.nativeAuthentication.models.map((model) => + nativeModel(model), + ), + ...(options.reviewer?.model + ? [ + nativeModel( + options.reviewer.model, + options.reviewer.variant, + ), + ] + : []), + ], + timeoutMs: Math.min(15000, deadline - Date.now()), + ...(options.signal ? { signal: options.signal } : {}), + }); + } if (options.requestBudget) { const receipt = JSON.parse( await readFile(join(scratch, "budget-ready.json"), "utf8"), @@ -2683,6 +2703,13 @@ export class EvalHost { }, ): Promise { await this.verifyArtifacts(); + checkCancellation(this.signal); + await this.nativeAuthHost?.requireReady(this.client, { + directory: this.project, + models: [nativeModel(model, options?.variant)], + ...(this.signal ? { signal: this.signal } : {}), + }); + checkCancellation(this.signal); const sessionId = await this.createSession(`flow-eval probe ${model}`); let usage = normalizeStudyUsage({ tokens: {}, costUsd: null }); try { @@ -2810,6 +2837,12 @@ export class EvalHost { checkCancellation(this.signal); await this.verifyArtifacts(); checkCancellation(this.signal); + await this.nativeAuthHost?.requireReady(this.client, { + directory: this.project, + models: [nativeModel(model, options.variant)], + ...(this.signal ? { signal: this.signal } : {}), + }); + checkCancellation(this.signal); this.escalationQuestions.delete(sessionId); await consumePaidDispatch({ model, kind: "command" }); await this.nativeRequest((request) => @@ -2859,6 +2892,12 @@ export class EvalHost { checkCancellation(this.signal); await this.verifyArtifacts(); checkCancellation(this.signal); + await this.nativeAuthHost?.requireReady(this.client, { + directory: this.project, + models: [nativeModel(model, options.variant)], + ...(this.signal ? { signal: this.signal } : {}), + }); + checkCancellation(this.signal); this.escalationQuestions.delete(sessionId); await consumePaidDispatch({ model, kind: "prompt" }); await this.nativeRequest((request) => diff --git a/evals/native-auth/client-plugin.ts b/evals/native-auth/client-plugin.ts index f2a17cad..1ea8778e 100644 --- a/evals/native-auth/client-plugin.ts +++ b/evals/native-auth/client-plugin.ts @@ -4,16 +4,145 @@ import { dirname, isAbsolute } from "node:path"; import { Credential, Integration, Plugin } from "@opencode/plugin"; import type { Context } from "@opencode/plugin/promise/plugin"; import { + boundedNativeAuth, type NativeAccount, + NativeAccountReadinessRpc, + type OAuthValue, object, PLACEHOLDER_REFRESH, parseChildOptions, + parseOAuth, + parseReadinessInput, parseResolved, + type ReadinessModel, + type ReadinessReason, + type ReadinessResult, refuse, responseJSON, sameMetadata, + UNRESOLVED_ACCESS, } from "./protocol.js"; +class ReadinessFailure extends Error { + readonly reason: ReadinessReason; + readonly pending: boolean; + constructor(reason: ReadinessReason, pending = false) { + super(reason); + this.reason = reason; + this.pending = pending; + } +} +const codex = "https://chatgpt.com/backend-api/codex"; +function legacy(account: NativeAccount): boolean { + const route = account.route; + return ( + route.providerID === "openai" && + route.integrationID === "openai" && + ["chatgpt-browser", "chatgpt-headless"].includes(route.methodID) + ); +} +function routeProjection( + account: NativeAccount, + provider: unknown, + model: unknown, + variant?: string, +) { + if (!object(provider) || !object(model) || !Array.isArray(model.variants)) + throw new ReadinessFailure("metadata-unavailable", true); + const variants = model.variants.filter( + (candidate) => object(candidate) && candidate.id === variant, + ); + if (variant !== undefined && variant !== "default" && variants.length !== 1) + throw new ReadinessFailure("profile-unapproved"); + const selected = + variant === undefined || variant === "default" ? undefined : variants[0]; + if (!legacy(account)) return { modelID: model.modelID, variant }; + const accountID = account.metadata?.accountID; + if (typeof accountID !== "string" || !accountID) + throw new ReadinessFailure("credential-invalid"); + for (const layer of [ + provider, + model, + selected, + { settings: account.metadata }, + ]) { + if (layer === undefined) continue; + if (!object(layer)) + throw new ReadinessFailure("metadata-unavailable", true); + if (layer.settings !== undefined) { + if (!object(layer.settings)) + throw new ReadinessFailure("metadata-unavailable", true); + const settings = layer.settings; + if ( + ["apiKey", "authToken", "accessToken", "auth"].some( + (key) => settings[key] !== undefined && settings[key] !== "", + ) + ) + throw new ReadinessFailure("route-unsettled", true); + } + if ( + layer.headers !== undefined && + (!object(layer.headers) || + Object.entries(layer.headers).some( + ([key, value]) => + typeof value !== "string" || + key.toLowerCase() === "authorization" || + (key.toLowerCase() === "chatgpt-account-id" && value !== accountID), + )) + ) + throw new ReadinessFailure("route-unsettled", true); + } + const settings = (layer: Record | undefined) => + layer?.settings === undefined + ? {} + : (layer.settings as Record); + const effective = { + ...settings(provider), + ...settings(model), + ...settings(selected), + }; + const headers = ( + layer: Record | undefined, + ): Record => + layer?.headers === undefined + ? {} + : (layer.headers as Record); + const merged = { + ...headers(provider), + ...headers(model), + ...headers(selected), + }; + const header = Object.entries(merged).filter( + ([key]) => key.toLowerCase() === "chatgpt-account-id", + ); + if ( + provider.package !== "@opencode/ai/providers/openai" || + (model.package !== undefined && model.package !== provider.package) || + model.modelID !== model.id || + provider.settings === undefined || + settings(provider).transport !== "http" || + settings(provider).baseURL !== codex || + effective.baseURL !== codex || + (account.metadata?.baseURL !== undefined && + account.metadata.baseURL !== codex) || + (effective.transport !== undefined && effective.transport !== "http") || + !Object.entries(headers(provider)).some( + ([key, value]) => + key.toLowerCase() === "chatgpt-account-id" && value === accountID, + ) || + header.length === 0 || + header.some(([, value]) => value !== accountID) + ) + throw new ReadinessFailure("route-unsettled", true); + return { + package: provider.package, + modelID: model.modelID, + baseURL: effective.baseURL, + transport: settings(provider).transport, + account: accountID, + }; +} + export async function registerNativeAccountClient( ctx: Context, ): Promise<() => Promise> { @@ -64,7 +193,11 @@ export async function registerNativeAccountClient( ) refuse(); }; - const request = async (path: "check" | "resolve", account: NativeAccount) => { + const request = async ( + path: "check" | "resolve", + account: NativeAccount, + signal?: AbortSignal, + ) => { await ensureSelected(account); return responseJSON( await fetch(new URL(path, options.connection.url), { @@ -80,6 +213,7 @@ export async function registerNativeAccountClient( }), signal: AbortSignal.any([ controller.signal, + ...(signal ? [signal] : []), AbortSignal.timeout(15000), ]), redirect: "error", @@ -134,31 +268,287 @@ export async function registerNativeAccountClient( }); } }); - const guard = async (input: { model: { providerID: string } }) => { + + const selection = async (account: NativeAccount, signal: AbortSignal) => { + const route = account.route; + const provider = ( + await boundedNativeAuth(signal, () => + ctx.provider.get({ providerID: route.providerID }), + ) + ).data; + const available = ( + await boundedNativeAuth(signal, () => ctx.provider.list()) + ).data; + if ( + provider.id !== route.providerID || + !(["auto", "enabled"] as const).some( + (activation) => activation === provider.activation, + ) || + provider.integrationID !== route.integrationID || + !available.some((candidate) => candidate.id === route.providerID) + ) + throw new ReadinessFailure("selection-changed"); + const integration = ( + await boundedNativeAuth(signal, () => + ctx.integration.get({ integrationID: route.integrationID }), + ) + ).data; + if ( + integration.id !== route.integrationID || + !integration.methods.some( + (method) => method.type === "oauth" && method.id === route.methodID, + ) + ) + throw new ReadinessFailure("selection-changed"); + const active = await boundedNativeAuth(signal, () => + ctx.integration.connection.active(route.integrationID), + ); + if ( + active?.type !== "credential" || + active.id !== route.credentialID || + active.label !== route.label || + active.method !== "oauth" || + active.status + ) + throw new ReadinessFailure("selection-changed"); + return { provider, active }; + }; + const usable = (raw: unknown, account: NativeAccount): OAuthValue => { + let value: OAuthValue; + try { + value = parseOAuth(raw); + } catch { + throw new ReadinessFailure("credential-invalid"); + } + if ( + value.methodID !== account.route.methodID || + !sameMetadata(value.metadata, account.metadata) + ) + throw new ReadinessFailure("credential-invalid"); + if (!value.access || value.access === UNRESOLVED_ACCESS) + throw new ReadinessFailure("credential-unresolved", true); + if (value.expires <= Date.now()) + throw new ReadinessFailure("credential-stale", true); + return value; + }; + const source = async (account: NativeAccount, signal: AbortSignal) => { + try { + const resolved = parseResolved( + await boundedNativeAuth(signal, () => + request("resolve", account, signal), + ), + ); + if ( + !sameMetadata(resolved.route, account.route) || + !sameMetadata(resolved.value.metadata, account.metadata) + ) + refuse(); + return usable(resolved.value, account); + } catch { + throw new ReadinessFailure("source-unavailable"); + } + }; + const sample = async ( + models: readonly ReadinessModel[], + signal: AbortSignal, + ) => { + const values: Array<{ value: OAuthValue; account: NativeAccount }> = []; + for (const providerID of new Set(models.map((model) => model.providerID))) { + const account = options.connection.accounts.find( + (candidate) => candidate.route.providerID === providerID, + ); + if (!account) throw new ReadinessFailure("profile-unapproved"); + const initial = await selection(account, signal); + await source(account, signal); + usable( + await boundedNativeAuth(signal, () => + ctx.integration.connection.resolve(initial.active), + ), + account, + ); + const projection = async () => { + const { provider } = await selection(account, signal); + const inventory = ( + await boundedNativeAuth(signal, () => ctx.model.list()) + ).data; + return models + .filter((model) => model.providerID === providerID) + .map((profile) => { + const matches = inventory.filter( + (model) => + model.providerID === providerID && model.id === profile.id, + ); + const model = matches[0]; + if ( + matches.length !== 1 || + model?.enabled !== true || + typeof model.modelID !== "string" || + !model.modelID + ) + throw new ReadinessFailure("metadata-unavailable", true); + return routeProjection(account, provider, model, profile.variant); + }); + }; + const before = await projection(); + const current = await selection(account, signal); + values.push({ + value: usable( + await boundedNativeAuth(signal, () => + ctx.integration.connection.resolve(current.active), + ), + account, + ), + account, + }); + if (!sameMetadata(before, await projection())) + throw new ReadinessFailure("route-unsettled", true); + values.push({ value: await source(account, signal), account }); + await selection(account, signal); + } + for (const { value, account } of values) usable(value, account); + if (signal.aborted) refuse(); + }; + const readiness = await ctx.rpc.register(NativeAccountReadinessRpc, { + ready: async (raw, call): Promise => { + let timer: ReturnType | undefined; + let reason: ReadinessReason = "readiness-unavailable"; + const timeout = new AbortController(); + try { + const input = parseReadinessInput(raw); + if ( + input.models.some( + (model) => + !options.models.includes(`${model.providerID}/${model.id}`), + ) + ) + throw new ReadinessFailure("profile-unapproved"); + const deadline = Date.now() + input.timeoutMs; + timer = setTimeout(() => timeout.abort(), input.timeoutMs); + const signal = AbortSignal.any([ + controller.signal, + call.signal, + timeout.signal, + ]); + for (;;) { + try { + await sample(input.models, signal); + if (signal.aborted || Date.now() >= deadline) refuse(); + return { version: 1, state: "ready" }; + } catch (error) { + if (call.signal.aborted || controller.signal.aborted) + throw new ReadinessFailure("readiness-cancelled"); + if (timeout.signal.aborted || Date.now() >= deadline) + throw new ReadinessFailure("readiness-deadline"); + if (!(error instanceof ReadinessFailure) || !error.pending) + throw error; + await boundedNativeAuth( + signal, + () => + new Promise((resolve) => { + const wake = setTimeout( + done, + Math.min(20, Math.max(1, deadline - Date.now())), + ); + function done() { + clearTimeout(wake); + signal.removeEventListener("abort", done); + resolve(); + } + signal.addEventListener("abort", done, { once: true }); + }), + ); + } + } + } catch (error) { + if (error instanceof ReadinessFailure) reason = error.reason; + else if (call.signal.aborted || controller.signal.aborted) + reason = "readiness-cancelled"; + else if (timeout.signal.aborted) reason = "readiness-deadline"; + controller.abort(); + return { version: 1, state: "not-ready", reason }; + } finally { + clearTimeout(timer); + } + }, + }); + const guard = async ( + input: { + model: { providerID: string; id: string; variant?: string | undefined }; + request?: Request; + }, + websocket = false, + ) => { try { + if (controller.signal.aborted) refuse(); const account = options.connection.accounts.find( (candidate) => candidate.route.providerID === input.model.providerID, ); if (!account) refuse(); - const provider = ( - await ctx.provider.get({ providerID: account.route.providerID }) - ).data; - if ( - provider.activation === "disabled" || - provider.integrationID !== account.route.integrationID - ) - refuse(); - await request("check", account); + const check = async (signal?: AbortSignal) => { + const read = async () => { + const provider = ( + await ctx.provider.get({ providerID: account.route.providerID }) + ).data; + if ( + provider.activation === "disabled" || + provider.integrationID !== account.route.integrationID + ) + refuse(); + await request("check", account, signal); + }; + if (signal) await boundedNativeAuth(signal, read); + else await read(); + }; + if (legacy(account)) { + if (websocket || !input.request) refuse(); + const timeout = new AbortController(); + const timer = setTimeout(() => timeout.abort(), 15000); + const signal = AbortSignal.any([controller.signal, timeout.signal]); + try { + await check(signal); + await sample( + [ + { + providerID: input.model.providerID, + id: input.model.id, + ...(input.model.variant === undefined + ? {} + : { variant: input.model.variant }), + }, + ], + signal, + ); + const url = new URL(input.request.url); + if ( + url.origin !== "https://chatgpt.com" || + !( + url.pathname === "/backend-api/codex" || + url.pathname.startsWith("/backend-api/codex/") + ) || + input.request.headers.get("chatgpt-account-id") !== + account.metadata?.accountID || + signal.aborted + ) + refuse(); + } finally { + clearTimeout(timer); + } + } else await check(); + if (controller.signal.aborted) refuse(); } catch { refuse(); } }; - const http = await ctx.session.hook("http.request", guard); - const websocket = await ctx.session.hook("experimental.ws.handshake", guard); + const http = await ctx.session.hook("http.request", (input) => guard(input)); + const websocket = await ctx.session.hook( + "experimental.ws.handshake", + (input) => guard(input, true), + ); let disposed: Promise | undefined; return () => { disposed ??= (async () => { controller.abort(); + await readiness.dispose(); await registration.dispose(); await http.dispose(); await websocket.dispose(); diff --git a/evals/native-auth/host.ts b/evals/native-auth/host.ts index 6f9429e0..38fdd1e3 100644 --- a/evals/native-auth/host.ts +++ b/evals/native-auth/host.ts @@ -11,16 +11,24 @@ import { } from "node:fs/promises"; import { dirname, isAbsolute, join } from "node:path"; import { isDeepStrictEqual } from "node:util"; -import type { CredentialEntry } from "@opencode/client"; +import type { CredentialEntry, OpenCodeClient } from "@opencode/client"; import { canonicalSha256 } from "../canonical-json.js"; +import { EvaluationPhaseError } from "../failure-origin.js"; import ClientPlugin from "./client-plugin.js"; import { + boundedNativeAuth, capability, type NativeAccountConnection, + NativeAccountReadinessRpc, object, PLACEHOLDER_REFRESH, + parseApprovedModels, parseConnection, + parseReadinessInput, + parseReadinessResult, providersForModels, + type ReadinessModel, + type ReadinessReason, refuse, responseJSON, UNRESOLVED_ACCESS, @@ -43,6 +51,15 @@ export type NativeAuthHost = { }): Promise; }; artifactHashes: { entrySha256: string; sourceSha256: string }; + requireReady( + client: Pick, + input: { + directory: string; + models: readonly ReadinessModel[]; + timeoutMs?: number; + signal?: AbortSignal; + }, + ): Promise; dispose(): Promise; }; export type NativeAuthentication = { @@ -189,18 +206,16 @@ export function connectNativeAccountSource( }): Promise { if (closed || options.signal?.aborted || !isAbsolute(options.directory)) refuse(); - const providers = providersForModels(options.models); + const models = parseApprovedModels(options.models); + const providers = providersForModels(models); const accounts = connection.accounts.filter((account) => providers.includes(account.route.providerID), ); if (accounts.length !== providers.length) refuse(); - const acquired = await request( - "acquire", - { models: options.models }, - options.signal, - ); + const acquired = await request("acquire", { models }, options.signal); if (!object(acquired)) refuse(); const lease = capability(acquired.lease); + const lifetime = new AbortController(); let ownedDirectory = false; let done: Promise | undefined; const configurationPath = @@ -208,6 +223,7 @@ export function connectNativeAccountSource( join(options.directory, "configuration.json"); const dispose = () => { done ??= (async () => { + lifetime.abort(); try { await request("release", { lease }); } finally { @@ -230,7 +246,11 @@ export function connectNativeAccountSource( await verifyNativeAuthClient(artifact); await writeFile( configurationPath, - JSON.stringify({ connection: { ...connection, accounts }, lease }), + JSON.stringify({ + connection: { ...connection, accounts }, + lease, + models, + }), { mode: 0o600, flag: "wx" }, ); const entries = [ @@ -257,6 +277,72 @@ export function connectNativeAccountSource( ).values(), ]; const attachment: NativeAuthHost = { + requireReady: async (client, input) => { + let reason: ReadinessReason = "readiness-unavailable"; + let timer: ReturnType | undefined; + const timeout = new AbortController(); + try { + input.signal?.throwIfAborted(); + if (lifetime.signal.aborted || closed) refuse(); + const timeoutMs = Math.min(15000, input.timeoutMs ?? 15000); + if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) { + reason = "readiness-deadline"; + refuse(); + } + const request = parseReadinessInput({ + models: input.models, + timeoutMs: Math.floor(timeoutMs), + }); + if ( + request.models.some( + (model) => + !models.includes(`${model.providerID}/${model.id}`), + ) + ) { + reason = "profile-unapproved"; + refuse(); + } + const deadline = Date.now() + request.timeoutMs; + timer = setTimeout(() => timeout.abort(), request.timeoutMs); + const signal = AbortSignal.any([ + lifetime.signal, + timeout.signal, + ...(input.signal ? [input.signal] : []), + ]); + const result = parseReadinessResult( + await boundedNativeAuth(signal, () => + client.rpc(NativeAccountReadinessRpc).ready(request, { + location: { directory: input.directory }, + signal, + }), + ), + ); + input.signal?.throwIfAborted(); + if (Date.now() >= deadline) { + reason = "readiness-deadline"; + refuse(); + } + if (result.state === "not-ready") { + reason = result.reason; + refuse(); + } + } catch { + if (timeout.signal.aborted) reason = "readiness-deadline"; + lifetime.abort(); + input.signal?.throwIfAborted(); + throw new EvaluationPhaseError( + { + origin: "host", + code: "native-auth-not-ready", + detail: `Native account authentication is not ready (${reason}).`, + retryable: false, + }, + undefined, + ); + } finally { + clearTimeout(timer); + } + }, plugin: { package: artifact.pluginDirectory, options: { configurationPath }, diff --git a/evals/native-auth/protocol.ts b/evals/native-auth/protocol.ts index 0729d64e..5c6f9ee2 100644 --- a/evals/native-auth/protocol.ts +++ b/evals/native-auth/protocol.ts @@ -25,6 +25,7 @@ export type NativeAccountConnection = { export type ChildOptions = { connection: NativeAccountConnection; lease: string; + models: readonly string[]; }; export class NativeAuthError extends Error { @@ -230,6 +231,7 @@ export function parseChildOptions(value: unknown): ChildOptions { return { connection: parseConnection(value.connection), lease: capability(value.lease), + models: parseApprovedModels(value.models), }; } export function parseSelection(value: unknown): { @@ -289,3 +291,169 @@ export const NativeAccountRpc = Rpc.define({ }, events: {}, }); + +export type ReadinessModel = { + providerID: string; + id: string; + variant?: string; +}; +export const READINESS_REASONS = [ + "profile-unapproved", + "selection-changed", + "source-unavailable", + "credential-invalid", + "credential-unresolved", + "credential-stale", + "route-unsettled", + "metadata-unavailable", + "readiness-deadline", + "readiness-cancelled", + "readiness-unavailable", +] as const; +export type ReadinessReason = (typeof READINESS_REASONS)[number]; +export type ReadinessResult = + | { version: 1; state: "ready" } + | { version: 1; state: "not-ready"; reason: ReadinessReason }; +function keys(value: Record, allowed: readonly string[]) { + if (Object.keys(value).some((key) => !allowed.includes(key))) refuse(); +} +export function parseApprovedModels(value: unknown): readonly string[] { + if ( + !Array.isArray(value) || + value.length === 0 || + value.length > 256 || + !value.every( + (model) => + typeof model === "string" && + model.length <= 512 && + /^[^\s/]+\/[^\s]+$/.test(model), + ) + ) + refuse(); + providersForModels(value); + return Object.freeze([...new Set(value)]); +} +export function parseReadinessInput(value: unknown): { + models: ReadinessModel[]; + timeoutMs: number; +} { + if (!object(value)) refuse(); + keys(value, ["models", "timeoutMs"]); + if ( + !Array.isArray(value.models) || + value.models.length === 0 || + value.models.length > 256 || + typeof value.timeoutMs !== "number" || + !Number.isInteger(value.timeoutMs) || + value.timeoutMs <= 0 || + value.timeoutMs > 15000 + ) + refuse(); + const models = value.models.map((model: unknown): ReadinessModel => { + if (!object(model)) refuse(); + keys(model, ["providerID", "id", "variant"]); + if ( + !identifier(model.providerID) || + typeof model.id !== "string" || + !model.id || + model.id.length > 512 || + /\s/.test(model.id) || + (model.variant !== undefined && + (typeof model.variant !== "string" || model.variant.length > 256)) + ) + refuse(); + return { + providerID: model.providerID, + id: model.id, + ...(model.variant === undefined ? {} : { variant: model.variant }), + }; + }); + return { models, timeoutMs: value.timeoutMs }; +} +export function parseReadinessResult(value: unknown): ReadinessResult { + if (!object(value) || value.version !== 1) refuse(); + if (value.state === "ready") { + keys(value, ["version", "state"]); + return { version: 1, state: "ready" }; + } + keys(value, ["version", "state", "reason"]); + if ( + value.state !== "not-ready" || + !READINESS_REASONS.some((reason) => reason === value.reason) + ) + refuse(); + return { + version: 1, + state: "not-ready", + reason: value.reason as ReadinessReason, + }; +} +export async function boundedNativeAuth( + signal: AbortSignal, + operation: () => Promise, +): Promise { + if (signal.aborted) refuse(); + let onAbort = () => {}; + const aborted = new Promise((_resolve, reject) => { + onAbort = () => reject(new NativeAuthError()); + signal.addEventListener("abort", onAbort, { once: true }); + }); + try { + const result = await Promise.race([operation(), aborted]); + if (signal.aborted) refuse(); + return result; + } finally { + signal.removeEventListener("abort", onAbort); + } +} +export const NativeAccountReadinessRpc = Rpc.define({ + id: "flow.native-account-readiness", + methods: { + ready: { + input: { + type: "object", + properties: { + models: { + type: "array", + minItems: 1, + maxItems: 256, + items: { + type: "object", + properties: { + providerID: { type: "string" }, + id: { type: "string" }, + variant: { type: "string" }, + }, + required: ["providerID", "id"], + additionalProperties: false, + }, + }, + timeoutMs: { type: "integer", minimum: 1, maximum: 15000 }, + }, + required: ["models", "timeoutMs"], + additionalProperties: false, + }, + output: { + oneOf: [ + { + type: "object", + properties: { version: { const: 1 }, state: { const: "ready" } }, + required: ["version", "state"], + additionalProperties: false, + }, + { + type: "object", + properties: { + version: { const: 1 }, + state: { const: "not-ready" }, + reason: { type: "string", enum: READINESS_REASONS }, + }, + required: ["version", "state", "reason"], + additionalProperties: false, + }, + ], + }, + }, + }, + events: {}, +}); diff --git a/scripts/canary-run.ts b/scripts/canary-run.ts index 4189e10b..0fa9f8a7 100644 --- a/scripts/canary-run.ts +++ b/scripts/canary-run.ts @@ -398,6 +398,13 @@ async function launchCanary( { signal }, ); } + if (attachment) + await attachment.requireReady(client, { + directory, + models: [requestedModel], + timeoutMs: Math.min(15000, deadline - Date.now()), + signal, + }); phase = "execution"; const manager = await client.session.create( { diff --git a/tests/canary-v2-native-auth.test.ts b/tests/canary-v2-native-auth.test.ts index d41192b0..14e36d49 100644 --- a/tests/canary-v2-native-auth.test.ts +++ b/tests/canary-v2-native-auth.test.ts @@ -15,11 +15,16 @@ import { import { tmpdir } from "node:os"; import { join } from "node:path"; import { pathToFileURL } from "node:url"; -import { type CredentialEntry, OpenCode } from "@opencode/client"; +import { + type CredentialEntry, + OpenCode, + type OpenCodeClient, +} from "@opencode/client"; import { Service } from "@opencode/client/service"; import { canonicalJson } from "../evals/canonical-json.js"; import { startNativeAccountBroker } from "../evals/native-auth/broker.js"; import { + connectNativeAccountSource, nativeAuthSourceSha256, stageNativeAuthClient, } from "../evals/native-auth/host.js"; @@ -371,6 +376,77 @@ test("charged source acquisition failure is generic, remains consumed, and remov "Native authentication diagnostics omitted.\n", ); }); +test("readiness failure before canary generation stays charged once and cannot repeat", async () => { + const f = await fixture(); + let launches = 0, + generations = 0, + releases = 0; + const relay = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + fetch(request) { + if (new URL(request.url).pathname === "/acquire") + return Response.json({ lease: "b".repeat(64) }); + releases++; + return Response.json({}); + }, + }); + try { + await writeFile( + f.source, + JSON.stringify({ ...connection, url: relay.url.href }), + { mode: 0o600 }, + ); + const launch = async ( + _directory: string, + model: string, + _prompt: string, + _prepared: unknown, + configuration?: NativeAccountConnection, + ) => { + launches++; + if (!configuration) throw new Error("Missing synthetic configuration"); + const source = connectNativeAccountSource(configuration); + const auth = await source.prepareHost({ + directory: join(f.root, "readiness-host"), + models: [model], + }); + try { + const client = { + rpc: () => ({ + ready: async () => ({ + version: 1, + state: "not-ready", + reason: "route-unsettled", + }), + }), + } as unknown as Pick; + await auth.requireReady(client, { + directory: "/synthetic/project", + models: [{ providerID: "fake", id: "model" }], + }); + generations++; + return 0; + } finally { + await auth.dispose(); + await source.dispose(); + } + }; + await expect(runPaidCanary(f.input, launch)).rejects.toThrow( + "Native-auth canary execution failed during execution.", + ); + expect({ + launches, + generations, + releases, + consumed: (await paidRunStatus(f.root)).consumed, + }).toEqual({ launches: 1, generations: 0, releases: 1, consumed: 1 }); + await expect(runPaidCanary(f.input, launch)).rejects.toThrow("exhausted"); + expect(launches).toBe(1); + } finally { + await relay.stop(true); + } +}); for (const failure of ["setup", "version", "startup", "release"] as const) test(`charged auth ${failure} failure stops its host and disposes private attachment independently`, async () => { const f = await fixture(); diff --git a/tests/native-auth-readiness.test.ts b/tests/native-auth-readiness.test.ts index 35e83979..d968b842 100644 --- a/tests/native-auth-readiness.test.ts +++ b/tests/native-auth-readiness.test.ts @@ -1,12 +1,20 @@ -import { afterEach, expect, test } from "bun:test"; -import { chmod, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { afterEach, expect, spyOn, test } from "bun:test"; +import { chmod, mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import type { OpenCodeClient } from "@opencode/client"; import type { Context } from "@opencode/plugin/promise/plugin"; +import { CampaignCancelled } from "../evals/campaign-stop.js"; import { EvaluationPhaseError } from "../evals/failure-origin.js"; import { EvalHost } from "../evals/harness.js"; import { registerNativeAccountClient } from "../evals/native-auth/client-plugin.js"; -import { PLACEHOLDER_REFRESH } from "../evals/native-auth/protocol.js"; +import { connectNativeAccountSource } from "../evals/native-auth/host.js"; +import { + PLACEHOLDER_REFRESH, + type ReadinessModel, + UNRESOLVED_ACCESS, +} from "../evals/native-auth/protocol.js"; +import * as paidBudget from "../scripts/paid-budget.js"; const roots: string[] = []; const servers: ReturnType[] = []; @@ -37,85 +45,162 @@ const value = { metadata, }; const profile = { providerID: "openai", id: "gpt-6.1-sol" }; -async function fixture() { +async function fixture(generic = false, additionalModels: string[] = []) { const root = await mkdtemp(join(tmpdir(), "flow-readiness-")); roots.push(root); await chmod(root, 0o700); + const pin = { + ...route, + ...(generic + ? { providerID: "synthetic-provider", methodID: "synthetic-method" } + : {}), + }; + const pinnedMetadata = generic ? { nested: { tier: "fixture" } } : metadata; + const selectedProfile = { ...profile, providerID: pin.providerID }; + const state: { + provider: Record; + active: Record; + models: Array>; + value: Record; + sourceValue: Record; + sourceRoute: Record; + sourceAlive: boolean; + leaseValid: boolean; + resolves: number; + sourceResolves: number; + activeRead?: () => Promise; + modelRead?: () => Promise; + resolve?: () => Promise; + } = { + provider: { + id: pin.providerID, + integrationID: pin.integrationID, + activation: "enabled", + package: "@opencode/ai/providers/openai", + settings: { + baseURL: "https://chatgpt.com/backend-api/codex", + transport: "http", + }, + headers: { "chatgpt-account-id": metadata.accountID }, + }, + active: { + type: "credential", + id: pin.credentialID, + label: pin.label, + method: "oauth", + }, + models: [ + { + ...selectedProfile, + modelID: generic ? "native-mapped-model" : profile.id, + enabled: true, + variants: [], + }, + ], + value: { ...value, methodID: pin.methodID, metadata: pinnedMetadata }, + sourceValue: { ...value, methodID: pin.methodID, metadata: pinnedMetadata }, + sourceRoute: pin, + sourceAlive: true, + leaseValid: true, + resolves: 0, + sourceResolves: 0, + }; + if (generic) { + state.provider.package = "synthetic-native-package"; + state.provider.settings = { transport: "websocket" }; + delete state.provider.headers; + } const server = Bun.serve({ hostname: "127.0.0.1", port: 0, - fetch(request) { - return Response.json( - new URL(request.url).pathname === "/resolve" ? { route, value } : {}, - ); + async fetch(request) { + const path = new URL(request.url).pathname; + if (path === "/acquire") return Response.json({ lease: "b".repeat(64) }); + if (path === "/release") return Response.json({}); + if (!state.sourceAlive || !state.leaseValid) + return new Response("private-source-error-sentinel", { status: 503 }); + if (path === "/resolve") { + state.sourceResolves++; + return Response.json({ + route: state.sourceRoute, + value: state.sourceValue, + }); + } + return Response.json({}); }, }); servers.push(server); - const configurationPath = join(root, "configuration.json"); - await writeFile( - configurationPath, - JSON.stringify({ - connection: { - version: 1, - url: server.url.href, - capability: "a".repeat(64), - accounts: [{ route, metadata }], - }, - lease: "b".repeat(64), - models: ["openai/gpt-6.1-sol"], - }), - { mode: 0o600 }, - ); + const source = connectNativeAccountSource({ + version: 1, + url: server.url.href, + capability: "a".repeat(64), + accounts: [{ route: pin, metadata: pinnedMetadata }], + }); + const auth = await source.prepareHost({ + directory: join(root, "attachment"), + models: [ + `${selectedProfile.providerID}/${selectedProfile.id}`, + ...additionalModels, + ], + }); const hooks = new Map Promise>(); - const provider = { - id: "openai", - integrationID: "openai", - activation: "enabled", - package: "@opencode/ai/providers/openai", - settings: { - baseURL: "https://chatgpt.com/backend-api/codex", - transport: "http", - }, - headers: { "chatgpt-account-id": metadata.accountID }, - }; - const active = { - type: "credential", - id: route.credentialID, - label: route.label, - method: "oauth", - }; + let handler: + | ((raw: unknown, call: { signal: AbortSignal }) => Promise) + | undefined; + let rpcDisposed = false; const ctx = { - options: { configurationPath }, + options: auth.plugin.options, provider: { - get: async () => ({ data: provider }), - list: async () => ({ data: [provider] }), + get: async () => ({ data: state.provider }), + list: async () => ({ data: [state.provider] }), }, model: { - list: async () => ({ - data: [ - { ...profile, modelID: profile.id, enabled: true, variants: [] }, - ], - }), + list: async () => { + await state.modelRead?.(); + return { data: state.models }; + }, }, integration: { get: async () => ({ data: { - id: "openai", - methods: [{ type: "oauth", id: route.methodID }], + id: pin.integrationID, + methods: [{ type: "oauth", id: pin.methodID }], }, }), - connection: { active: async () => active, resolve: async () => value }, + connection: { + active: async () => { + await state.activeRead?.(); + return state.active; + }, + resolve: async () => { + state.resolves++; + return state.resolve ? state.resolve() : state.value; + }, + }, transform: async (callback: (editor: unknown) => void) => { callback({ method: { - list: () => [{ type: "oauth", id: route.methodID }], + list: () => [{ type: "oauth", id: pin.methodID }], update: () => {}, }, }); return { dispose: async () => {} }; }, }, - rpc: { register: async () => ({ dispose: async () => {} }) }, + rpc: { + register: async ( + definition: { id: string }, + handlers: { ready: typeof handler }, + ) => { + expect(definition.id).toBe("flow.native-account-readiness"); + handler = handlers.ready; + return { + dispose: async () => { + rpcDisposed = true; + }, + }; + }, + }, session: { hook: async ( name: string, @@ -130,8 +215,47 @@ async function fixture() { }, }, } as unknown as Context; - disposers.push(await registerNativeAccountClient(ctx)); + const dispose = await registerNativeAccountClient(ctx); + disposers.push(async () => { + await dispose(); + await auth.dispose(); + await source.dispose(); + }); + const client = { + rpc: () => ({ + ready: (raw: unknown, request: { signal: AbortSignal }) => { + if (!handler) throw new Error("Missing readiness handler"); + return handler(raw, request); + }, + }), + } as unknown as Pick; return { + auth, + client, + state, + profile: selectedProfile, + dispose, + get rpcDisposed() { + return rpcDisposed; + }, + ready( + input: { + timeoutMs?: number; + signal?: AbortSignal; + models?: ReadinessModel[]; + } = {}, + ) { + return auth.requireReady(client, { + directory: "/synthetic/project", + models: input.models ?? [selectedProfile], + timeoutMs: input.timeoutMs ?? 250, + ...(input.signal ? { signal: input.signal } : {}), + }); + }, + raw(raw: unknown, signal = new AbortController().signal) { + if (!handler) throw new Error("Missing readiness handler"); + return handler(raw, { signal }); + }, async guard( url: string, account = metadata.accountID, @@ -140,7 +264,7 @@ async function fixture() { const hook = hooks.get(name); if (!hook) throw new Error("Missing guard"); await hook({ - model: profile, + model: selectedProfile, request: new Request(url, { headers: { "chatgpt-account-id": account }, }), @@ -191,6 +315,7 @@ test("probe readiness fails as host infrastructure before creating a session or callbacks = 0; Object.assign(host, { project: "/synthetic/project", + nativeClient: {}, verifyArtifacts: async () => {}, nativeAuthHost: { requireReady: async () => { @@ -211,3 +336,491 @@ test("probe readiness fails as host infrastructure before creating a session or ).rejects.toBe(failure); expect({ sessions, callbacks }).toEqual({ sessions: 0, callbacks: 0 }); }); + +function deferred() { + let resolve: (value: T) => void = () => {}; + const promise = new Promise((done) => { + resolve = done; + }); + return { promise, resolve }; +} +async function hostFailure(operation: Promise) { + const error: unknown = await operation.catch((error) => error); + expect(error).toBeInstanceOf(EvaluationPhaseError); + if (!(error instanceof EvaluationPhaseError)) + throw new Error("Missing host infrastructure failure"); + expect(error.failure.origin).toBe("host"); + expect(error.failure.code).toBe("native-auth-not-ready"); + expect(error.failure.retryable).toBe(false); + expect(error.cause).toBeUndefined(); + expect(JSON.stringify(error)).not.toContain("private-"); + return error; +} +test("production adapter waits for observed route settlement with valid cached credentials", async () => { + const f = await fixture(); + f.state.provider.settings = { transport: "http" }; + delete f.state.provider.headers; + const observed = deferred(); + f.state.modelRead = async () => { + observed.resolve(); + }; + let admitted = false; + const ready = f.ready().then(() => { + admitted = true; + }); + await observed.promise; + expect({ admitted, resolutions: f.state.resolves > 0 }).toEqual({ + admitted: false, + resolutions: true, + }); + f.state.provider.settings = { + transport: "http", + baseURL: "https://chatgpt.com/backend-api/codex", + }; + f.state.provider.headers = { "ChatGPT-Account-ID": metadata.accountID }; + await ready; + expect(admitted).toBe(true); + expect(f.state.sourceResolves).toBeGreaterThanOrEqual(2); +}); +test("future token below five minutes and rotating bearer values remain usable", async () => { + const f = await fixture(); + f.state.value.expires = Date.now() + 30000; + f.state.sourceValue.expires = Date.now() + 30000; + f.state.resolve = async () => ({ + ...f.state.value, + access: `synthetic-rotation-${f.state.resolves}`, + }); + await f.ready(); + expect(f.state.resolves).toBe(2); + expect(f.state.sourceResolves).toBe(2); +}); +for (const change of [ + "source", + "lease", + "source-id", + "source-metadata", + "id", + "label", + "category", + "integration", + "method", + "metadata", + "placeholder", +] as const) + test(`fresh cached access cannot admit after ${change} conflict`, async () => { + const f = await fixture(); + if (change === "source") f.state.sourceAlive = false; + if (change === "lease") f.state.leaseValid = false; + if (change === "source-id") + f.state.sourceRoute = { ...f.state.sourceRoute, credentialID: "other" }; + if (change === "source-metadata") + f.state.sourceValue.metadata = { ...metadata, nested: { tier: "other" } }; + if (change === "id") f.state.active.id = "other"; + if (change === "label") f.state.active.label = "other"; + if (change === "category") f.state.active.method = "key"; + if (change === "integration") f.state.provider.integrationID = "other"; + if (change === "method") f.state.value.methodID = "other"; + if (change === "metadata") + f.state.value.metadata = { ...metadata, nested: { tier: "other" } }; + if (change === "placeholder") + f.state.value.refresh = "private-unowned-refresh"; + await hostFailure(f.ready()); + }); +test("selection changed during awaited model metadata is refused without adoption", async () => { + const f = await fixture(); + f.state.modelRead = async () => { + f.state.active.label = "other"; + }; + await hostFailure(f.ready()); + expect(f.state.resolves).toBe(1); +}); +for (const variant of [undefined, "default", "high"]) + test(`exact native variant semantics ${variant}`, async () => { + const f = await fixture(); + f.state.models[0] = { + ...f.state.models[0], + variants: [ + { id: "default", settings: { baseURL: "https://wrong.invalid" } }, + { + id: "high", + settings: { baseURL: "https://chatgpt.com/backend-api/codex" }, + }, + ], + }; + await f.ready({ + models: [{ ...profile, ...(variant === undefined ? {} : { variant }) }], + }); + }); +for (const change of [ + "variant", + "variant-route", + "wire-id", + "model-disabled", + "duplicate", + "model-missing", + "unapproved", + "package", + "transport", + "backend", + "header", + "duplicate-header", + "authorization", + "key", + "null-settings", +] as const) + test(`readiness refuses conflicting model projection ${change}`, async () => { + const f = await fixture(); + const model = f.state.models[0]; + if (!model) throw new Error("Missing model"); + let models: ReadinessModel[] = [profile]; + if (change === "variant") models = [{ ...profile, variant: "missing" }]; + if (change === "variant-route") { + models = [{ ...profile, variant: "high" }]; + model.variants = [ + { id: "high", settings: { baseURL: "https://wrong.invalid" } }, + ]; + } + if (change === "wire-id") model.modelID = "remapped"; + if (change === "model-disabled") model.enabled = false; + if (change === "duplicate") f.state.models.push({ ...model }); + if (change === "model-missing") f.state.models = []; + if (change === "unapproved") models = [{ ...profile, id: "other" }]; + if (change === "package") model.package = "other"; + if (change === "transport") model.settings = { transport: "websocket" }; + if (change === "backend") + model.settings = { baseURL: "https://api.openai.com/v1" }; + if (change === "header") model.headers = { "CHATGPT-ACCOUNT-ID": "other" }; + if (change === "duplicate-header") + model.headers = { + "chatgpt-account-id": metadata.accountID, + "CHATGPT-ACCOUNT-ID": "other", + }; + if (change === "authorization") + model.headers = { Authorization: "private-auth-sentinel" }; + if (change === "key") model.settings = { apiKey: "private-key-sentinel" }; + if (change === "null-settings") model.settings = null; + await hostFailure(f.ready({ models, timeoutMs: 35 })); + }); +test("generic OAuth preserves native mapping, nested metadata and WebSocket guard", async () => { + const f = await fixture(true); + await f.ready(); + await f.guard( + "https://synthetic.invalid/other", + "not-an-account", + "experimental.ws.handshake", + ); + expect(f.state.resolves).toBe(2); +}); +for (const change of ["unresolved", "expired", "expires-during-read"] as const) + test(`credential expiry is checked at completion ${change}`, async () => { + const f = await fixture(); + if (change === "unresolved") f.state.value.access = UNRESOLVED_ACCESS; + if (change === "expired") f.state.value.expires = Date.now() - 1; + if (change === "expires-during-read") + f.state.modelRead = async () => { + f.state.value.expires = 0; + }; + await hostFailure(f.ready({ timeoutMs: 35 })); + }); +test("source metadata remains fresh after a successful startup observation and at HTTP", async () => { + const f = await fixture(); + await f.ready(); + f.state.sourceValue.metadata = { ...metadata, nested: { tier: "changed" } }; + await expect( + f.guard("https://chatgpt.com/backend-api/codex/responses"), + ).rejects.toThrow("Native account authentication failed."); + await hostFailure(f.ready()); +}); +test("bounded native resolve cancellation propagates campaign reason and rejects late admission", async () => { + const f = await fixture(); + const started = deferred(), + resolved = deferred(); + f.state.resolve = async () => { + started.resolve(); + return resolved.promise; + }; + const controller = new AbortController(); + const pending = f.ready({ signal: controller.signal }); + await started.promise; + const reason = new CampaignCancelled(130); + controller.abort(reason); + await expect(pending).rejects.toBe(reason); + resolved.resolve(f.state.value); + await Promise.resolve(); + await hostFailure(f.ready()); +}); +test("deadline bounds a suspended native resolve; disposal unregisters RPC and hooks", async () => { + const f = await fixture(); + f.state.resolve = () => new Promise(() => {}); + const start = Date.now(); + await hostFailure(f.ready({ timeoutMs: 25 })); + expect(Date.now() - start).toBeLessThan(1000); + await f.dispose(); + expect(f.rpcDisposed).toBe(true); +}); +for (const result of [ + { version: 1, state: "ready", credential: "private-value" }, + { version: 2, state: "ready" }, + { version: 1, state: "not-ready", reason: "private-value" }, + null, +]) + test(`host strictly parses RPC result ${JSON.stringify(result)}`, async () => { + const f = await fixture(); + const client = { + rpc: () => ({ ready: async () => result }), + } as unknown as Pick; + const error = await hostFailure( + f.auth.requireReady(client, { + directory: "/synthetic/project", + models: [profile], + }), + ); + expect(error.message).not.toContain("private-value"); + }); +test("missing RPC and raw SDK errors remain safe infrastructure failures", async () => { + for (const missing of [false, true]) { + const f = await fixture(); + if (missing) + Reflect.set(f.client, "rpc", () => { + throw new Error("private-rpc-secret"); + }); + else + f.state.resolve = async () => { + throw new Error("private-sdk-secret"); + }; + const error = await hostFailure(f.ready()); + expect(error.stack).not.toContain("private-"); + } +}); +test("strict child input rejects authority fields without returning observations", async () => { + const f = await fixture(); + expect( + await f.raw({ + models: [profile], + timeoutMs: 10, + accountID: "private-account", + }), + ).toEqual({ + version: 1, + state: "not-ready", + reason: "readiness-unavailable", + }); +}); +for (const kind of ["probe", "command", "prompt"] as const) + test(`failed ${kind} readiness precedes fake accounting and generation`, async () => { + const f = await fixture(); + f.state.sourceAlive = false; + const host = Object.create(EvalHost.prototype) as EvalHost; + let accounting = 0, + sessions = 0, + generation = 0, + callbacks = 0; + const accountingSpy = spyOn( + paidBudget, + "consumePaidDispatch", + ).mockImplementation(async () => { + accounting++; + }); + Object.assign(host, { + project: "/synthetic/project", + nativeClient: f.client, + nativeAuthHost: f.auth, + verifyArtifacts: async () => {}, + escalationQuestions: new Map(), + createSession: async () => { + sessions++; + return "synthetic-session"; + }, + nativeRequest: async () => { + generation++; + throw new Error("Generation prohibited"); + }, + deleteSession: async () => {}, + }); + try { + await hostFailure( + kind === "probe" + ? host.probeModel("openai/gpt-6.1-sol", { + onDispatch: () => { + callbacks++; + }, + }) + : kind === "command" + ? host.runCommand( + "synthetic-session", + "flow", + "", + "openai/gpt-6.1-sol", + ) + : host.runPrompt( + "synthetic-session", + "synthetic", + "openai/gpt-6.1-sol", + ), + ); + expect({ accounting, sessions, generation, callbacks }).toEqual({ + accounting: 0, + sessions: 0, + generation: 0, + callbacks: 0, + }); + } finally { + accountingSpy.mockRestore(); + } + }); + +test("all startup profiles are checked and a reviewer variant cannot escape the gate", async () => { + const f = await fixture(false, ["openai/reviewer"]); + f.state.models.push({ + ...profile, + id: "reviewer", + modelID: "reviewer", + enabled: true, + variants: [{ id: "high", settings: { baseURL: "https://wrong.invalid" } }], + }); + await hostFailure( + f.ready({ + models: [ + profile, + { providerID: "openai", id: "reviewer", variant: "high" }, + ], + timeoutMs: 35, + }), + ); +}); +test("cancellation immediately after readiness prevents fake dispatch accounting", async () => { + const f = await fixture(); + const controller = new AbortController(); + const reason = new CampaignCancelled(130); + const host = Object.create(EvalHost.prototype) as EvalHost; + let accounting = 0, + generation = 0; + const accountingSpy = spyOn( + paidBudget, + "consumePaidDispatch", + ).mockImplementation(async () => { + accounting++; + }); + Object.assign(host, { + project: "/synthetic/project", + nativeClient: f.client, + signal: controller.signal, + nativeAuthHost: { + requireReady: async (...args: Parameters) => { + await f.auth.requireReady(...args); + controller.abort(reason); + }, + }, + verifyArtifacts: async () => {}, + escalationQuestions: new Map(), + nativeRequest: async () => { + generation++; + }, + }); + try { + await expect( + host.runPrompt("synthetic", "synthetic", "openai/gpt-6.1-sol"), + ).rejects.toBe(reason); + expect({ accounting, generation }).toEqual({ + accounting: 0, + generation: 0, + }); + } finally { + accountingSpy.mockRestore(); + } +}); +test("plugin disposal while resolving is bounded and does not admit a late value", async () => { + const f = await fixture(); + const started = deferred(), + resolved = deferred(); + f.state.resolve = async () => { + started.resolve(); + return resolved.promise; + }; + const pending = f.ready(); + await started.promise; + await f.dispose(); + const error = await hostFailure(pending); + expect(error.message).toBe( + "Native account authentication is not ready (readiness-cancelled).", + ); + resolved.resolve(f.state.value); +}); + +test("closing source resolve catches a metadata change during the final model read", async () => { + const f = await fixture(); + let reads = 0; + f.state.modelRead = async () => { + if (++reads === 2) + f.state.sourceValue.metadata = { + ...metadata, + nested: { tier: "changed-at-close" }, + }; + }; + const error = await hostFailure(f.ready()); + expect(error.message).toBe( + "Native account authentication is not ready (source-unavailable).", + ); +}); +test("late HTTP refusal remains consumed after a successful pre-account gate", async () => { + const f = await fixture(); + const host = Object.create(EvalHost.prototype) as EvalHost; + let accounting = 0, + callbacks = 0, + transports = 0; + const accountingSpy = spyOn( + paidBudget, + "consumePaidDispatch", + ).mockImplementation(async () => { + accounting++; + }); + Object.assign(host, { + project: "/synthetic/project", + nativeAuthHost: f.auth, + serverLog: "", + nativeClient: { + ...f.client, + session: { + switchModel: async () => {}, + prompt: async () => { + await f.guard("https://api.openai.com/v1/responses"); + transports++; + }, + }, + }, + verifyArtifacts: async () => {}, + createSession: async () => "synthetic-session", + deleteSession: async () => {}, + nativeRequest: async (operation: (request: object) => Promise) => + operation({}), + }); + try { + expect( + await host.probeModel("openai/gpt-6.1-sol", { + onDispatch: () => { + callbacks++; + }, + }), + ).toBe("Native account authentication failed."); + expect({ accounting, callbacks, transports }).toEqual({ + accounting: 1, + callbacks: 1, + transports: 0, + }); + } finally { + accountingSpy.mockRestore(); + } +}); + +for (const variant of ["high effort", "高"] as const) + test(`parent valid native variant ${variant} remains usable`, async () => { + const { Model } = await import("@opencode/schema/model"); + const ref = Model.Ref.parse(`synthetic-provider/gpt-6.1-sol#${variant}`); + expect(ref.variant).toBe(variant); + const f = await fixture(true); + f.state.models[0] = { + ...f.state.models[0], + variants: [{ id: variant, settings: {} }], + }; + await f.ready({ models: [{ ...ref, variant }] }); + }); diff --git a/tests/native-auth.test.ts b/tests/native-auth.test.ts index a99db663..e60ca8a9 100644 --- a/tests/native-auth.test.ts +++ b/tests/native-auth.test.ts @@ -283,7 +283,11 @@ async function clientFixture() { const path = join(root, "configuration.json"); await writeFile( path, - JSON.stringify({ connection: remote.configuration, lease }), + JSON.stringify({ + connection: remote.configuration, + lease, + models: ["alias/model"], + }), { mode: 0o600 }, ); let active: unknown = { @@ -294,7 +298,9 @@ async function clientFixture() { }; const methods: IntegrationOAuthMethodRegistration[] = []; const hooks = new Map(); + const rpc = rpcCapture(); const ctx = context({ + ...rpc, options: { configurationPath: path }, provider: { get: async () => ({ @@ -413,12 +419,15 @@ for (const conflict of [true, false]) accounts, }, lease, + models: ["alias/model", "alias-two/model"], }), { mode: 0o600 }, ); let transforms = 0; const methods: IntegrationOAuthMethodRegistration[] = []; + const rpc = rpcCapture(); const ctx = context({ + ...rpc, options: { configurationPath: path }, integration: { connection: { From 842300f7d74bc40d94539558cb75a5f465af0451 Mon Sep 17 00:00:00 2001 From: Douwe de Vries Date: Sun, 25 Oct 2026 16:34:31 +0100 Subject: [PATCH 3/4] test(evals): cover native catalog deadline diagnostics --- tests/native-auth-readiness.test.ts | 87 +++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/tests/native-auth-readiness.test.ts b/tests/native-auth-readiness.test.ts index d968b842..0ae44a7f 100644 --- a/tests/native-auth-readiness.test.ts +++ b/tests/native-auth-readiness.test.ts @@ -824,3 +824,90 @@ for (const variant of ["high effort", "高"] as const) }; await f.ready({ models: [{ ...ref, variant }] }); }); + +for (const unavailable of ["missing", "misspelled", "disabled"] as const) + test(`catalog diagnostic ${unavailable} retains zero fake accounting`, async () => { + const f = await fixture(true, ["synthetic-provider/gpt-6.1-sol-typo"]); + const requested = + unavailable === "misspelled" + ? { ...f.profile, id: "gpt-6.1-sol-typo" } + : f.profile; + if (unavailable === "missing") f.state.models = []; + if (unavailable === "disabled") + f.state.models[0] = { ...f.state.models[0], enabled: false }; + const host = Object.create(EvalHost.prototype) as EvalHost; + let accounting = 0, + sessions = 0, + dispatches = 0; + const accountingSpy = spyOn( + paidBudget, + "consumePaidDispatch", + ).mockImplementation(async () => { + accounting++; + }); + Object.assign(host, { + project: "/synthetic/project", + nativeClient: f.client, + nativeAuthHost: { + requireReady: async () => + f.ready({ models: [requested], timeoutMs: 40 }), + }, + verifyArtifacts: async () => {}, + createSession: async () => { + sessions++; + return "synthetic-session"; + }, + }); + try { + const error = await hostFailure( + host.probeModel(`${requested.providerID}/${requested.id}`, { + onDispatch: () => { + dispatches++; + }, + }), + ); + expect({ accounting, sessions, dispatches }).toEqual({ + accounting: 0, + sessions: 0, + dispatches: 0, + }); + expect(error.message).toBe( + "Native account authentication is not ready (readiness-deadline). Check that each requested model is present and enabled in the native catalog; OAuth initialization may also be incomplete.", + ); + } finally { + accountingSpy.mockRestore(); + } + }); +for (const unavailable of ["missing", "disabled"] as const) + test(`catalog diagnostic transient ${unavailable} waits for valid OAuth bootstrap`, async () => { + const f = await fixture(true); + const settled = [...f.state.models]; + f.state.models = + unavailable === "missing" + ? [] + : settled.map((model) => ({ ...model, enabled: false })); + const observed = deferred(), + release = deferred(); + let reads = 0, + accounting = 0; + f.state.modelRead = async () => { + if (++reads === 2) { + observed.resolve(); + await release.promise; + } + }; + const pending = f.ready({ timeoutMs: 1000 }).then(() => { + accounting++; + }); + try { + await observed.promise; + expect({ reads, accounting }).toEqual({ reads: 2, accounting: 0 }); + f.state.models = settled; + release.resolve(); + await pending; + expect(accounting).toBe(1); + } finally { + release.resolve(); + await pending.catch(() => {}); + } + }); From 37eda3d936158eb4e5d4f6dddd8cdb9a747ce474 Mon Sep 17 00:00:00 2001 From: Douwe de Vries Date: Sun, 25 Oct 2026 16:34:31 +0100 Subject: [PATCH 4/4] fix(evals): clarify native readiness deadline diagnostics --- evals/native-auth/host.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evals/native-auth/host.ts b/evals/native-auth/host.ts index 38fdd1e3..74c0ab65 100644 --- a/evals/native-auth/host.ts +++ b/evals/native-auth/host.ts @@ -334,7 +334,7 @@ export function connectNativeAccountSource( { origin: "host", code: "native-auth-not-ready", - detail: `Native account authentication is not ready (${reason}).`, + detail: `Native account authentication is not ready (${reason}).${reason === "readiness-deadline" ? " Check that each requested model is present and enabled in the native catalog; OAuth initialization may also be incomplete." : ""}`, retryable: false, }, undefined,