diff --git a/docs-site/src/content/docs/reference/cli/providers-accounts.md b/docs-site/src/content/docs/reference/cli/providers-accounts.md index 1c9cac81800..3f40362c515 100644 --- a/docs-site/src/content/docs/reference/cli/providers-accounts.md +++ b/docs-site/src/content/docs/reference/cli/providers-accounts.md @@ -246,6 +246,12 @@ when account failover is enabled, the request can retry on another eligible acco Google's account verification, then run `ocx login google-antigravity`. Silent token refresh does not clear this verification requirement. If OpenCodex cannot save the quarantine, this adapter exchange preserves the original `403` without another recovery send; the account has not been durably quarantined. If a sibling request cannot be built or admitted for sending, the exchange delivers the original `403` through normal error formatting. An enclosing combo or policy route can still apply its existing fallback rules. +For fetch-based web-search requests, automatic sibling rotation accepts only a complete, bounded +Google error envelope containing `error.details[].reason === "VALIDATION_REQUIRED"`. Verification +wording alone does not rotate. This request-scoped recovery uses the sibling's token and project +without persisting a `needs-reauth` mark; cancellation, committed output, budget exhaustion or an +unavailable sibling preserves the failure. + A proxy that is already running picks up the new credential without a restart: the CLI asks it to reload that one provider from disk, and the request carries no credential of its own. If the running proxy cannot accept that request — most often because it started from a build that predates diff --git a/docs-site/src/content/docs/reference/configuration/providers.md b/docs-site/src/content/docs/reference/configuration/providers.md index ce28f9ffaa3..bfc084cf0ae 100644 --- a/docs-site/src/content/docs/reference/configuration/providers.md +++ b/docs-site/src/content/docs/reference/configuration/providers.md @@ -1019,7 +1019,15 @@ The same main dispatch may also switch once on a 403 when Google's bounded error finds a complete structured `VALIDATION_REQUIRED` reason. The 401 and 403 paths share one sibling attempt per request; an unrelated or incomplete 403 keeps its original error. A rejected sibling, cancellation or exhausted send budget does not cause another upstream send. -Continuations, native Responses passthrough, image and web-search sidecars, and output already sent +The fetch-based web-search loop also rotates once on this structured 403 reason, using the sibling's +OAuth token and matching Cloud Code Assist project. It accepts only complete, bounded Google error +envelopes with `error.details[].reason === "VALIDATION_REQUIRED"`; verification wording alone does +not authorize rotation, and this path does not persist a reauthentication mark. Each physical send, +including a 429 retry, consumes the same request budget exactly once. A disabled pool, cancellation, +exhausted budget or unavailable sibling preserves the failure. If sibling request construction or +dispatch fails, the original bounded 403 remains the error; a replay rejected before physical +dispatch returns its unused send allowance. +Continuations, native Responses passthrough, image sidecars, and output already sent to the client do not use this rotation. Current scope is the ordinary Responses request paths. Cursor reports rate limits as adapter diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index 5a6f6c0f21f..da650c7ba9b 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -1667,7 +1667,7 @@ "server-combo-reasoning-replay-eligibility.test.ts": "server", "server-combo-zero-output-failover.test.ts": "server", "server-google-antigravity-oauth-401-replay.test.ts": "server", - "server-google-antigravity-oauth-429-budget.test.ts": "server", + "server-antigravity-web-search-validation.test.ts": "server", "server-google-antigravity-oauth-429-budget.test.ts": "server", "server-gui-bundle-freshness.test.ts": "server", "server-images-bodyless-content-length.test.ts": "server", "server-images-pool-admission.test.ts": "server", diff --git a/src/server/responses/antigravity-validation-refusal.ts b/src/server/responses/antigravity-validation-refusal.ts index 02430f26593..7c3512698a4 100644 --- a/src/server/responses/antigravity-validation-refusal.ts +++ b/src/server/responses/antigravity-validation-refusal.ts @@ -1,19 +1,33 @@ -import { ANTIGRAVITY_VALIDATION_REQUIRED_PREFIX } from "../../adapters/google-errors"; +import { ANTIGRAVITY_VALIDATION_REQUIRED_PREFIX, safeAntigravityHttpErrorMessage } from "../../adapters/google-errors"; import { readBoundedResponseBody } from "../../lib/bounded-body"; +/** Match only the normalized Antigravity validation-refusal marker. */ +export function hasAntigravityValidationRefusalMarker(text: string): boolean { + return text.startsWith(`${ANTIGRAVITY_VALIDATION_REQUIRED_PREFIX}: `); +} + /** Inspect only the Google adapter's normalized, already bounded 403 response. */ export async function isAntigravityValidationRefusal(response: Response, signal?: AbortSignal): Promise { + return inspectValidationRefusal(response, signal, false); +} + +/** Sidecar dispatch retains the raw Google envelope; normalize only a complete bounded body. */ +export async function isAntigravityRawValidationRefusal(response: Response, signal?: AbortSignal): Promise { + return inspectValidationRefusal(response, signal, true); +} + +async function inspectValidationRefusal(response: Response, signal: AbortSignal | undefined, raw: boolean): Promise { if (response.status !== 403 || signal?.aborted) return false; try { const body = await readBoundedResponseBody(response.clone(), { - maxBytes: 1024, - totalTimeoutMs: 1000, - firstByteTimeoutMs: 1000, - inactivityTimeoutMs: 1000, + maxBytes: raw ? 4096 : 1024, + totalTimeoutMs: raw ? 2000 : 1000, + firstByteTimeoutMs: raw ? 2000 : 1000, + inactivityTimeoutMs: raw ? 2000 : 1000, signal, }); return !signal?.aborted && body.displaySafe && !body.truncated && !body.timedOut && !body.oversized - && body.text.startsWith(`${ANTIGRAVITY_VALIDATION_REQUIRED_PREFIX}: `); + && hasAntigravityValidationRefusalMarker(raw ? safeAntigravityHttpErrorMessage(403, body.text) : body.text); } catch { return false; } diff --git a/src/server/responses/request-send-budget.ts b/src/server/responses/request-send-budget.ts index b8bc18e4fd9..d56836b7135 100644 --- a/src/server/responses/request-send-budget.ts +++ b/src/server/responses/request-send-budget.ts @@ -171,19 +171,21 @@ export function createResponsesSendBudget( * The hop hands its reservation down through `pendingHopPermit`, the same seam the * passthrough ladder already uses, and this view spends it on the adapter's FIRST * reservation. Every later send in that ladder is a new physical send and is charged - * normally. A permit the adapter takes but never sends under is released through the same - * call it would have used for a reservation of its own, so an abandoned replay is refunded - * rather than left charged. + * normally. The refundable view below keeps an unused hop open until executor invocation; + * its adapter release hands back an abandoned replay. The ordinary view retains early + * settlement for existing callers. */ + const claimHopPermit = (): SingleUseDispatchPermit | undefined => { + const permit = pendingHopPermit; + pendingHopPermit = undefined; + return permit; + }; const adapterDispatchBudget: RequestExecutionBudget | undefined = adapterSendBudget === undefined - ? undefined - : adapterDispatchBudgetView(adapterSendBudget, { - claimHopPermit: () => { - const permit = pendingHopPermit; - pendingHopPermit = undefined; - return permit; - }, - }); + ? undefined : adapterDispatchBudgetView(adapterSendBudget, { claimHopPermit }); + // Opt in only where the adapter's executor is the confirmation boundary. Ordinary callers + // keep the existing early settlement contract, including already-settled permit fallback. + const refundableAdapterDispatchBudget: RequestExecutionBudget | undefined = adapterSendBudget === undefined + ? undefined : adapterDispatchBudgetView(adapterSendBudget, { claimHopPermit }, true); /** * How many sends a recovery leg may make, and the permit that authorises the last one. * @@ -279,6 +281,7 @@ export function createResponsesSendBudget( get sendsUsed(): number { return sendBudget.used; }, adapterSendBudget, adapterDispatchBudget, + refundableAdapterDispatchBudget, noteAdapterPhysicalSend, noteAdapterRecoveryWithheld, sendBudgetExhausted, @@ -312,6 +315,7 @@ export type ResponsesSendBudget = Exclude SingleUseDispatchPermit | undefined }, + refundableHop = false, ): RequestExecutionBudget { return { get used(): number { return budget.used; }, @@ -338,30 +342,29 @@ function adapterDispatchBudgetView( hopPermit.release(); return budget.reserveDispatch({ ...intent, sendClass: hopPermit.sendClass }); } - // Confirmed here rather than in `use()`: the adapter reserves immediately before it - // opens the transport, which is the same boundary the hop's own confirmation uses. - // A permit some other leg already settled returns false, and this falls through to a - // real reservation rather than handing the adapter a dead permit -- an adapter whose - // `use()` fails treats the request as exhausted and stops sending entirely. - if (hopPermit !== undefined && hopPermit.assumeCharge()) { - let spent = false; + if (hopPermit !== undefined && (refundableHop || hopPermit.assumeCharge())) { + let settled = false; + const confirm = (): boolean => { + if (settled) return false; + settled = true; + // Admission can still reject this replay. Close its external booking only when + // the adapter invokes its executor, leaving release() able to refund it until then. + if (!refundableHop || hopPermit.assumeCharge()) return true; + // Preserve the old fallback for a permit another leg already settled. + const fresh = budget.reserveDispatch(intent); + return fresh.allowed && fresh.permit.assumeCharge(); + }; return { allowed: true, permit: { sendClass: hopPermit.sendClass, - use: (): boolean => { - if (spent) return false; - spent = true; - return true; - }, - assumeCharge: (): boolean => { - if (spent) return false; - spent = true; - return true; + use: confirm, + assumeCharge: confirm, + release: (): void => { + if (!refundableHop || settled) return; + settled = true; + hopPermit.release(); }, - // The hop's charge is already settled and belongs to the leg that asked for it, - // so there is nothing here to refund. - release: (): void => {}, }, }; } diff --git a/src/server/responses/sidecar-execution.ts b/src/server/responses/sidecar-execution.ts index d3e71f07a23..d19f8516726 100644 --- a/src/server/responses/sidecar-execution.ts +++ b/src/server/responses/sidecar-execution.ts @@ -24,10 +24,12 @@ import { isGenericOAuthFailoverEnabled, rotateGenericOAuthAccountOn429, rotateGenericOAuthAccountOnRefusal, + rotateAntigravityAccountOnAuthRefusal, quarantineKiroSuspendedAccount, failoverAccountSnapshot, } from "../../oauth/generic-account-failover"; import { classifyKiroRefusal } from "../../adapters/kiro-refusal"; +import { isNonReplayableResponse } from "../../lib/upstream-retry"; import { noteKiroMonthlyRefusal, noteKiroServedSuccess } from "../../providers/kiro-usage"; import { persistKiroAccountState } from "../../providers/kiro-account-state-disk"; import { readDisplaySafeErrorText } from "./core-errors"; @@ -45,6 +47,7 @@ import { recordAdapterReasoning, recordAdapterTier } from "../request-log"; import { normalizeLogConversationId } from "../request-log-conversation"; import { rememberResponseState } from "../../responses/state"; import { trackStreamLifetime } from "../lifecycle"; +import { isAntigravityRawValidationRefusal } from "./antigravity-validation-refusal"; /** One responsibility of the Responses request pipeline; state owners are explicit. */ export async function executeResponsesSidecars( @@ -85,9 +88,11 @@ export async function executeResponsesSidecars( | "notifyResponseComplete" | "cancelResponseCompletion" >, - sendBudgetState: Pick, + sendBudgetState: Pick, ) { const { config, options, logCtx } = requestContext; + const antigravityPoolActivated = requestState.route.providerName === "google-antigravity" + && isGenericOAuthFailoverEnabled(config, requestState.route.providerName); const { applyFailoverSnapshot, anthropicSessionKey, @@ -182,13 +187,29 @@ export async function executeResponsesSidecars( || (logCtx.activeAttempt?.deliverySummary?.semanticBytes ?? 0) > 0 || (logCtx.activeAttempt?.deliverySummary?.sideEffectEvents ?? 0) > 0; const noteSidecarOutput = () => { sidecarOutputStarted = true; options.onFirstOutput?.(); }; + // A single request may rotate once for a verification refusal. Keep this separate from the + // broader OAuth failover count: a sidecar turn can also spend that count on rate-limit recovery. + let antigravityValidationRotationAttempted = false; const rotateSidecarProviderOn429 = async ( retryAfter: string | null, responseHeaders?: Headers, retryParsed?: OcxParsedRequest, originalResponse?: Response, ): Promise<{ adapter: ProviderAdapter; recoveryKind: AttemptRecoveryKind } | null> => { - if (route.providerName !== "kiro" && !(route.providerName === "anthropic" && originalResponse?.status === 403) && originalResponse && originalResponse.status !== 429) return null; + const antigravityValidationResponse = canRunWebSearch && route.providerName === "google-antigravity" + && route.provider.authMode === "oauth" && originalResponse?.status === 403; + if (route.providerName !== "kiro" && !(route.providerName === "anthropic" && originalResponse?.status === 403) + && !antigravityValidationResponse && originalResponse && originalResponse.status !== 429) return null; + let antigravityVerification = false; + if (antigravityValidationResponse) { + if (transportState.adapter.name !== "google" || !antigravityPoolActivated + || options.abortSignal?.aborted || isNonReplayableResponse(originalResponse) + || antigravityValidationRotationAttempted + || transportState.genericFailovers >= transportState.genericFailoverLimit + || sidecarHasCommittedOutput()) return null; + antigravityVerification = await isAntigravityRawValidationRefusal(originalResponse, options.abortSignal); + if (!antigravityVerification) return null; + } const refusal = route.providerName === "kiro" && originalResponse ? classifyKiroRefusal(originalResponse.status, await readDisplaySafeErrorText(originalResponse.clone(), options.abortSignal ?? new AbortController().signal, "")).kind @@ -205,6 +226,42 @@ export async function executeResponsesSidecars( // sidecar loops used to flatten all three to `key-429`, so an account rotation read as a key // rotation in the attempt row and in the Logs UI. let recoveryKind: AttemptRecoveryKind = "key-429"; + if (antigravityVerification) { + const sent = transportState.replayOAuthCredentialSnapshot; + const failedAccountId = transportState.genericFailoverAccountId; + if (!sent || !failedAccountId || sent.accountId !== failedAccountId) return null; + if (options.abortSignal?.aborted || sidecarHasCommittedOutput()) return null; + + const hop = reserveCredentialHop( + "auth-recovery", + `${route.providerName}|${route.modelId}|sidecar-oauth-verify`, + ); + if (!hop.allowed) return null; + antigravityValidationRotationAttempted = true; + const nextAccountId = rotateAntigravityAccountOnAuthRefusal( + antigravityPoolActivated, sent.accountId, sent.generation, route.modelId, + ); + if (!nextAccountId) { + hop.permit?.release(); + return null; + } + try { + // Keep the sibling's bearer and its account-matched project together; never inherit the + // failed account's identity or project. + const snapshot = await failoverAccountSnapshot(route.providerName, nextAccountId); + if (options.abortSignal?.aborted || !await applyFailoverSnapshot(snapshot, retryParsed) + || transportState.replayOAuthCredentialSnapshot?.accountId !== nextAccountId) { + hop.permit?.release(); + return null; + } + transportState.genericFailovers += 1; + } catch { + hop.permit?.release(); + return null; + } + recoveryKind = "oauth-account-403"; + sendBudgetState.pendingHopPermit = hop.permit; + } const rotated = !originalResponse || originalResponse.status === 429 ? rotateProviderTransportOn429(config, route.providerName, route.provider, { retryAfter, @@ -212,7 +269,9 @@ export async function executeResponsesSidecars( attemptedKey: route.provider.apiKey, promptCacheKey: parsed.options.promptCacheKey, }) : null; - if (rotated) { + if (antigravityVerification) { + // `applyFailoverSnapshot` has already rebound the route to the sibling identity/project. + } else if (rotated) { route.provider = rotated; } else if ( // A POSITIVE gate, not an early return. An early `return null` here made every later arm @@ -225,7 +284,8 @@ export async function executeResponsesSidecars( ) { // Intersection with the request's shared budget. The sidecar replay is dispatched by the // web-search/image loop and never reaches `onSendsConsumed`, so this reservation is the - // charge; a refusal returns null and the caller keeps the real 429 it already has. + // charge for other sidecars. Antigravity web search hands its reservation to the adapter + // physical-send owner instead; a refusal preserves the real 429. const hop = reserveCredentialHop( "auth-recovery", `${route.providerName}|${route.modelId}|sidecar-oauth-429`, @@ -259,7 +319,8 @@ export async function executeResponsesSidecars( return null; } recoveryKind = "oauth-account-429"; - hop.permit?.use(); + if (canRunWebSearch && route.providerName === "google-antigravity") sendBudgetState.pendingHopPermit = hop.permit; + else hop.permit?.use(); } else if ( // Anthropic's pool is excluded from generic failover, so without this arm a 429 inside a // web-search or image-bridge turn was terminal even with the pool fully enabled -- while @@ -321,6 +382,7 @@ export async function executeResponsesSidecars( providerName: route.providerName, provider: route.provider, adapterName: rotatedAdapter.name, + ...(antigravityVerification ? { oauthCredentialSnapshot: transportState.replayOAuthCredentialSnapshot } : {}), }); } bindRouteReasoningReplayScope({ @@ -328,6 +390,7 @@ export async function executeResponsesSidecars( providerName: route.providerName, provider: route.provider, adapterName: rotatedAdapter.name, + ...(antigravityVerification ? { oauthCredentialSnapshot: transportState.replayOAuthCredentialSnapshot } : {}), }); return { adapter: rotatedAdapter, recoveryKind }; }; @@ -484,6 +547,11 @@ export async function executeResponsesSidecars( abortSignal: options.abortSignal, translatorBudget, providerFetch: routedProviderFetch, + ...(route.providerName === "google-antigravity" ? { + sendBudget: sendBudgetState.refundableAdapterDispatchBudget, + onPhysicalSend: (send: { ordinal: number; recovery?: AttemptRecoveryKind }) => + sendBudgetState.noteAdapterPhysicalSend(logCtx.usageLogInputTokens, send), + } : {}), }, backend: wsPlan.backend, forwardProvider: wsPlan.forwardSidecar?.provider, @@ -516,6 +584,13 @@ export async function executeResponsesSidecars( stallTimeoutSec: wsPlan.stallTimeoutSec, streamRoutedModelOutput: wsPlan.streamRoutedModelOutput, on429: rotateSidecarProviderOn429, + ...(route.providerName === "google-antigravity" ? { onIterationEnd: () => { + // The next Antigravity adapter dispatch claims this permit. If request construction, + // admission or cancellation stopped it first, return the unused send allowance. + const permit = sendBudgetState.pendingHopPermit; + sendBudgetState.pendingHopPermit = undefined; + permit?.release(); + } } : {}), retryOn429Policy: route.providerName === "kiro" && isGenericOAuthFailoverEnabled(config, "kiro") ? null : rateLimitRetryPolicyFor(route.provider), onCompletedResponse: response => { diff --git a/src/web-search/loop.ts b/src/web-search/loop.ts index 940ef296d1f..0cbf89fbd79 100644 --- a/src/web-search/loop.ts +++ b/src/web-search/loop.ts @@ -353,6 +353,8 @@ export interface WebSearchLoopDeps { | { adapter: ProviderAdapter; recoveryKind: AttemptRecoveryKind } | null | Promise<{ adapter: ProviderAdapter; recoveryKind: AttemptRecoveryKind } | null>; + /** Release a credential-hop permit if this iteration ended before the adapter used it. */ + onIterationEnd?: () => void; /** Opt-in same-target 429 policy (key-auth providers). When present, 429 replays on the SAME key before on429 rotation. */ retryOn429Policy?: Required | null; /** Called only when the final bridged Responses stream reaches completed or incomplete. */ @@ -501,6 +503,8 @@ export async function runWithWebSearch(deps: WebSearchLoopDeps): Promise {}); } catch { /* already closed */ } + const retainRefusal = prepared.response.status === 403 + && deps.incomingMeta?.providerName === "google-antigravity" && rotated.recoveryKind === "oauth-account-403"; + // Keep the bounded refusal readable until the sibling actually returns headers. + // Other rotation paths retain their existing cancellation boundary. + if (!retainRefusal) { + try { void prepared.response.body?.cancel().catch(() => {}); } catch { /* already closed */ } + } adapter = rotated.adapter; // Stall-watchdog seam between bounded retry fetches (audit 011 B3). yield { type: "heartbeat" }; - prepared = await fetchOnce(adapter, rotated.recoveryKind); + let replacement: IterationResponse; + try { + replacement = await fetchOnce(adapter, rotated.recoveryKind); + } catch (error) { + if (!retainRefusal || signal.aborted || headerDeadline.didExpire() || isTranslatorBudgetExceededError(error)) throw error; + break; // Format the original 403 safely; replacement diagnostics are not client errors. + } + if (retainRefusal) { + try { void prepared.response.body?.cancel().catch(() => {}); } catch { /* already closed */ } + } + prepared = replacement; } // Final headers have arrived. Clear only the deadline timer before ANY body read. @@ -623,6 +643,7 @@ export async function runWithWebSearch(deps: WebSearchLoopDeps): Promise resetProviderRequestPacingForTest()); * A credential hop has already reserved the replay it hands to the adapter, so the adapter's * first send spends that permit through the dispatch view instead of reserving again. */ -function prepaid() { +function prepaid(refundable = false) { const parent = createRequestExecutionBudget(); parent.used = 3; const { owner, dispose } = budgetOwner(parent); const hop = owner.reserveCredentialHop("auth-recovery", url, true); if (!hop.allowed || !hop.permit) throw new Error("Expected prepaid final send"); owner.pendingHopPermit = hop.permit; - const scope = owner.adapterDispatchBudget; + const scope = refundable ? owner.refundableAdapterDispatchBudget : owner.adapterDispatchBudget; if (!scope) throw new Error("Expected an adapter dispatch budget"); return { parent, scope, dispose }; } @@ -112,6 +112,30 @@ describe("adapter physical inference admission", () => { } finally { dispose(); } }); + test.each(["pacing", "backoff", "abort", "adapter"] as const)("an opted-in hop refunds when the %s leg never dispatches", async phase => { + const { parent, scope, dispose } = prepaid(true); + let sends = 0; + const controller = new AbortController(); + const failure = new Error(`fixture ${phase} refusal`); + const executor = Object.assign(async () => { sends += 1; return new Response("unexpected"); }, { + waitForPacing: async () => { if (phase === "pacing") throw failure; }, + }) as typeof fetch; + try { + const send = createAdapterPhysicalSend({ sendBudget: scope, abortSignal: controller.signal }, executor); + await expect(send({ url, beforeDispatch: () => { + if (phase === "backoff") throw failure; + if (phase === "abort") controller.abort(failure); + }, dispatch: physical => { + if (phase === "adapter") throw failure; + return physical(url); + } })).rejects.toBe(failure); + // The refundable view leaves the hop open until actual executor invocation. + expect(parent.used).toBe(3); + expect(parent.reserveSpent).toBe(false); + expect(sends).toBe(0); + } finally { dispose(); } + }); + test("an exhausted initial send performs no inference or retry preparation", async () => { const budget = createRequestExecutionBudget(); budget.used = 4; diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index 700841acd0a..264156a6a0c 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -951,7 +951,7 @@ "server-combo-cooldown-fallback.test.ts": "server", "server-combo-failover-e2e.test.ts": "server", "server-combo-held-response.test.ts": "server", "server-combo-reasoning-replay-eligibility.test.ts": "server", "server-combo-zero-output-failover.test.ts": "server", "server-google-antigravity-oauth-401-replay.test.ts": "server", - "server-google-antigravity-oauth-429-budget.test.ts": "server", "server-gui-bundle-freshness.test.ts": "server", + "server-antigravity-web-search-validation.test.ts": "server", "server-google-antigravity-oauth-429-budget.test.ts": "server", "server-gui-bundle-freshness.test.ts": "server", "server-images-bodyless-content-length.test.ts": "server", "server-images-pool-admission.test.ts": "server", "server-images.test.ts": "server", "server-key-failover-e2e.test.ts": "server", "server-jev-combo-e2e.test.ts": "server", "server-kiro-completion-e2e.test.ts": "server", diff --git a/tests/server/inference-send-budget.test.ts b/tests/server/inference-send-budget.test.ts index 0e3ca1597a9..ba32b9722df 100644 --- a/tests/server/inference-send-budget.test.ts +++ b/tests/server/inference-send-budget.test.ts @@ -93,6 +93,65 @@ describe("createInferenceSendBudget", () => { expect(budget.used).toBe(3); } finally { dispose(); } }); + test.each([false, true])("an undispatched adapter hop remains refundable (externally counted=%s)", external => { + const budget = createRequestExecutionBudget(); + const { owner, dispose } = budgetOwner(budget); + try { + const targetKey = "https://upstream.example/model"; + const initial = owner.refundableAdapterDispatchBudget!.reserveDispatch({ sendClass: "initial", targetKey }); + if (!initial.allowed) throw new Error("Expected initial permit"); + initial.permit.use(); + const hop = owner.reserveCredentialHop("auth-recovery", targetKey, external); + owner.pendingHopPermit = hop.permit; + const replay = owner.refundableAdapterDispatchBudget!.reserveDispatch({ sendClass: "transient", targetKey }); + if (!replay.allowed) throw new Error("Expected replay permit"); + expect(budget.used).toBe(2); + replay.permit.release(); + replay.permit.release(); + expect(budget.used).toBe(1); + expect(budget.remainingBaseSends(3)).toBe(2); + expect(replay.permit.use()).toBe(false); + expect(replay.permit.assumeCharge()).toBe(false); + owner.noteTransientSends(1); + expect(budget.used).toBe(2); // No abandoned external booking can swallow a later send. + } finally { dispose(); } + }); + + test.each(["use", "assumeCharge"] as const)("adapter hop %s settles an external booking once and cannot refund a send", confirm => { + const budget = createRequestExecutionBudget(); + const { owner, dispose } = budgetOwner(budget); + try { + const targetKey = "https://upstream.example/model"; + const hop = owner.reserveCredentialHop("auth-recovery", targetKey, true); + owner.pendingHopPermit = hop.permit; + const replay = owner.refundableAdapterDispatchBudget!.reserveDispatch({ sendClass: "transient", targetKey }); + if (!replay.allowed) throw new Error("Expected replay permit"); + expect(replay.permit[confirm]()).toBe(true); + expect(replay.permit.use()).toBe(false); + expect(replay.permit.assumeCharge()).toBe(false); + replay.permit.release(); + expect(budget.used).toBe(1); + owner.noteTransientSends(1); + expect(budget.used).toBe(2); + } finally { dispose(); } + }); + + test("a previously settled adapter hop falls back to a fresh reservation", () => { + const budget = createRequestExecutionBudget(); + const { owner, dispose } = budgetOwner(budget); + try { + const targetKey = "https://upstream.example/model"; + const hop = owner.reserveCredentialHop("auth-recovery", targetKey, true); + expect(hop.permit?.assumeCharge()).toBe(true); + owner.pendingHopPermit = hop.permit; + const replay = owner.refundableAdapterDispatchBudget!.reserveDispatch({ sendClass: "transient", targetKey }); + if (!replay.allowed) throw new Error("Expected replay permit"); + expect(replay.permit.use()).toBe(true); + expect(budget.used).toBe(2); + replay.permit.release(); + expect(budget.used).toBe(2); + } finally { dispose(); } + }); test("mints a default-policy holder and parks this request's spend tracker on the log", () => { const logCtx: RequestLogContext = { model: "m", provider: "p" }; const req = new Request("http://localhost/v1/responses", { method: "POST" }); diff --git a/tests/server/server-antigravity-web-search-validation.test.ts b/tests/server/server-antigravity-web-search-validation.test.ts new file mode 100644 index 00000000000..80d1e8cfff1 --- /dev/null +++ b/tests/server/server-antigravity-web-search-validation.test.ts @@ -0,0 +1,444 @@ +import { afterEach, beforeEach, describe, expect, spyOn, test } from "bun:test"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { saveConfig } from "../../src/config"; +import { clearGenericFailoverHealth } from "../../src/oauth/generic-account-failover"; +import { getAccountSet, saveCredential, setActiveAccount, setAccountPaused } from "../../src/oauth/store"; +import { handleResponses } from "../../src/server/responses"; +import type { OcxConfig } from "../../src/types"; +import { installIsolatedCodexHome, type IsolatedCodexHome } from "../helpers/isolated-codex-home"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; +import { removeTreeWithRetry } from "../helpers/remove-tree"; +import * as retry from "../../src/lib/upstream-retry"; +import * as webSearch from "../../src/web-search"; +import * as adapterResolve from "../../src/server/adapter-resolve"; +import * as pacing from "../../src/providers/request-pacing"; +import { createRequestExecutionBudget } from "../../src/lib/request-execution-budget"; + +const DAILY_API_BASE = "https://daily-cloudcode-pa.googleapis.com"; + +let testDir = ""; +let previousHome: string | undefined; +let isolatedCodexHome: IsolatedCodexHome | null = null; +let originalFetch: typeof fetch; +let releaseSpendHome: (() => void) | undefined; +let sleepSpy: ReturnType | undefined; + +const takeSpendHome = (): void => { + releaseSpendHome ??= acquireOwnedSpendHome(); +}; + +beforeEach(() => { + originalFetch = globalThis.fetch; + previousHome = process.env.OPENCODEX_HOME; + isolatedCodexHome = installIsolatedCodexHome("ocx-google-429-codex-"); + testDir = mkdtempSync(join(tmpdir(), "ocx-google-429-")); + process.env.OPENCODEX_HOME = testDir; + takeSpendHome(); + clearGenericFailoverHealth(); + sleepSpy = spyOn(retry, "sleepWithAbort").mockImplementation(async () => {}); +}); + +afterEach(() => { + sleepSpy?.mockRestore(); + sleepSpy = undefined; + releaseSpendHome?.(); + releaseSpendHome = undefined; + clearGenericFailoverHealth(); + globalThis.fetch = originalFetch; + if (previousHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previousHome; + isolatedCodexHome?.restore(); + isolatedCodexHome = null; + if (testDir) removeTreeWithRetry(testDir); +}); + +function antigravityConfig(): OcxConfig { + return { + port: 0, + hostname: "127.0.0.1", + defaultProvider: "google-antigravity", + providers: { + "google-antigravity": { + adapter: "google", + baseUrl: DAILY_API_BASE, + authMode: "oauth", + googleMode: "cloud-code-assist", + project: "initial-project-id", + models: ["gemini-3.8-flash"], + }, + }, + } as OcxConfig; +} + +function jsonSuccessBody(text: string): Record { + return { + response: { + candidates: [{ + content: { + role: "model", + parts: [{ text }], + }, + finishReason: "STOP", + }], + usageMetadata: { + promptTokenCount: 5, + candidatesTokenCount: 3, + totalTokenCount: 8, + }, + }, + }; +} + +function sseSuccessBody(text: string): string { + return `data: ${JSON.stringify(jsonSuccessBody(text))}\n\n`; +} + +function createResponsesRequest(bodyOverrides: Record = {}): Request { + return new Request("http://127.0.0.1/v1/responses", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: "google-antigravity/gemini-3.8-flash", + input: "hello", + stream: false, + ...bodyOverrides, + }), + }); +} + +async function seedAntigravityAccounts(count: number): Promise> { + for (let i = 1; i <= count; i++) { + await saveCredential("google-antigravity", { + access: `token-${i}`, + refresh: `refresh-${i}`, + expires: Date.now() + 3_600_000, + accountId: `account-${i}`, + projectId: `project-${i}`, + }); + } + const accounts = getAccountSet("google-antigravity")!.accounts; + await setActiveAccount("google-antigravity", accounts[0]!.id); + return accounts.map((a, idx) => ({ + id: a.id, + auth: `Bearer token-${idx + 1}`, + project: `project-${idx + 1}`, + })); +} + +function installAntigravityFetchMock( + handler: (info: { auth: string; project: string; sendIndex: number }) => Response | Promise, +): void { + let sendIndex = 0; + globalThis.fetch = (async (input, init) => { + const url = input instanceof Request ? input.url : String(input); + const parsedUrl = new URL(url); + + if (parsedUrl.origin === DAILY_API_BASE + && ["/v1internal:streamGenerateContent", "/v1internal:generateContent"].includes(parsedUrl.pathname)) { + sendIndex += 1; + const auth = new Headers(init?.headers).get("authorization") ?? ""; + let project = ""; + if (typeof init?.body === "string") { + try { + const parsed = JSON.parse(init.body) as { project?: string }; + project = parsed.project ?? ""; + } catch { /* ignore */ } + } + return handler({ auth, project, sendIndex }); + } + + if (parsedUrl.hostname === "127.0.0.1" || parsedUrl.hostname === "localhost") return originalFetch(input, init); + throw new Error(`Unexpected external request: ${url}`); + }) as typeof fetch; +} + +const structuredRefusal = JSON.stringify({ error: { + status: "PERMISSION_DENIED", message: "validate", details: [{ reason: "VALIDATION_REQUIRED" }], +} }); + +function webSearchConfig(): OcxConfig { + const cfg = antigravityConfig(); + cfg.webSearchSidecar = { backend: "exa", exaApiKey: "synthetic-exa-key" }; + saveConfig(cfg); + return cfg; +} + +function budget(sends: number) { + return createRequestExecutionBudget({ + maxTotalModelSends: sends, baseSendAllowance: sends, finalRecoveryAllowance: 0, + maxAlternateTargetSends: 0, maxTargetTransitions: 0, + }); +} + +const request = () => createResponsesRequest({ tools: [{ type: "web_search" }] }); +const route = { model: "gemini-3.8-flash", provider: "google-antigravity" }; +const durableHealth = () => getAccountSet("google-antigravity")!.accounts.map(row => ({ + id: row.id, needsReauth: row.needsReauth, needsReauthReason: row.needsReauthReason, +})); + +describe("Antigravity web-search structured validation rotation (#6666)", () => { + test("structured A to B uses B's token/project in exactly two sends and leaves durable health unchanged", async () => { + const accounts = await seedAntigravityAccounts(2); + const cfg = webSearchConfig(); + const before = durableHealth(); + const sends: Array<{ auth: string; project: string }> = []; + installAntigravityFetchMock(({ auth, project, sendIndex }) => { + sends.push({ auth, project }); + return sendIndex === 1 ? new Response(structuredRefusal, { status: 403 }) + : new Response(sseSuccessBody("sibling answered"), { headers: { "content-type": "text/event-stream" } }); + }); + const sendBudget = budget(2); + const response = await handleResponses(request(), cfg, route, { sendBudget }); + expect(response.status).toBe(200); + expect(await response.text()).toContain("sibling answered"); + expect(sends).toEqual(accounts.map(({ auth, project }) => ({ auth, project }))); + expect(sendBudget.used).toBe(2); + expect(durableHealth()).toEqual(before); + }); + + test("repeated structured refusals stop after one sibling", async () => { + const accounts = await seedAntigravityAccounts(3); + const cfg = webSearchConfig(); + const before = durableHealth(); + const sends: Array<{ auth: string; project: string }> = []; + installAntigravityFetchMock(({ auth, project }) => { + sends.push({ auth, project }); + return new Response(structuredRefusal, { status: 403 }); + }); + const sendBudget = budget(3); + const response = await handleResponses(request(), cfg, route, { sendBudget }); + expect(response.status).toBe(403); + await response.text(); + expect(sends).toEqual(accounts.slice(0, 2).map(({ auth, project }) => ({ auth, project }))); + expect(sendBudget.used).toBe(2); + expect(durableHealth()).toEqual(before); + }); + + test.each([ + ["sentence only", JSON.stringify({ error: { message: "Verify your account to continue." } })], + ["legacy sentence only", JSON.stringify({ error: { message: "Please verify your account to continue using Antigravity." } })], + ["unrelated", JSON.stringify({ error: { message: "Feature unavailable", details: [{ reason: "OTHER" }] } })], + ["spoofed message", JSON.stringify({ error: { message: "Antigravity account validation required (VALIDATION_REQUIRED): validate" } })], + ["spoofed top level", JSON.stringify({ reason: "VALIDATION_REQUIRED", error: { message: "validate" } })], + ["spoofed nested reason", JSON.stringify({ error: { details: [{ metadata: { reason: "VALIDATION_REQUIRED" } }] } })], + ["non-array details", JSON.stringify({ error: { details: { reason: "VALIDATION_REQUIRED" } } })], + ["malformed", structuredRefusal.slice(0, -1)], + ["oversized complete JSON", JSON.stringify({ error: { details: [{ reason: "VALIDATION_REQUIRED" }] }, padding: "x".repeat(4096) })], + ["oversized trailing whitespace", structuredRefusal + " ".repeat(4096)], + ])("%s never rotates", async (_name, body) => { + const accounts = await seedAntigravityAccounts(2); + const cfg = webSearchConfig(); + const before = durableHealth(); + const sends: Array<{ auth: string; project: string }> = []; + installAntigravityFetchMock(({ auth, project }) => { + sends.push({ auth, project }); + return new Response(body, { status: 403 }); + }); + const response = await handleResponses(request(), cfg, route); + expect(response.status).toBe(403); + await response.text(); + expect(sends).toEqual([{ auth: accounts[0]!.auth, project: accounts[0]!.project }]); + expect(durableHealth()).toEqual(before); + }); + + test("a spent physical-send budget preserves the refusal and durable health", async () => { + await seedAntigravityAccounts(2); + const cfg = webSearchConfig(); + const before = durableHealth(); + let sends = 0; + installAntigravityFetchMock(() => { sends++; return new Response(structuredRefusal, { status: 403 }); }); + const sendBudget = budget(1); + const response = await handleResponses(request(), cfg, route, { sendBudget }); + expect(response.status).toBe(403); + await response.text(); + expect(sends).toBe(1); + expect(sendBudget.used).toBe(1); + expect(durableHealth()).toEqual(before); + }); + + test("cancellation prevents a sibling send and durable health changes", async () => { + await seedAntigravityAccounts(2); + const cfg = webSearchConfig(); + const before = durableHealth(); + const abort = new AbortController(); + let sends = 0; + installAntigravityFetchMock(() => { + sends++; abort.abort(); return new Response(structuredRefusal, { status: 403 }); + }); + const response = await handleResponses(request(), cfg, route, { abortSignal: abort.signal }); + await response.text(); + expect(sends).toBe(1); + expect(durableHealth()).toEqual(before); + }); + + test("a pool absent before dispatch cannot activate after the first refusal", async () => { + await seedAntigravityAccounts(1); + const cfg = webSearchConfig(); + let sends = 0; + installAntigravityFetchMock(async () => { + sends++; + await saveCredential("google-antigravity", { + access: "late-token", refresh: "late-refresh", expires: Date.now() + 3_600_000, + accountId: "late-account", projectId: "late-project", + }, { addAccount: true }); + clearGenericFailoverHealth(); + return new Response(structuredRefusal, { status: 403 }); + }); + const response = await handleResponses(request(), cfg, route); + expect(response.status).toBe(403); + await response.text(); + expect(sends).toBe(1); + expect(durableHealth().every(row => !row.needsReauth)).toBe(true); + }); + + test.each(["unavailable", "projectless"])("an %s sibling preserves the refusal and releases its unused permit", async kind => { + const accounts = await seedAntigravityAccounts(kind === "projectless" ? 1 : 2); + if (kind === "projectless") { + await saveCredential("google-antigravity", { + access: "projectless-token", refresh: "projectless-refresh", expires: Date.now() + 3_600_000, + accountId: "projectless-account", + }, { addAccount: true }); + await setActiveAccount("google-antigravity", accounts[0]!.id); + } + const cfg = webSearchConfig(); + const before = durableHealth(); + let sends = 0; + installAntigravityFetchMock(async () => { + sends++; + if (kind === "unavailable") await setAccountPaused("google-antigravity", accounts[1]!.id, true); + return new Response(structuredRefusal, { status: 403 }); + }); + const sendBudget = budget(2); + const response = await handleResponses(request(), cfg, route, { sendBudget }); + expect(response.status).toBe(403); + await response.text(); + expect(sends).toBe(1); + expect(sendBudget.used).toBe(1); + expect(sendBudget.remainingBaseSends(2)).toBe(1); + expect(durableHealth()).toEqual(before); + }); +}); + +test("web-search preserves structured refusal once the roster's failover limit is spent", async () => { + const accounts = await seedAntigravityAccounts(5); + const cfg = webSearchConfig(); + const sends: string[] = []; + installAntigravityFetchMock(async ({ auth, sendIndex }) => { + sends.push(auth); + if (sendIndex <= 4) return Response.json({ error: { + status: "RESOURCE_EXHAUSTED", message: "Quota exceeded for quota metric", + } }, { status: 429 }); + // A new eligible sibling exists, but it cannot enlarge the captured failover ceiling. + await saveCredential("google-antigravity", { + access: "late-token", refresh: "late-refresh", expires: Date.now() + 3_600_000, + accountId: "late-account", projectId: "late-project", + }, { addAccount: true }); + return new Response(structuredRefusal, { status: 403 }); + }); + const sendBudget = budget(10); + const response = await handleResponses(request(), cfg, route, { sendBudget }); + expect(response.status).toBe(403); + await response.text(); + expect(sends).toEqual(accounts.map(row => row.auth)); + expect(sendBudget.used).toBe(5); + expect(durableHealth().every(row => !row.needsReauth)).toBe(true); +}); + +test.each(["non-replayable", "committed output", "non-Google adapter"])( + "sidecar callback rejects %s before reserving or rotating", + async gate => { + await seedAntigravityAccounts(2); + const cfg = webSearchConfig(); + const before = durableHealth(); + const sendBudget = budget(2); + const loopSpy = spyOn(webSearch, "runWithWebSearch").mockImplementation(async deps => { + const refusal = new Response(structuredRefusal, { status: 403 }); + if (gate === "non-replayable") retry.markResponseNonReplayable(refusal); + if (gate === "committed output") deps.onFirstOutput?.(); + const name = deps.adapter.name; + if (gate === "non-Google adapter") deps.adapter.name = "mock-other"; + try { + expect(await deps.on429?.(null, refusal.headers, deps.parsed, refusal)).toBeNull(); + expect(sendBudget.used).toBe(0); + expect(sendBudget.remainingBaseSends(2)).toBe(2); + return refusal; + } finally { deps.adapter.name = name; } + }); + try { + const response = await handleResponses(request(), cfg, route, { sendBudget }); + expect(response.status).toBe(403); + await response.text(); + expect(durableHealth()).toEqual(before); + } finally { loopSpy.mockRestore(); } + }, +); + +test.each(["construction", "admission", "admission cancellation"])( + "real web-search sibling %s failure refunds the unused send and preserves error ownership", + async failure => { + const accounts = await seedAntigravityAccounts(2); + const cfg = webSearchConfig(); + const before = durableHealth(); + const sendBudget = budget(2); + const abort = new AbortController(); + const sends: Array<{ auth: string; project: string }> = []; + let rejected = 0; + installAntigravityFetchMock(({ auth, project }) => { + sends.push({ auth, project }); + return new Response(structuredRefusal, { status: 403 }); + }); + const resolve = adapterResolve.resolveAdapter; + const wait = pacing.waitForProviderRequestSlot; + const resolveSpy = spyOn(adapterResolve, "resolveAdapter").mockImplementation((provider, ...args) => { + const adapter = resolve(provider, ...args); + if (failure === "construction" && provider.project === accounts[1]!.project) { + adapter.buildRequest = async () => { rejected++; throw new Error("replacement-build-canary"); }; + } + return adapter; + }); + const pacingSpy = spyOn(pacing, "waitForProviderRequestSlot").mockImplementation(async (name, provider, ...args) => { + if (failure.startsWith("admission") && provider.project === accounts[1]!.project) { + rejected++; + if (failure === "admission cancellation") abort.abort(); + throw new Error("replacement-admission-canary"); + } + return wait(name, provider, ...args); + }); + try { + const response = await handleResponses(request(), cfg, route, { sendBudget, abortSignal: abort.signal }); + const body = await response.text(); + expect(rejected).toBeGreaterThan(0); + expect(response.status).toBe(failure === "admission cancellation" ? 499 : 403); + if (failure !== "admission cancellation") { + expect(body).toContain("Antigravity account validation required (VALIDATION_REQUIRED): validate"); + } + expect(body).not.toContain("canary"); + expect(sends).toEqual([{ auth: accounts[0]!.auth, project: accounts[0]!.project }]); + expect(sendBudget.used).toBe(1); + expect(sendBudget.remainingBaseSends(2)).toBe(1); + expect(durableHealth()).toEqual(before); + } finally { resolveSpy.mockRestore(); pacingSpy.mockRestore(); } + }, +); + +test("web-search 429 recovery still uses the sibling's identity and charges exactly two physical sends", async () => { + const accounts = await seedAntigravityAccounts(2); + const cfg = webSearchConfig(); + const sends: Array<{ auth: string; project: string }> = []; + installAntigravityFetchMock(({ auth, project, sendIndex }) => { + sends.push({ auth, project }); + return sendIndex === 1 ? Response.json({ error: { + status: "RESOURCE_EXHAUSTED", message: "Quota exceeded for quota metric", + } }, { status: 429 }) : new Response(sseSuccessBody("429 sibling answered"), { + headers: { "content-type": "text/event-stream" }, + }); + }); + const sendBudget = budget(2); + const response = await handleResponses(request(), cfg, route, { sendBudget }); + expect(response.status).toBe(200); + expect(await response.text()).toContain("429 sibling answered"); + expect(sends).toEqual(accounts.map(({ auth, project }) => ({ auth, project }))); + expect(sendBudget.used).toBe(2); +}); diff --git a/tests/server/server-google-antigravity-oauth-429-budget.test.ts b/tests/server/server-google-antigravity-oauth-429-budget.test.ts index 348eacd7fd4..05cc7afc92c 100644 --- a/tests/server/server-google-antigravity-oauth-429-budget.test.ts +++ b/tests/server/server-google-antigravity-oauth-429-budget.test.ts @@ -169,7 +169,7 @@ function installAntigravityFetchMock( describe("Google Antigravity OAuth 429 retry and multi-account budget (#5880)", () => { test("an observed caller abort before a validation refusal sends no sibling request", async () => { - await seedAntigravityAccounts(2); + const accounts = await seedAntigravityAccounts(2); const cfg = antigravityConfig(); saveConfig(cfg); const abort = new AbortController(); @@ -184,6 +184,10 @@ describe("Google Antigravity OAuth 429 retry and multi-account budget (#5880)", { model: "gemini-3.8-flash", provider: "google-antigravity" }, { abortSignal: abort.signal }); expect(sends).toBe(1); await response.text(); + const cancelledAccount = getAccountSet("google-antigravity")!.accounts + .find(row => row.id === accounts[0]!.id)!; + expect(cancelledAccount.needsReauth).toBeFalsy(); + expect(cancelledAccount.needsReauthReason).toBeUndefined(); }); test("a spent caller send budget keeps a classified 403 and sends no sibling request", async () => { diff --git a/tests/web-search/web-search-recovery-kind.test.ts b/tests/web-search/web-search-recovery-kind.test.ts index 980fb9f718b..73053601323 100644 --- a/tests/web-search/web-search-recovery-kind.test.ts +++ b/tests/web-search/web-search-recovery-kind.test.ts @@ -90,3 +90,53 @@ test("429 rotation reports the rotator's recovery kind", async () => { expect(await recoveryKindsFor(next => ({ adapter: next, recoveryKind: "key-429" }))) .toEqual([undefined, "key-429"]); }); + +test("web-search does not replay a verification refusal after routed output was committed", async () => { + globalThis.fetch = (async () => Response.json({ + results: [{ title: "Fixture", url: "https://example.test/result", content: "synthetic result" }], + })) as typeof fetch; + let routedSends = 0; + let rotations = 0; + const adapter: ProviderAdapter = { + name: "mock-antigravity", + buildRequest: () => ({ url: "https://routed.test/v1", method: "POST", headers: {}, body: "{}" }), + fetchResponse: async () => { + routedSends++; + if (routedSends === 1) return new Response("ok", { status: 200 }); + return new Response(JSON.stringify({ error: { + status: "PERMISSION_DENIED", message: "validate", details: [{ reason: "VALIDATION_REQUIRED" }], + } }), { status: 403 }); + }, + async *parseStream() { + yield { type: "text_delta", text: "I will check. " } satisfies AdapterEvent; + yield { type: "tool_call_start", id: "search-1", name: "web_search" } satisfies AdapterEvent; + yield { type: "tool_call_delta", arguments: JSON.stringify({ query: "fixture query" }) } satisfies AdapterEvent; + yield { type: "tool_call_end" } satisfies AdapterEvent; + yield { type: "done" } satisfies AdapterEvent; + }, + async parseResponse() { throw new Error("parseResponse must be unreachable"); }, + }; + + const response = await runWithWebSearch({ + parsed: parseRequest({ model: "routed/model", input: "hi", stream: true, tools: [{ type: "web_search" }] }), + adapter, + incomingMeta: { headers: new Headers(), providerName: "google-antigravity", translatorBudget: createTestTranslatorBudget() }, + backend: "exa", + exaApiKey: "synthetic-exa-key", + hostedTool: { type: "web_search" }, + selectedForwardHeaders: new Headers(), + settings: { model: "exa-fixture-model", reasoning: "low", timeoutMs: 30_000 }, + maxSearches: 1, + streamRoutedModelOutput: true, + on429: () => { rotations++; return null; }, + }); + const frames = await collectSse(response.body!); + const text = frames + .filter(frame => frame.data.type === "response.output_text.delta") + .map(frame => String(frame.data.delta ?? "")) + .join(""); + expect(text).toContain("I will check."); + expect(routedSends).toBe(2); + expect(rotations).toBe(0); + expect(frames.some(frame => frame.event === "response.failed")).toBe(true); +});