From 5152e6f12b5e688087c3de666efd6442fe0f03a0 Mon Sep 17 00:00:00 2001 From: baozheng Date: Wed, 7 Oct 2026 17:28:16 +0800 Subject: [PATCH 1/5] fix(antigravity): rotate web-search validation refusals --- src/server/responses/sidecar-execution.ts | 97 ++++++++- src/web-search/loop.ts | 9 +- structure/transports/responses-failover.md | 2 +- ...oogle-antigravity-oauth-429-budget.test.ts | 186 ++++++++++++++++++ .../web-search-recovery-kind.test.ts | 50 +++++ 5 files changed, 339 insertions(+), 5 deletions(-) diff --git a/src/server/responses/sidecar-execution.ts b/src/server/responses/sidecar-execution.ts index d3e71f07a23..80d4316be5a 100644 --- a/src/server/responses/sidecar-execution.ts +++ b/src/server/responses/sidecar-execution.ts @@ -24,10 +24,13 @@ import { isGenericOAuthFailoverEnabled, rotateGenericOAuthAccountOn429, rotateGenericOAuthAccountOnRefusal, + rotateAntigravityAccountOnAuthRefusal, quarantineKiroSuspendedAccount, failoverAccountSnapshot, } from "../../oauth/generic-account-failover"; import { classifyKiroRefusal } from "../../adapters/kiro-refusal"; +import { ANTIGRAVITY_VALIDATION_REQUIRED_PREFIX, safeAntigravityHttpErrorMessage } from "../../adapters/google-errors"; +import { markAccountNeedsReauthIfGeneration } from "../../oauth/store"; import { noteKiroMonthlyRefusal, noteKiroServedSuccess } from "../../providers/kiro-usage"; import { persistKiroAccountState } from "../../providers/kiro-account-state-disk"; import { readDisplaySafeErrorText } from "./core-errors"; @@ -85,7 +88,7 @@ export async function executeResponsesSidecars( | "notifyResponseComplete" | "cancelResponseCompletion" >, - sendBudgetState: Pick, + sendBudgetState: Pick, ) { const { config, options, logCtx } = requestContext; const { @@ -182,13 +185,40 @@ export async function executeResponsesSidecars( || (logCtx.activeAttempt?.deliverySummary?.semanticBytes ?? 0) > 0 || (logCtx.activeAttempt?.deliverySummary?.sideEffectEvents ?? 0) > 0; const noteSidecarOutput = () => { sidecarOutputStarted = true; options.onFirstOutput?.(); }; + // A single request may rotate once for a verification refusal. Keep this separate from the + // broader OAuth failover count: a sidecar turn can also spend that count on rate-limit recovery. + let antigravityValidationRotationAttempted = false; const rotateSidecarProviderOn429 = async ( retryAfter: string | null, responseHeaders?: Headers, retryParsed?: OcxParsedRequest, originalResponse?: Response, ): Promise<{ adapter: ProviderAdapter; recoveryKind: AttemptRecoveryKind } | null> => { - if (route.providerName !== "kiro" && !(route.providerName === "anthropic" && originalResponse?.status === 403) && originalResponse && originalResponse.status !== 429) return null; + const antigravityValidationResponse = canRunWebSearch && route.providerName === "google-antigravity" + && route.provider.authMode === "oauth" && originalResponse?.status === 403; + if (route.providerName !== "kiro" && !(route.providerName === "anthropic" && originalResponse?.status === 403) + && !antigravityValidationResponse && originalResponse && originalResponse.status !== 429) return null; + let antigravityVerification = false; + if (antigravityValidationResponse) { + if (options.abortSignal?.aborted) return null; + const body = await readDisplaySafeErrorText( + originalResponse.clone(), options.abortSignal ?? new AbortController().signal, "", + ); + if (options.abortSignal?.aborted) return null; + let exactMessage = false; + try { + const envelope = JSON.parse(body) as { error?: { message?: unknown } }; + const message = envelope.error?.message; + exactMessage = typeof message === "string" + && /^(?:Please )?verify your account to continue(?: using Antigravity)?[.!]?$/i.test(message.trim()); + } catch { /* an unparseable body cannot authorize account quarantine */ } + // The structured reason is authoritative. For older upstream envelopes, accept only the + // exact verification message in error.message; unrelated 403 text and other fields do not + // convict an account. + antigravityVerification = safeAntigravityHttpErrorMessage(403, body) + .startsWith(`${ANTIGRAVITY_VALIDATION_REQUIRED_PREFIX}: `) || exactMessage; + if (!antigravityVerification) return null; + } const refusal = route.providerName === "kiro" && originalResponse ? classifyKiroRefusal(originalResponse.status, await readDisplaySafeErrorText(originalResponse.clone(), options.abortSignal ?? new AbortController().signal, "")).kind @@ -205,6 +235,51 @@ export async function executeResponsesSidecars( // sidecar loops used to flatten all three to `key-429`, so an account rotation read as a key // rotation in the attempt row and in the Logs UI. let recoveryKind: AttemptRecoveryKind = "key-429"; + if (antigravityVerification) { + const sent = transportState.replayOAuthCredentialSnapshot; + const failedAccountId = transportState.genericFailoverAccountId; + if (!sent || !failedAccountId || sent.accountId !== failedAccountId) return null; + const poolActivated = isGenericOAuthFailoverEnabled(config, route.providerName); + try { + // Fence the durable health mark to the exact credential generation sent upstream. + if (!await markAccountNeedsReauthIfGeneration( + route.providerName, sent.accountId, sent.generation, undefined, "verify_account", + )) return null; + } catch { return null; } + if (options.abortSignal?.aborted || !poolActivated || antigravityValidationRotationAttempted + || transportState.genericFailovers >= transportState.genericFailoverLimit + || sidecarHasCommittedOutput()) return null; + + const hop = reserveCredentialHop( + "auth-recovery", + `${route.providerName}|${route.modelId}|sidecar-oauth-verify`, + ); + if (!hop.allowed) return null; + antigravityValidationRotationAttempted = true; + const nextAccountId = rotateAntigravityAccountOnAuthRefusal( + poolActivated, sent.accountId, sent.generation, route.modelId, + ); + if (!nextAccountId) { + hop.permit?.release(); + return null; + } + try { + // Keep the sibling's bearer and its account-matched project together; never inherit the + // quarantined account's identity or project. + const snapshot = await failoverAccountSnapshot(route.providerName, nextAccountId); + if (options.abortSignal?.aborted || !await applyFailoverSnapshot(snapshot, retryParsed) + || transportState.replayOAuthCredentialSnapshot?.accountId !== nextAccountId) { + hop.permit?.release(); + return null; + } + transportState.genericFailovers += 1; + } catch { + hop.permit?.release(); + return null; + } + recoveryKind = "oauth-account-403"; + sendBudgetState.pendingHopPermit = hop.permit; + } const rotated = !originalResponse || originalResponse.status === 429 ? rotateProviderTransportOn429(config, route.providerName, route.provider, { retryAfter, @@ -212,7 +287,9 @@ export async function executeResponsesSidecars( attemptedKey: route.provider.apiKey, promptCacheKey: parsed.options.promptCacheKey, }) : null; - if (rotated) { + if (antigravityVerification) { + // `applyFailoverSnapshot` has already rebound the route to the sibling identity/project. + } else if (rotated) { route.provider = rotated; } else if ( // A POSITIVE gate, not an early return. An early `return null` here made every later arm @@ -321,6 +398,7 @@ export async function executeResponsesSidecars( providerName: route.providerName, provider: route.provider, adapterName: rotatedAdapter.name, + ...(antigravityVerification ? { oauthCredentialSnapshot: transportState.replayOAuthCredentialSnapshot } : {}), }); } bindRouteReasoningReplayScope({ @@ -328,6 +406,7 @@ export async function executeResponsesSidecars( providerName: route.providerName, provider: route.provider, adapterName: rotatedAdapter.name, + ...(antigravityVerification ? { oauthCredentialSnapshot: transportState.replayOAuthCredentialSnapshot } : {}), }); return { adapter: rotatedAdapter, recoveryKind }; }; @@ -484,6 +563,11 @@ export async function executeResponsesSidecars( abortSignal: options.abortSignal, translatorBudget, providerFetch: routedProviderFetch, + ...(route.providerName === "google-antigravity" ? { + sendBudget: sendBudgetState.adapterDispatchBudget, + onPhysicalSend: (send: { ordinal: number; recovery?: AttemptRecoveryKind }) => + sendBudgetState.noteAdapterPhysicalSend(logCtx.usageLogInputTokens, send), + } : {}), }, backend: wsPlan.backend, forwardProvider: wsPlan.forwardSidecar?.provider, @@ -516,6 +600,13 @@ export async function executeResponsesSidecars( stallTimeoutSec: wsPlan.stallTimeoutSec, streamRoutedModelOutput: wsPlan.streamRoutedModelOutput, on429: rotateSidecarProviderOn429, + ...(route.providerName === "google-antigravity" ? { onIterationEnd: () => { + // The next Antigravity adapter dispatch claims this permit. If request construction, + // admission or cancellation stopped it first, return the unused send allowance. + const permit = sendBudgetState.pendingHopPermit; + sendBudgetState.pendingHopPermit = undefined; + permit?.release(); + } } : {}), retryOn429Policy: route.providerName === "kiro" && isGenericOAuthFailoverEnabled(config, "kiro") ? null : rateLimitRetryPolicyFor(route.provider), onCompletedResponse: response => { diff --git a/src/web-search/loop.ts b/src/web-search/loop.ts index 940ef296d1f..5bb2c017e62 100644 --- a/src/web-search/loop.ts +++ b/src/web-search/loop.ts @@ -353,6 +353,8 @@ export interface WebSearchLoopDeps { | { adapter: ProviderAdapter; recoveryKind: AttemptRecoveryKind } | null | Promise<{ adapter: ProviderAdapter; recoveryKind: AttemptRecoveryKind } | null>; + /** Release a credential-hop permit if this iteration ended before the adapter used it. */ + onIterationEnd?: () => void; /** Opt-in same-target 429 policy (key-auth providers). When present, 429 replays on the SAME key before on429 rotation. */ retryOn429Policy?: Required | null; /** Called only when the final bridged Responses stream reaches completed or incomplete. */ @@ -501,6 +503,8 @@ export async function runWithWebSearch(deps: WebSearchLoopDeps): Promise { }; } +function sseSuccessBody(text: string): string { + return `data: ${JSON.stringify(jsonSuccessBody(text))}\n\n`; +} + function transient429ErrorBody(): Record { return { error: { @@ -208,6 +212,188 @@ describe("Google Antigravity OAuth 429 retry and multi-account budget (#5880)", expect(sendBudget.used).toBe(1); }); + test.each([ + { error: { status: "PERMISSION_DENIED", message: "validate", details: [{ reason: "VALIDATION_REQUIRED" }] } }, + { error: { status: "PERMISSION_DENIED", message: "Verify your account to continue." } }, + { error: { status: "PERMISSION_DENIED", message: "Please verify your account to continue using Antigravity." } }, + ])("web_search rotates only a recognized verification 403 with the sibling identity and project", async refusal => { + const accounts = await seedAntigravityAccounts(2); + const cfg = antigravityConfig(); + cfg.webSearchSidecar = { backend: "exa", exaApiKey: "synthetic-exa-key" }; + saveConfig(cfg); + const observedSends: Array<{ auth: string; project: string }> = []; + installAntigravityFetchMock(({ auth, project, sendIndex }) => { + observedSends.push({ auth, project }); + if (sendIndex === 1) return new Response(JSON.stringify(refusal), { status: 403 }); + return new Response(sseSuccessBody("sibling answered"), { + status: 200, headers: { "content-type": "text/event-stream" }, + }); + }); + const sendBudget = createRequestExecutionBudget({ + maxTotalModelSends: 2, baseSendAllowance: 2, finalRecoveryAllowance: 0, + maxAlternateTargetSends: 0, maxTargetTransitions: 0, + }); + + const response = await handleResponses( + createResponsesRequest({ tools: [{ type: "web_search" }] }), + cfg, + { model: "gemini-3.8-flash", provider: "google-antigravity" }, + { sendBudget }, + ); + + expect(response.status).toBe(200); + expect(await response.text()).toContain("sibling answered"); + expect(observedSends).toEqual([ + { auth: accounts[0]!.auth, project: accounts[0]!.project }, + { auth: accounts[1]!.auth, project: accounts[1]!.project }, + ]); + expect(getAccountSet("google-antigravity")!.accounts.find(row => row.id === accounts[0]!.id)) + .toMatchObject({ needsReauth: true, needsReauthReason: "verify_account" }); + expect(sendBudget.used).toBe(2); + }); + + test("web_search leaves an unrelated 403 intact and does not quarantine the account", async () => { + const accounts = await seedAntigravityAccounts(2); + const cfg = antigravityConfig(); + cfg.webSearchSidecar = { backend: "exa", exaApiKey: "synthetic-exa-key" }; + saveConfig(cfg); + const observedSends: Array<{ auth: string; project: string }> = []; + installAntigravityFetchMock(({ auth, project }) => { + observedSends.push({ auth, project }); + return new Response(JSON.stringify({ error: { + status: "PERMISSION_DENIED", message: "The requested feature is not available to this account.", + } }), { status: 403 }); + }); + + const response = await handleResponses( + createResponsesRequest({ tools: [{ type: "web_search" }] }), + cfg, + { model: "gemini-3.8-flash", provider: "google-antigravity" }, + ); + + expect(response.status).toBe(403); + expect(await response.text()).toContain("access denied"); + expect(observedSends).toEqual([{ auth: accounts[0]!.auth, project: accounts[0]!.project }]); + expect(getAccountSet("google-antigravity")!.accounts.find(row => row.id === accounts[0]!.id)?.needsReauth).toBeFalsy(); + }); + + test("web_search cannot spend a second send when the caller budget is exhausted", async () => { + const accounts = await seedAntigravityAccounts(2); + const cfg = antigravityConfig(); + cfg.webSearchSidecar = { backend: "exa", exaApiKey: "synthetic-exa-key" }; + saveConfig(cfg); + const observedSends: Array<{ auth: string; project: string }> = []; + installAntigravityFetchMock(({ auth, project }) => { + observedSends.push({ auth, project }); + return new Response(JSON.stringify({ error: { status: "PERMISSION_DENIED", message: "validate", + details: [{ reason: "VALIDATION_REQUIRED" }] } }), { status: 403 }); + }); + const sendBudget = createRequestExecutionBudget({ + maxTotalModelSends: 1, baseSendAllowance: 1, finalRecoveryAllowance: 0, + maxAlternateTargetSends: 0, maxTargetTransitions: 0, + }); + + const response = await handleResponses( + createResponsesRequest({ tools: [{ type: "web_search" }] }), + cfg, + { model: "gemini-3.8-flash", provider: "google-antigravity" }, + { sendBudget }, + ); + + expect(response.status).toBe(403); + expect(await response.text()).toContain("Antigravity account validation required"); + expect(observedSends).toEqual([{ auth: accounts[0]!.auth, project: accounts[0]!.project }]); + expect(sendBudget.used).toBe(1); + }); + + test("web_search cancellation after the refusal sends no sibling request", async () => { + await seedAntigravityAccounts(2); + const cfg = antigravityConfig(); + cfg.webSearchSidecar = { backend: "exa", exaApiKey: "synthetic-exa-key" }; + saveConfig(cfg); + const abort = new AbortController(); + let sends = 0; + installAntigravityFetchMock(() => { + sends += 1; + abort.abort(); + return new Response(JSON.stringify({ error: { status: "PERMISSION_DENIED", message: "validate", + details: [{ reason: "VALIDATION_REQUIRED" }] } }), { status: 403 }); + }); + + const response = await handleResponses( + createResponsesRequest({ tools: [{ type: "web_search" }] }), + cfg, + { model: "gemini-3.8-flash", provider: "google-antigravity" }, + { abortSignal: abort.signal }, + ); + + expect(sends).toBe(1); + await response.text(); + }); + + test("a stale credential generation is not quarantined or replayed from the web_search path", async () => { + const accounts = await seedAntigravityAccounts(2); + const cfg = antigravityConfig(); + cfg.webSearchSidecar = { backend: "exa", exaApiKey: "synthetic-exa-key" }; + saveConfig(cfg); + const observedSends: Array<{ auth: string; project: string }> = []; + installAntigravityFetchMock(async ({ auth, project, sendIndex }) => { + observedSends.push({ auth, project }); + if (sendIndex === 1) { + await saveCredential("google-antigravity", { + access: "relogin-token", refresh: "relogin-refresh", expires: Date.now() + 3_600_000, + accountId: "account-1", projectId: "project-1", + }); + return new Response(JSON.stringify({ error: { status: "PERMISSION_DENIED", message: "validate", + details: [{ reason: "VALIDATION_REQUIRED" }] } }), { status: 403 }); + } + return new Response(JSON.stringify(jsonSuccessBody("unexpected sibling")), { status: 200 }); + }); + + const response = await handleResponses( + createResponsesRequest({ tools: [{ type: "web_search" }] }), + cfg, + { model: "gemini-3.8-flash", provider: "google-antigravity" }, + ); + + expect(response.status).toBe(403); + await response.text(); + expect(observedSends).toHaveLength(1); + const failed = getAccountSet("google-antigravity")!.accounts.find(row => row.id === accounts[0]!.id)!; + expect(failed.needsReauth).toBeFalsy(); + expect(failed.needsReauthReason).toBeUndefined(); + }); + + test("web_search marks the refused account but preserves the 403 when no sibling has a project", async () => { + const accounts = await seedAntigravityAccounts(1); + const cfg = antigravityConfig(); + cfg.webSearchSidecar = { backend: "exa", exaApiKey: "synthetic-exa-key" }; + saveConfig(cfg); + await saveCredential("google-antigravity", { + access: "no-project-token", refresh: "no-project-refresh", expires: Date.now() + 3_600_000, + accountId: "no-project-account", + }, { addAccount: true }); + await setActiveAccount("google-antigravity", accounts[0]!.id); + const observedSends: Array<{ auth: string; project: string }> = []; + installAntigravityFetchMock(({ auth, project }) => { + observedSends.push({ auth, project }); + return new Response(JSON.stringify({ error: { status: "PERMISSION_DENIED", message: "validate", + details: [{ reason: "VALIDATION_REQUIRED" }] } }), { status: 403 }); + }); + + const response = await handleResponses( + createResponsesRequest({ tools: [{ type: "web_search" }] }), + cfg, + { model: "gemini-3.8-flash", provider: "google-antigravity" }, + ); + + expect(response.status).toBe(403); + expect(await response.text()).toContain("Antigravity account validation required"); + expect(observedSends).toEqual([{ auth: accounts[0]!.auth, project: accounts[0]!.project }]); + expect(getAccountSet("google-antigravity")!.accounts.find(row => row.id === accounts[0]!.id)) + .toMatchObject({ needsReauth: true, needsReauthReason: "verify_account" }); + }); + test.each([4, 5])("%i accounts each receive three transient sends before terminal 429", async accountCount => { const accounts = await seedAntigravityAccounts(accountCount); const cfg = antigravityConfig(); diff --git a/tests/web-search/web-search-recovery-kind.test.ts b/tests/web-search/web-search-recovery-kind.test.ts index 980fb9f718b..73053601323 100644 --- a/tests/web-search/web-search-recovery-kind.test.ts +++ b/tests/web-search/web-search-recovery-kind.test.ts @@ -90,3 +90,53 @@ test("429 rotation reports the rotator's recovery kind", async () => { expect(await recoveryKindsFor(next => ({ adapter: next, recoveryKind: "key-429" }))) .toEqual([undefined, "key-429"]); }); + +test("web-search does not replay a verification refusal after routed output was committed", async () => { + globalThis.fetch = (async () => Response.json({ + results: [{ title: "Fixture", url: "https://example.test/result", content: "synthetic result" }], + })) as typeof fetch; + let routedSends = 0; + let rotations = 0; + const adapter: ProviderAdapter = { + name: "mock-antigravity", + buildRequest: () => ({ url: "https://routed.test/v1", method: "POST", headers: {}, body: "{}" }), + fetchResponse: async () => { + routedSends++; + if (routedSends === 1) return new Response("ok", { status: 200 }); + return new Response(JSON.stringify({ error: { + status: "PERMISSION_DENIED", message: "validate", details: [{ reason: "VALIDATION_REQUIRED" }], + } }), { status: 403 }); + }, + async *parseStream() { + yield { type: "text_delta", text: "I will check. " } satisfies AdapterEvent; + yield { type: "tool_call_start", id: "search-1", name: "web_search" } satisfies AdapterEvent; + yield { type: "tool_call_delta", arguments: JSON.stringify({ query: "fixture query" }) } satisfies AdapterEvent; + yield { type: "tool_call_end" } satisfies AdapterEvent; + yield { type: "done" } satisfies AdapterEvent; + }, + async parseResponse() { throw new Error("parseResponse must be unreachable"); }, + }; + + const response = await runWithWebSearch({ + parsed: parseRequest({ model: "routed/model", input: "hi", stream: true, tools: [{ type: "web_search" }] }), + adapter, + incomingMeta: { headers: new Headers(), providerName: "google-antigravity", translatorBudget: createTestTranslatorBudget() }, + backend: "exa", + exaApiKey: "synthetic-exa-key", + hostedTool: { type: "web_search" }, + selectedForwardHeaders: new Headers(), + settings: { model: "exa-fixture-model", reasoning: "low", timeoutMs: 30_000 }, + maxSearches: 1, + streamRoutedModelOutput: true, + on429: () => { rotations++; return null; }, + }); + const frames = await collectSse(response.body!); + const text = frames + .filter(frame => frame.data.type === "response.output_text.delta") + .map(frame => String(frame.data.delta ?? "")) + .join(""); + expect(text).toContain("I will check."); + expect(routedSends).toBe(2); + expect(rotations).toBe(0); + expect(frames.some(frame => frame.event === "response.failed")).toBe(true); +}); From 14ae987f9ca58685d7d1f5c4edcc31e3a08a4037 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=87=8D=E9=9F=B3?= Date: Wed, 7 Oct 2026 19:09:11 +0800 Subject: [PATCH 2/5] refactor(antigravity): share validation refusal predicate --- src/server/responses/antigravity-validation-refusal.ts | 7 ++++++- src/server/responses/sidecar-execution.ts | 8 +++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/server/responses/antigravity-validation-refusal.ts b/src/server/responses/antigravity-validation-refusal.ts index 02430f26593..b9ba85b12cf 100644 --- a/src/server/responses/antigravity-validation-refusal.ts +++ b/src/server/responses/antigravity-validation-refusal.ts @@ -1,6 +1,11 @@ import { ANTIGRAVITY_VALIDATION_REQUIRED_PREFIX } from "../../adapters/google-errors"; import { readBoundedResponseBody } from "../../lib/bounded-body"; +/** Match only the normalized Antigravity validation-refusal marker. */ +export function hasAntigravityValidationRefusalMarker(text: string): boolean { + return text.startsWith(`${ANTIGRAVITY_VALIDATION_REQUIRED_PREFIX}: `); +} + /** Inspect only the Google adapter's normalized, already bounded 403 response. */ export async function isAntigravityValidationRefusal(response: Response, signal?: AbortSignal): Promise { if (response.status !== 403 || signal?.aborted) return false; @@ -13,7 +18,7 @@ export async function isAntigravityValidationRefusal(response: Response, signal? signal, }); return !signal?.aborted && body.displaySafe && !body.truncated && !body.timedOut && !body.oversized - && body.text.startsWith(`${ANTIGRAVITY_VALIDATION_REQUIRED_PREFIX}: `); + && hasAntigravityValidationRefusalMarker(body.text); } catch { return false; } diff --git a/src/server/responses/sidecar-execution.ts b/src/server/responses/sidecar-execution.ts index 80d4316be5a..f471a577f38 100644 --- a/src/server/responses/sidecar-execution.ts +++ b/src/server/responses/sidecar-execution.ts @@ -29,7 +29,7 @@ import { failoverAccountSnapshot, } from "../../oauth/generic-account-failover"; import { classifyKiroRefusal } from "../../adapters/kiro-refusal"; -import { ANTIGRAVITY_VALIDATION_REQUIRED_PREFIX, safeAntigravityHttpErrorMessage } from "../../adapters/google-errors"; +import { safeAntigravityHttpErrorMessage } from "../../adapters/google-errors"; import { markAccountNeedsReauthIfGeneration } from "../../oauth/store"; import { noteKiroMonthlyRefusal, noteKiroServedSuccess } from "../../providers/kiro-usage"; import { persistKiroAccountState } from "../../providers/kiro-account-state-disk"; @@ -48,6 +48,7 @@ import { recordAdapterReasoning, recordAdapterTier } from "../request-log"; import { normalizeLogConversationId } from "../request-log-conversation"; import { rememberResponseState } from "../../responses/state"; import { trackStreamLifetime } from "../lifecycle"; +import { hasAntigravityValidationRefusalMarker } from "./antigravity-validation-refusal"; /** One responsibility of the Responses request pipeline; state owners are explicit. */ export async function executeResponsesSidecars( @@ -215,8 +216,9 @@ export async function executeResponsesSidecars( // The structured reason is authoritative. For older upstream envelopes, accept only the // exact verification message in error.message; unrelated 403 text and other fields do not // convict an account. - antigravityVerification = safeAntigravityHttpErrorMessage(403, body) - .startsWith(`${ANTIGRAVITY_VALIDATION_REQUIRED_PREFIX}: `) || exactMessage; + antigravityVerification = hasAntigravityValidationRefusalMarker( + safeAntigravityHttpErrorMessage(403, body), + ) || exactMessage; if (!antigravityVerification) return null; } const refusal = route.providerName === "kiro" && originalResponse From 6dfff855dca5c3acfc5fcc5cfa7a276271f928ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=87=8D=E9=9F=B3?= Date: Wed, 7 Oct 2026 19:18:16 +0800 Subject: [PATCH 3/5] test(antigravity): assert cancelled refusal stays healthy --- .../server-google-antigravity-oauth-429-budget.test.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/server/server-google-antigravity-oauth-429-budget.test.ts b/tests/server/server-google-antigravity-oauth-429-budget.test.ts index cc3b773f89c..b4a7c790646 100644 --- a/tests/server/server-google-antigravity-oauth-429-budget.test.ts +++ b/tests/server/server-google-antigravity-oauth-429-budget.test.ts @@ -173,7 +173,7 @@ function installAntigravityFetchMock( describe("Google Antigravity OAuth 429 retry and multi-account budget (#5880)", () => { test("an observed caller abort before a validation refusal sends no sibling request", async () => { - await seedAntigravityAccounts(2); + const accounts = await seedAntigravityAccounts(2); const cfg = antigravityConfig(); saveConfig(cfg); const abort = new AbortController(); @@ -188,6 +188,10 @@ describe("Google Antigravity OAuth 429 retry and multi-account budget (#5880)", { model: "gemini-3.8-flash", provider: "google-antigravity" }, { abortSignal: abort.signal }); expect(sends).toBe(1); await response.text(); + const cancelledAccount = getAccountSet("google-antigravity")!.accounts + .find(row => row.id === accounts[0]!.id)!; + expect(cancelledAccount.needsReauth).toBeFalsy(); + expect(cancelledAccount.needsReauthReason).toBeUndefined(); }); test("a spent caller send budget keeps a classified 403 and sends no sibling request", async () => { From 0e90451c1eedd7259fac1783f4d220ae5bf789d6 Mon Sep 17 00:00:00 2001 From: JUN Date: Wed, 7 Oct 2026 22:53:43 +0900 Subject: [PATCH 4/5] fix: narrow Antigravity web-search rotation to structured refusals MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Match ordinary dispatch gates, keep durable account health unchanged, and charge Antigravity sidecar rotations once per physical send. Co-authored-by: 重音 --- .../docs/reference/cli/providers-accounts.md | 6 + .../docs/reference/configuration/providers.md | 8 +- scripts/test-layout/layout.json | 2 +- .../antigravity-validation-refusal.ts | 21 +- src/server/responses/sidecar-execution.ts | 52 +-- structure/providers-and-adapters.md | 2 +- structure/transports/inventory.md | 2 +- structure/transports/responses-failover.md | 2 +- tests/fixtures/test-layout-expected.json | 2 +- ...-antigravity-web-search-validation.test.ts | 415 ++++++++++++++++++ ...oogle-antigravity-oauth-429-budget.test.ts | 186 -------- 11 files changed, 465 insertions(+), 233 deletions(-) create mode 100644 tests/server/server-antigravity-web-search-validation.test.ts diff --git a/docs-site/src/content/docs/reference/cli/providers-accounts.md b/docs-site/src/content/docs/reference/cli/providers-accounts.md index 1c9cac81800..3f40362c515 100644 --- a/docs-site/src/content/docs/reference/cli/providers-accounts.md +++ b/docs-site/src/content/docs/reference/cli/providers-accounts.md @@ -246,6 +246,12 @@ when account failover is enabled, the request can retry on another eligible acco Google's account verification, then run `ocx login google-antigravity`. Silent token refresh does not clear this verification requirement. If OpenCodex cannot save the quarantine, this adapter exchange preserves the original `403` without another recovery send; the account has not been durably quarantined. If a sibling request cannot be built or admitted for sending, the exchange delivers the original `403` through normal error formatting. An enclosing combo or policy route can still apply its existing fallback rules. +For fetch-based web-search requests, automatic sibling rotation accepts only a complete, bounded +Google error envelope containing `error.details[].reason === "VALIDATION_REQUIRED"`. Verification +wording alone does not rotate. This request-scoped recovery uses the sibling's token and project +without persisting a `needs-reauth` mark; cancellation, committed output, budget exhaustion or an +unavailable sibling preserves the failure. + A proxy that is already running picks up the new credential without a restart: the CLI asks it to reload that one provider from disk, and the request carries no credential of its own. If the running proxy cannot accept that request — most often because it started from a build that predates diff --git a/docs-site/src/content/docs/reference/configuration/providers.md b/docs-site/src/content/docs/reference/configuration/providers.md index ce28f9ffaa3..1545bda5f0a 100644 --- a/docs-site/src/content/docs/reference/configuration/providers.md +++ b/docs-site/src/content/docs/reference/configuration/providers.md @@ -1019,7 +1019,13 @@ The same main dispatch may also switch once on a 403 when Google's bounded error finds a complete structured `VALIDATION_REQUIRED` reason. The 401 and 403 paths share one sibling attempt per request; an unrelated or incomplete 403 keeps its original error. A rejected sibling, cancellation or exhausted send budget does not cause another upstream send. -Continuations, native Responses passthrough, image and web-search sidecars, and output already sent +The fetch-based web-search loop also rotates once on this structured 403 reason, using the sibling's +OAuth token and matching Cloud Code Assist project. It accepts only complete, bounded Google error +envelopes with `error.details[].reason === "VALIDATION_REQUIRED"`; verification wording alone does +not authorize rotation, and this path does not persist a reauthentication mark. Each physical send, +including a 429 retry, consumes the same request budget exactly once. A disabled pool, cancellation, +exhausted budget or unavailable sibling preserves the failure. +Continuations, native Responses passthrough, image sidecars, and output already sent to the client do not use this rotation. Current scope is the ordinary Responses request paths. Cursor reports rate limits as adapter diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index 5a6f6c0f21f..da650c7ba9b 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -1667,7 +1667,7 @@ "server-combo-reasoning-replay-eligibility.test.ts": "server", "server-combo-zero-output-failover.test.ts": "server", "server-google-antigravity-oauth-401-replay.test.ts": "server", - "server-google-antigravity-oauth-429-budget.test.ts": "server", + "server-antigravity-web-search-validation.test.ts": "server", "server-google-antigravity-oauth-429-budget.test.ts": "server", "server-gui-bundle-freshness.test.ts": "server", "server-images-bodyless-content-length.test.ts": "server", "server-images-pool-admission.test.ts": "server", diff --git a/src/server/responses/antigravity-validation-refusal.ts b/src/server/responses/antigravity-validation-refusal.ts index b9ba85b12cf..7c3512698a4 100644 --- a/src/server/responses/antigravity-validation-refusal.ts +++ b/src/server/responses/antigravity-validation-refusal.ts @@ -1,4 +1,4 @@ -import { ANTIGRAVITY_VALIDATION_REQUIRED_PREFIX } from "../../adapters/google-errors"; +import { ANTIGRAVITY_VALIDATION_REQUIRED_PREFIX, safeAntigravityHttpErrorMessage } from "../../adapters/google-errors"; import { readBoundedResponseBody } from "../../lib/bounded-body"; /** Match only the normalized Antigravity validation-refusal marker. */ @@ -8,17 +8,26 @@ export function hasAntigravityValidationRefusalMarker(text: string): boolean { /** Inspect only the Google adapter's normalized, already bounded 403 response. */ export async function isAntigravityValidationRefusal(response: Response, signal?: AbortSignal): Promise { + return inspectValidationRefusal(response, signal, false); +} + +/** Sidecar dispatch retains the raw Google envelope; normalize only a complete bounded body. */ +export async function isAntigravityRawValidationRefusal(response: Response, signal?: AbortSignal): Promise { + return inspectValidationRefusal(response, signal, true); +} + +async function inspectValidationRefusal(response: Response, signal: AbortSignal | undefined, raw: boolean): Promise { if (response.status !== 403 || signal?.aborted) return false; try { const body = await readBoundedResponseBody(response.clone(), { - maxBytes: 1024, - totalTimeoutMs: 1000, - firstByteTimeoutMs: 1000, - inactivityTimeoutMs: 1000, + maxBytes: raw ? 4096 : 1024, + totalTimeoutMs: raw ? 2000 : 1000, + firstByteTimeoutMs: raw ? 2000 : 1000, + inactivityTimeoutMs: raw ? 2000 : 1000, signal, }); return !signal?.aborted && body.displaySafe && !body.truncated && !body.timedOut && !body.oversized - && hasAntigravityValidationRefusalMarker(body.text); + && hasAntigravityValidationRefusalMarker(raw ? safeAntigravityHttpErrorMessage(403, body.text) : body.text); } catch { return false; } diff --git a/src/server/responses/sidecar-execution.ts b/src/server/responses/sidecar-execution.ts index f471a577f38..65d8872aaa4 100644 --- a/src/server/responses/sidecar-execution.ts +++ b/src/server/responses/sidecar-execution.ts @@ -29,8 +29,7 @@ import { failoverAccountSnapshot, } from "../../oauth/generic-account-failover"; import { classifyKiroRefusal } from "../../adapters/kiro-refusal"; -import { safeAntigravityHttpErrorMessage } from "../../adapters/google-errors"; -import { markAccountNeedsReauthIfGeneration } from "../../oauth/store"; +import { isNonReplayableResponse } from "../../lib/upstream-retry"; import { noteKiroMonthlyRefusal, noteKiroServedSuccess } from "../../providers/kiro-usage"; import { persistKiroAccountState } from "../../providers/kiro-account-state-disk"; import { readDisplaySafeErrorText } from "./core-errors"; @@ -48,7 +47,7 @@ import { recordAdapterReasoning, recordAdapterTier } from "../request-log"; import { normalizeLogConversationId } from "../request-log-conversation"; import { rememberResponseState } from "../../responses/state"; import { trackStreamLifetime } from "../lifecycle"; -import { hasAntigravityValidationRefusalMarker } from "./antigravity-validation-refusal"; +import { isAntigravityRawValidationRefusal } from "./antigravity-validation-refusal"; /** One responsibility of the Responses request pipeline; state owners are explicit. */ export async function executeResponsesSidecars( @@ -92,6 +91,8 @@ export async function executeResponsesSidecars( sendBudgetState: Pick, ) { const { config, options, logCtx } = requestContext; + const antigravityPoolActivated = requestState.route.providerName === "google-antigravity" + && isGenericOAuthFailoverEnabled(config, requestState.route.providerName); const { applyFailoverSnapshot, anthropicSessionKey, @@ -201,24 +202,12 @@ export async function executeResponsesSidecars( && !antigravityValidationResponse && originalResponse && originalResponse.status !== 429) return null; let antigravityVerification = false; if (antigravityValidationResponse) { - if (options.abortSignal?.aborted) return null; - const body = await readDisplaySafeErrorText( - originalResponse.clone(), options.abortSignal ?? new AbortController().signal, "", - ); - if (options.abortSignal?.aborted) return null; - let exactMessage = false; - try { - const envelope = JSON.parse(body) as { error?: { message?: unknown } }; - const message = envelope.error?.message; - exactMessage = typeof message === "string" - && /^(?:Please )?verify your account to continue(?: using Antigravity)?[.!]?$/i.test(message.trim()); - } catch { /* an unparseable body cannot authorize account quarantine */ } - // The structured reason is authoritative. For older upstream envelopes, accept only the - // exact verification message in error.message; unrelated 403 text and other fields do not - // convict an account. - antigravityVerification = hasAntigravityValidationRefusalMarker( - safeAntigravityHttpErrorMessage(403, body), - ) || exactMessage; + if (transportState.adapter.name !== "google" || !antigravityPoolActivated + || options.abortSignal?.aborted || isNonReplayableResponse(originalResponse) + || antigravityValidationRotationAttempted + || transportState.genericFailovers >= transportState.genericFailoverLimit + || sidecarHasCommittedOutput()) return null; + antigravityVerification = await isAntigravityRawValidationRefusal(originalResponse, options.abortSignal); if (!antigravityVerification) return null; } const refusal = route.providerName === "kiro" && originalResponse @@ -241,16 +230,7 @@ export async function executeResponsesSidecars( const sent = transportState.replayOAuthCredentialSnapshot; const failedAccountId = transportState.genericFailoverAccountId; if (!sent || !failedAccountId || sent.accountId !== failedAccountId) return null; - const poolActivated = isGenericOAuthFailoverEnabled(config, route.providerName); - try { - // Fence the durable health mark to the exact credential generation sent upstream. - if (!await markAccountNeedsReauthIfGeneration( - route.providerName, sent.accountId, sent.generation, undefined, "verify_account", - )) return null; - } catch { return null; } - if (options.abortSignal?.aborted || !poolActivated || antigravityValidationRotationAttempted - || transportState.genericFailovers >= transportState.genericFailoverLimit - || sidecarHasCommittedOutput()) return null; + if (options.abortSignal?.aborted || sidecarHasCommittedOutput()) return null; const hop = reserveCredentialHop( "auth-recovery", @@ -259,7 +239,7 @@ export async function executeResponsesSidecars( if (!hop.allowed) return null; antigravityValidationRotationAttempted = true; const nextAccountId = rotateAntigravityAccountOnAuthRefusal( - poolActivated, sent.accountId, sent.generation, route.modelId, + antigravityPoolActivated, sent.accountId, sent.generation, route.modelId, ); if (!nextAccountId) { hop.permit?.release(); @@ -267,7 +247,7 @@ export async function executeResponsesSidecars( } try { // Keep the sibling's bearer and its account-matched project together; never inherit the - // quarantined account's identity or project. + // failed account's identity or project. const snapshot = await failoverAccountSnapshot(route.providerName, nextAccountId); if (options.abortSignal?.aborted || !await applyFailoverSnapshot(snapshot, retryParsed) || transportState.replayOAuthCredentialSnapshot?.accountId !== nextAccountId) { @@ -304,7 +284,8 @@ export async function executeResponsesSidecars( ) { // Intersection with the request's shared budget. The sidecar replay is dispatched by the // web-search/image loop and never reaches `onSendsConsumed`, so this reservation is the - // charge; a refusal returns null and the caller keeps the real 429 it already has. + // charge for other sidecars. Antigravity web search hands its reservation to the adapter + // physical-send owner instead; a refusal preserves the real 429. const hop = reserveCredentialHop( "auth-recovery", `${route.providerName}|${route.modelId}|sidecar-oauth-429`, @@ -338,7 +319,8 @@ export async function executeResponsesSidecars( return null; } recoveryKind = "oauth-account-429"; - hop.permit?.use(); + if (canRunWebSearch && route.providerName === "google-antigravity") sendBudgetState.pendingHopPermit = hop.permit; + else hop.permit?.use(); } else if ( // Anthropic's pool is excluded from generic failover, so without this arm a 429 inside a // web-search or image-bridge turn was terminal even with the pool fully enabled -- while diff --git a/structure/providers-and-adapters.md b/structure/providers-and-adapters.md index 3bd23a59d0a..ad1b3616bb7 100644 --- a/structure/providers-and-adapters.md +++ b/structure/providers-and-adapters.md @@ -36,7 +36,7 @@ Direct MCP names emitted in a verified custom code-mode catalog follow the [Responses restoration boundary](transports/responses-wire-shapes.md#direct-mcp-calls-in-code-mode). Ordinary structured functions named `exec` do not opt into this compatibility path. -RunTurn hosted search uses `src/web-search/run-turn-loop.ts`: synthetic calls remain private, progress reaches the bridge during collection, and a validated terminal precedes search execution. Complete search calls remain actionable at a truncated `done`; cancellation prevents subsequent queries and calls. OAuth preflight replay in `src/server/responses/run-turn-execution.ts` retains the synthetic tool while refreshing credential-scoped route state. In `src/server/responses/sidecar-execution.ts`, a search plan takes priority over image/video bridge execution for both transports; only fetch-capable adapters enter the fetch search loop. +RunTurn hosted search uses `src/web-search/run-turn-loop.ts`: synthetic calls remain private, progress reaches the bridge during collection, and a validated terminal precedes search execution. Complete search calls remain actionable at a truncated `done`; cancellation prevents subsequent queries and calls. OAuth preflight replay in `src/server/responses/run-turn-execution.ts` retains the synthetic tool while refreshing credential-scoped route state. In `src/server/responses/sidecar-execution.ts`, a search plan takes priority over image/video bridge execution for both transports; only fetch-capable adapters enter the fetch search loop. Its Antigravity structured-403 rotation and physical-send accounting follow [Account refusal and rotation boundaries](transports/responses-failover.md#account-refusal-and-rotation-boundaries). Both search loops keep `web_search` declared after the search budget is exhausted. Further calls receive a paired limit-reached result without another physical search; at most `maxSearches + 3` model iterations run before a terminal error. Ordinary caller tools and cancellation still end the loop. Empty-answer recovery alone removes all tools. diff --git a/structure/transports/inventory.md b/structure/transports/inventory.md index 4d00e1fedd1..762992198ee 100644 --- a/structure/transports/inventory.md +++ b/structure/transports/inventory.md @@ -43,7 +43,7 @@ surface is listed here so a maintainer can find the owner without grepping: | Image/video generation loop | `src/images/loop.ts`, `src/images/plan.ts`, `src/images/fulfill.ts`, `src/images/xai-client.ts`, `src/images/xai-video-client.ts`, `src/images/artifacts.ts` | A provider-returned image URL is downloaded into a local artifact once, then served locally; warnings stay URL-free because provider CDN URLs may embed credentials. Artifact downloads go through the pinned-IP transport with a 10 s connect deadline (`DOWNLOAD_CONNECT_TIMEOUT_MS`) that bounds TCP/TLS setup on its own, in addition to the 60 s idle timer, and `pinnedHttpsGet` accepts a per-call `connectTimeoutMs`. | | GitHub Copilot | `src/providers/xai-transport.ts` (`resolveProviderTransport`), `src/providers/github-copilot-transport.ts` | `resolveProviderTransport` selects the Copilot transport when the routed provider name is `github-copilot`; the Copilot module then resolves its headers and base URL, and the registry seeds the provider row and model fallback. | | API-key pools | `src/providers/api-key-selection.ts`, `src/providers/key-failover.ts` | A configured `apiKeyPoolStrategy` plus a cooling committed key rotates before the first send (`selectProactiveApiKeyTransport`); a 429 still rotates after the send and records a cooldown. `provider.apiKey` keeps mirroring the active entry so routing stays single-key. The pick is inert without a strategy or while the committed key is healthy. | -| OAuth account failover | `src/oauth/generic-account-failover.ts`, `src/oauth/anthropic-routing.ts`, `src/server/responses/antigravity-validation-refusal.ts`, `src/adapters/antigravity-refusal.ts` | Reactive pre-output 429 recovery is presence-driven with 2+ eligible accounts. Anthropic also recovers classified pre-output account-entitlement 403s under its [account-pool contract](../providers/anthropic-account-pool.md#account-entitlement-refusal-recovery), and records the refused account's cooldown even when the request has no retry sends left. Primary Antigravity inference through the Google adapter main dispatch may send once on a live sibling after a replayed 401, a terminal credential-refresh failure that marks the account for reauthentication, or a normalized structured `VALIDATION_REQUIRED` 403, within the existing budget. A bounded message-worded verification 403 persists a generation-fenced `verify_account` cause in `src/oauth/store.ts` and `src/oauth/health.ts`, including singleton and last-account refusals. It reads as `needs-reauth(verify)` in `ocx account list`; silent refresh preserves quarantine, while explicit re-login clears it and retires its failover cooldown. Quarantine does not require a retry slot; if persisting it fails, dispatch exits this adapter exchange’s recovery loop and preserves the original bounded 403 without another recovery send in that exchange; enclosing combo and policy fallback rules remain unchanged. A preserved original 403 after sibling build or pre-send admission failure also exits recovery through common bounded error formatting and the combo failure callback; newly constructed failure responses keep their own delivery. Successful sibling replay uses the captured pool activation and the existing send budget. The 401 and structured 403 sibling paths share one request guard; unrelated 403s and transient refresh failures do not rotate. Continuations, native passthrough and sidecars keep their existing behavior. Generic OAuth accounts can be operator-paused; paused accounts are excluded from selection, failover, and proactive Token Guardian refresh. If every account is paused, the request fails with 403 rather than as a login failure. Pool and `oauthAccountFailover` flags govern proactive routing, not the reactive retry: a disabled Anthropic pool recovers through quota ordering rather than its dormant strategy, a per-provider `enabled` beats the global default in either direction, and a non-positive fill-first threshold disables proactive usage-based rotation. Kiro's `kiroAutoSelection` projects the same candidate eligibility into routing and account-list status; unknown evidence stays eligible, while pause, reauth, suspension, cooldown, and confirmed exhaustion have closed reasons. An active singleton or a pool excluded only from automatic Kiro selection may still attempt its active account; an explicitly paused active account is never dispatched, and an all-paused pool fails with 403. A configured Kiro capacity cap can also time out. Kiro's optional process-local account lease is released at response completion or cancellation; other providers retain their admission path. | +| OAuth account failover | `src/oauth/generic-account-failover.ts`, `src/oauth/anthropic-routing.ts`, `src/server/responses/antigravity-validation-refusal.ts`, `src/adapters/antigravity-refusal.ts` | Reactive pre-output 429 recovery is presence-driven with 2+ eligible accounts. Anthropic also recovers classified pre-output account-entitlement 403s under its [account-pool contract](../providers/anthropic-account-pool.md#account-entitlement-refusal-recovery), and records the refused account's cooldown even when the request has no retry sends left. Primary Antigravity inference through the Google adapter main dispatch may send once on a live sibling after a replayed 401, a terminal credential-refresh failure that marks the account for reauthentication, or a normalized structured `VALIDATION_REQUIRED` 403, within the existing budget. A bounded message-worded verification 403 persists a generation-fenced `verify_account` cause in `src/oauth/store.ts` and `src/oauth/health.ts`, including singleton and last-account refusals. It reads as `needs-reauth(verify)` in `ocx account list`; silent refresh preserves quarantine, while explicit re-login clears it and retires its failover cooldown. Quarantine does not require a retry slot; if persisting it fails, dispatch exits this adapter exchange’s recovery loop and preserves the original bounded 403 without another recovery send in that exchange; enclosing combo and policy fallback rules remain unchanged. A preserved original 403 after sibling build or pre-send admission failure also exits recovery through common bounded error formatting and the combo failure callback; newly constructed failure responses keep their own delivery. Successful sibling replay uses the captured pool activation and the existing send budget. The 401 and structured 403 sibling paths share one request guard; unrelated 403s and transient refresh failures do not rotate. The fetch web-search sidecar accepts only complete bounded structured `VALIDATION_REQUIRED` 403s through the Google normalizer, carries the sibling token/project without a durable reauthentication mark, and uses adapter-owned physical-send accounting; its contract is [Account refusal and rotation boundaries](responses-failover.md#account-refusal-and-rotation-boundaries). Continuations, native passthrough and image sidecars keep their existing behavior. Generic OAuth accounts can be operator-paused; paused accounts are excluded from selection, failover, and proactive Token Guardian refresh. If every account is paused, the request fails with 403 rather than as a login failure. Pool and `oauthAccountFailover` flags govern proactive routing, not the reactive retry: a disabled Anthropic pool recovers through quota ordering rather than its dormant strategy, a per-provider `enabled` beats the global default in either direction, and a non-positive fill-first threshold disables proactive usage-based rotation. Kiro's `kiroAutoSelection` projects the same candidate eligibility into routing and account-list status; unknown evidence stays eligible, while pause, reauth, suspension, cooldown, and confirmed exhaustion have closed reasons. An active singleton or a pool excluded only from automatic Kiro selection may still attempt its active account; an explicitly paused active account is never dispatched, and an all-paused pool fails with 403. A configured Kiro capacity cap can also time out. Kiro's optional process-local account lease is released at response completion or cancellation; other providers retain their admission path. | | OAuth login callback (inbound) | `src/oauth/callback-server.ts` | Every response, including non-callback 404s, closes its connection so a pooled socket cannot deliver a later login to a retired flow on the same callback port. | | Alibaba regions | `src/providers/alibaba-region-backup.ts`, `src/providers/alibaba-region-migration.ts`, `src/providers/alibaba-region-startup.ts` | Region migration backs up before rewriting and is idempotent across restarts. | | Discovery and quota | `src/providers/model-discovery.ts`, `src/providers/quota.ts`, `src/providers/registry.ts` | Discovery rejects a response over 4 MiB or past 2,000 raw rows before caching it. Provider-scoped hints fill capabilities omitted by live rosters; OpenCode Go's `deepseek-v4.1-flash` keeps its 1,048,576-token context window. The fixed-key Opper preset uses the shared OpenAI Chat adapter at `https://api.opper.ai/v3/compat`, discovers models through its conventional authenticated `/models` path, preserves an older same-named custom destination, and falls back to bare pool ids while passing vendor-prefixed ids through unchanged. Codex quota DTOs suppress retired Spark evidence under the [OpenAI scope contract](../providers/openai-tiers.md#public-provider-contract), retaining ordinary custom windows. | diff --git a/structure/transports/responses-failover.md b/structure/transports/responses-failover.md index 1886fc5339d..f63e3998aa4 100644 --- a/structure/transports/responses-failover.md +++ b/structure/transports/responses-failover.md @@ -575,7 +575,7 @@ stable ceiling without making a cooled account eligible. Same-provider auth reco physical target, rather than a diagnostic key, and a real send is charged once even when recovery rebuilds the request. -Antigravity main Google adapter dispatch: after same-account refresh, a second pre-output 401 or terminal refresh failure may switch once to a live sibling within existing budgets. A pre-output 403 switches once only when the bounded Google adapter normalization found a complete structured `VALIDATION_REQUIRED` reason and `src/server/responses/antigravity-validation-refusal.ts` recognizes its exact marker. The 401 and 403 paths share one per-request sibling-attempt guard and carry the sibling's full token/project snapshot; an ineligible sibling, cancellation or budget refusal preserves the 403. The web-search sidecar uses a generation-fenced health mark and full sibling snapshot for a pre-output structured `VALIDATION_REQUIRED` 403, or the exact known verification sentence in `error.message`, and passes its replay through the shared physical-send budget. Other sidecar paths, continuations and passthrough do not use this rotation (contract: `docs-site/src/content/docs/reference/configuration/providers.md`, `rotateAntigravityAccountOnAuthRefusal` in `src/oauth/generic-account-failover.ts`). +Antigravity main Google adapter dispatch: after same-account refresh, a second pre-output 401 or terminal refresh failure may switch once to a live sibling within existing budgets. A pre-output 403 switches once only when the bounded Google adapter normalization found a complete structured `VALIDATION_REQUIRED` reason and `src/server/responses/antigravity-validation-refusal.ts` recognizes its exact marker. The 401 and 403 paths share one per-request sibling-attempt guard and carry the sibling's full token/project snapshot; an ineligible sibling, cancellation or budget refusal preserves the 403. The fetch web-search sidecar captures pool activation before dispatch and applies the same Google-adapter, replayability, cancellation, failover-limit and pre-output gates to a complete 4 KiB-bounded raw envelope normalized through the existing Google error normalizer. Only structured `error.details[].reason === "VALIDATION_REQUIRED"` authorizes one sibling rotation with recovery kind `oauth-account-403`; sentence-only errors do not rotate. It carries the sibling's full token/project snapshot without persisting a needs-reauth mark. Its 403 and 429 replays hand the reserved permit to the adapter physical-send owner, charging each physical send once and releasing unused permits when the iteration ends. Other sidecar paths, continuations and passthrough do not use this rotation (contract: `docs-site/src/content/docs/reference/configuration/providers.md`, `rotateAntigravityAccountOnAuthRefusal` in `src/oauth/generic-account-failover.ts`). Precommit Codex model refusals use bounded account recovery for HTTP `detail` and WebSocket-projected `error.message` bodies. Only an exact HTTP 400 refusal naming the requested or wire model establishes diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index 700841acd0a..264156a6a0c 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -951,7 +951,7 @@ "server-combo-cooldown-fallback.test.ts": "server", "server-combo-failover-e2e.test.ts": "server", "server-combo-held-response.test.ts": "server", "server-combo-reasoning-replay-eligibility.test.ts": "server", "server-combo-zero-output-failover.test.ts": "server", "server-google-antigravity-oauth-401-replay.test.ts": "server", - "server-google-antigravity-oauth-429-budget.test.ts": "server", "server-gui-bundle-freshness.test.ts": "server", + "server-antigravity-web-search-validation.test.ts": "server", "server-google-antigravity-oauth-429-budget.test.ts": "server", "server-gui-bundle-freshness.test.ts": "server", "server-images-bodyless-content-length.test.ts": "server", "server-images-pool-admission.test.ts": "server", "server-images.test.ts": "server", "server-key-failover-e2e.test.ts": "server", "server-jev-combo-e2e.test.ts": "server", "server-kiro-completion-e2e.test.ts": "server", diff --git a/tests/server/server-antigravity-web-search-validation.test.ts b/tests/server/server-antigravity-web-search-validation.test.ts new file mode 100644 index 00000000000..32bb4928272 --- /dev/null +++ b/tests/server/server-antigravity-web-search-validation.test.ts @@ -0,0 +1,415 @@ +import { afterEach, beforeEach, describe, expect, spyOn, test } from "bun:test"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { saveConfig } from "../../src/config"; +import { clearGenericFailoverHealth } from "../../src/oauth/generic-account-failover"; +import { getAccountSet, saveCredential, setActiveAccount, setAccountPaused } from "../../src/oauth/store"; +import { handleResponses } from "../../src/server/responses"; +import type { OcxConfig } from "../../src/types"; +import { installIsolatedCodexHome, type IsolatedCodexHome } from "../helpers/isolated-codex-home"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; +import { removeTreeWithRetry } from "../helpers/remove-tree"; +import * as retry from "../../src/lib/upstream-retry"; +import * as webSearch from "../../src/web-search"; +import { createRequestExecutionBudget } from "../../src/lib/request-execution-budget"; + +const DAILY_API_BASE = "https://daily-cloudcode-pa.googleapis.com"; + +let testDir = ""; +let previousHome: string | undefined; +let isolatedCodexHome: IsolatedCodexHome | null = null; +let originalFetch: typeof fetch; +let releaseSpendHome: (() => void) | undefined; +let sleepSpy: ReturnType | undefined; + +const takeSpendHome = (): void => { + releaseSpendHome ??= acquireOwnedSpendHome(); +}; + +beforeEach(() => { + originalFetch = globalThis.fetch; + previousHome = process.env.OPENCODEX_HOME; + isolatedCodexHome = installIsolatedCodexHome("ocx-google-429-codex-"); + testDir = mkdtempSync(join(tmpdir(), "ocx-google-429-")); + process.env.OPENCODEX_HOME = testDir; + takeSpendHome(); + clearGenericFailoverHealth(); + sleepSpy = spyOn(retry, "sleepWithAbort").mockImplementation(async () => {}); +}); + +afterEach(() => { + sleepSpy?.mockRestore(); + sleepSpy = undefined; + releaseSpendHome?.(); + releaseSpendHome = undefined; + clearGenericFailoverHealth(); + globalThis.fetch = originalFetch; + if (previousHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previousHome; + isolatedCodexHome?.restore(); + isolatedCodexHome = null; + if (testDir) removeTreeWithRetry(testDir); +}); + +function antigravityConfig(): OcxConfig { + return { + port: 0, + hostname: "127.0.0.1", + defaultProvider: "google-antigravity", + providers: { + "google-antigravity": { + adapter: "google", + baseUrl: DAILY_API_BASE, + authMode: "oauth", + googleMode: "cloud-code-assist", + project: "initial-project-id", + models: ["gemini-3.8-flash"], + }, + }, + } as OcxConfig; +} + +function jsonSuccessBody(text: string): Record { + return { + response: { + candidates: [{ + content: { + role: "model", + parts: [{ text }], + }, + finishReason: "STOP", + }], + usageMetadata: { + promptTokenCount: 5, + candidatesTokenCount: 3, + totalTokenCount: 8, + }, + }, + }; +} + +function sseSuccessBody(text: string): string { + return `data: ${JSON.stringify(jsonSuccessBody(text))}\n\n`; +} + +function createResponsesRequest(bodyOverrides: Record = {}): Request { + return new Request("http://127.0.0.1/v1/responses", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: "google-antigravity/gemini-3.8-flash", + input: "hello", + stream: false, + ...bodyOverrides, + }), + }); +} + +async function seedAntigravityAccounts(count: number): Promise> { + for (let i = 1; i <= count; i++) { + await saveCredential("google-antigravity", { + access: `token-${i}`, + refresh: `refresh-${i}`, + expires: Date.now() + 3_600_000, + accountId: `account-${i}`, + projectId: `project-${i}`, + }); + } + const accounts = getAccountSet("google-antigravity")!.accounts; + await setActiveAccount("google-antigravity", accounts[0]!.id); + return accounts.map((a, idx) => ({ + id: a.id, + auth: `Bearer token-${idx + 1}`, + project: `project-${idx + 1}`, + })); +} + +function installAntigravityFetchMock( + handler: (info: { auth: string; project: string; sendIndex: number }) => Response | Promise, +): void { + let sendIndex = 0; + globalThis.fetch = (async (input, init) => { + const url = input instanceof Request ? input.url : String(input); + const parsedUrl = new URL(url); + + if (parsedUrl.origin === DAILY_API_BASE + && ["/v1internal:streamGenerateContent", "/v1internal:generateContent"].includes(parsedUrl.pathname)) { + sendIndex += 1; + const auth = new Headers(init?.headers).get("authorization") ?? ""; + let project = ""; + if (typeof init?.body === "string") { + try { + const parsed = JSON.parse(init.body) as { project?: string }; + project = parsed.project ?? ""; + } catch { /* ignore */ } + } + return handler({ auth, project, sendIndex }); + } + + if (parsedUrl.hostname === "127.0.0.1" || parsedUrl.hostname === "localhost") return originalFetch(input, init); + throw new Error(`Unexpected external request: ${url}`); + }) as typeof fetch; +} + +const structuredRefusal = JSON.stringify({ error: { + status: "PERMISSION_DENIED", message: "validate", details: [{ reason: "VALIDATION_REQUIRED" }], +} }); + +function webSearchConfig(): OcxConfig { + const cfg = antigravityConfig(); + cfg.webSearchSidecar = { backend: "exa", exaApiKey: "synthetic-exa-key" }; + saveConfig(cfg); + return cfg; +} + +function budget(sends: number) { + return createRequestExecutionBudget({ + maxTotalModelSends: sends, baseSendAllowance: sends, finalRecoveryAllowance: 0, + maxAlternateTargetSends: 0, maxTargetTransitions: 0, + }); +} + +const request = () => createResponsesRequest({ tools: [{ type: "web_search" }] }); +const route = { model: "gemini-3.8-flash", provider: "google-antigravity" }; +const durableHealth = () => getAccountSet("google-antigravity")!.accounts.map(row => ({ + id: row.id, needsReauth: row.needsReauth, needsReauthReason: row.needsReauthReason, +})); + +describe("Antigravity web-search structured validation rotation (#6666)", () => { + test("structured A to B uses B's token/project in exactly two sends and leaves durable health unchanged", async () => { + const accounts = await seedAntigravityAccounts(2); + const cfg = webSearchConfig(); + const before = durableHealth(); + const sends: Array<{ auth: string; project: string }> = []; + installAntigravityFetchMock(({ auth, project, sendIndex }) => { + sends.push({ auth, project }); + return sendIndex === 1 ? new Response(structuredRefusal, { status: 403 }) + : new Response(sseSuccessBody("sibling answered"), { headers: { "content-type": "text/event-stream" } }); + }); + const sendBudget = budget(2); + const response = await handleResponses(request(), cfg, route, { sendBudget }); + expect(response.status).toBe(200); + expect(await response.text()).toContain("sibling answered"); + expect(sends).toEqual(accounts.map(({ auth, project }) => ({ auth, project }))); + expect(sendBudget.used).toBe(2); + expect(durableHealth()).toEqual(before); + }); + + test("repeated structured refusals stop after one sibling", async () => { + const accounts = await seedAntigravityAccounts(3); + const cfg = webSearchConfig(); + const before = durableHealth(); + const sends: Array<{ auth: string; project: string }> = []; + installAntigravityFetchMock(({ auth, project }) => { + sends.push({ auth, project }); + return new Response(structuredRefusal, { status: 403 }); + }); + const sendBudget = budget(3); + const response = await handleResponses(request(), cfg, route, { sendBudget }); + expect(response.status).toBe(403); + await response.text(); + expect(sends).toEqual(accounts.slice(0, 2).map(({ auth, project }) => ({ auth, project }))); + expect(sendBudget.used).toBe(2); + expect(durableHealth()).toEqual(before); + }); + + test.each([ + ["sentence only", JSON.stringify({ error: { message: "Verify your account to continue." } })], + ["legacy sentence only", JSON.stringify({ error: { message: "Please verify your account to continue using Antigravity." } })], + ["unrelated", JSON.stringify({ error: { message: "Feature unavailable", details: [{ reason: "OTHER" }] } })], + ["spoofed message", JSON.stringify({ error: { message: "Antigravity account validation required (VALIDATION_REQUIRED): validate" } })], + ["spoofed top level", JSON.stringify({ reason: "VALIDATION_REQUIRED", error: { message: "validate" } })], + ["spoofed nested reason", JSON.stringify({ error: { details: [{ metadata: { reason: "VALIDATION_REQUIRED" } }] } })], + ["non-array details", JSON.stringify({ error: { details: { reason: "VALIDATION_REQUIRED" } } })], + ["malformed", structuredRefusal.slice(0, -1)], + ["oversized complete JSON", JSON.stringify({ error: { details: [{ reason: "VALIDATION_REQUIRED" }] }, padding: "x".repeat(4096) })], + ["oversized trailing whitespace", structuredRefusal + " ".repeat(4096)], + ])("%s never rotates", async (_name, body) => { + const accounts = await seedAntigravityAccounts(2); + const cfg = webSearchConfig(); + const before = durableHealth(); + const sends: Array<{ auth: string; project: string }> = []; + installAntigravityFetchMock(({ auth, project }) => { + sends.push({ auth, project }); + return new Response(body, { status: 403 }); + }); + const response = await handleResponses(request(), cfg, route); + expect(response.status).toBe(403); + await response.text(); + expect(sends).toEqual([{ auth: accounts[0]!.auth, project: accounts[0]!.project }]); + expect(durableHealth()).toEqual(before); + }); + + test("a spent physical-send budget preserves the refusal and durable health", async () => { + await seedAntigravityAccounts(2); + const cfg = webSearchConfig(); + const before = durableHealth(); + let sends = 0; + installAntigravityFetchMock(() => { sends++; return new Response(structuredRefusal, { status: 403 }); }); + const sendBudget = budget(1); + const response = await handleResponses(request(), cfg, route, { sendBudget }); + expect(response.status).toBe(403); + await response.text(); + expect(sends).toBe(1); + expect(sendBudget.used).toBe(1); + expect(durableHealth()).toEqual(before); + }); + + test("cancellation prevents a sibling send and durable health changes", async () => { + await seedAntigravityAccounts(2); + const cfg = webSearchConfig(); + const before = durableHealth(); + const abort = new AbortController(); + let sends = 0; + installAntigravityFetchMock(() => { + sends++; abort.abort(); return new Response(structuredRefusal, { status: 403 }); + }); + const response = await handleResponses(request(), cfg, route, { abortSignal: abort.signal }); + await response.text(); + expect(sends).toBe(1); + expect(durableHealth()).toEqual(before); + }); + + test("a pool absent before dispatch cannot activate after the first refusal", async () => { + await seedAntigravityAccounts(1); + const cfg = webSearchConfig(); + let sends = 0; + installAntigravityFetchMock(async () => { + sends++; + await saveCredential("google-antigravity", { + access: "late-token", refresh: "late-refresh", expires: Date.now() + 3_600_000, + accountId: "late-account", projectId: "late-project", + }, { addAccount: true }); + clearGenericFailoverHealth(); + return new Response(structuredRefusal, { status: 403 }); + }); + const response = await handleResponses(request(), cfg, route); + expect(response.status).toBe(403); + await response.text(); + expect(sends).toBe(1); + expect(durableHealth().every(row => !row.needsReauth)).toBe(true); + }); + + test.each(["unavailable", "projectless"])("an %s sibling preserves the refusal and releases its unused permit", async kind => { + const accounts = await seedAntigravityAccounts(kind === "projectless" ? 1 : 2); + if (kind === "projectless") { + await saveCredential("google-antigravity", { + access: "projectless-token", refresh: "projectless-refresh", expires: Date.now() + 3_600_000, + accountId: "projectless-account", + }, { addAccount: true }); + await setActiveAccount("google-antigravity", accounts[0]!.id); + } + const cfg = webSearchConfig(); + const before = durableHealth(); + let sends = 0; + installAntigravityFetchMock(async () => { + sends++; + if (kind === "unavailable") await setAccountPaused("google-antigravity", accounts[1]!.id, true); + return new Response(structuredRefusal, { status: 403 }); + }); + const sendBudget = budget(2); + const response = await handleResponses(request(), cfg, route, { sendBudget }); + expect(response.status).toBe(403); + await response.text(); + expect(sends).toBe(1); + expect(sendBudget.used).toBe(1); + expect(sendBudget.remainingBaseSends(2)).toBe(1); + expect(durableHealth()).toEqual(before); + }); +}); + +test("web-search preserves structured refusal once the roster's failover limit is spent", async () => { + const accounts = await seedAntigravityAccounts(5); + const cfg = webSearchConfig(); + const sends: string[] = []; + installAntigravityFetchMock(async ({ auth, sendIndex }) => { + sends.push(auth); + if (sendIndex <= 4) return Response.json({ error: { + status: "RESOURCE_EXHAUSTED", message: "Quota exceeded for quota metric", + } }, { status: 429 }); + // A new eligible sibling exists, but it cannot enlarge the captured failover ceiling. + await saveCredential("google-antigravity", { + access: "late-token", refresh: "late-refresh", expires: Date.now() + 3_600_000, + accountId: "late-account", projectId: "late-project", + }, { addAccount: true }); + return new Response(structuredRefusal, { status: 403 }); + }); + const sendBudget = budget(10); + const response = await handleResponses(request(), cfg, route, { sendBudget }); + expect(response.status).toBe(403); + await response.text(); + expect(sends).toEqual(accounts.map(row => row.auth)); + expect(sendBudget.used).toBe(5); + expect(durableHealth().every(row => !row.needsReauth)).toBe(true); +}); + +test.each(["non-replayable", "committed output", "non-Google adapter"])( + "sidecar callback rejects %s before reserving or rotating", + async gate => { + await seedAntigravityAccounts(2); + const cfg = webSearchConfig(); + const before = durableHealth(); + const sendBudget = budget(2); + const loopSpy = spyOn(webSearch, "runWithWebSearch").mockImplementation(async deps => { + const refusal = new Response(structuredRefusal, { status: 403 }); + if (gate === "non-replayable") retry.markResponseNonReplayable(refusal); + if (gate === "committed output") deps.onFirstOutput?.(); + const name = deps.adapter.name; + if (gate === "non-Google adapter") deps.adapter.name = "mock-other"; + try { + expect(await deps.on429?.(null, refusal.headers, deps.parsed, refusal)).toBeNull(); + expect(sendBudget.used).toBe(0); + expect(sendBudget.remainingBaseSends(2)).toBe(2); + return refusal; + } finally { deps.adapter.name = name; } + }); + try { + const response = await handleResponses(request(), cfg, route, { sendBudget }); + expect(response.status).toBe(403); + await response.text(); + expect(durableHealth()).toEqual(before); + } finally { loopSpy.mockRestore(); } + }, +); + +test("iteration cleanup releases a sibling permit if construction ends before physical dispatch", async () => { + await seedAntigravityAccounts(2); + const cfg = webSearchConfig(); + const sendBudget = budget(2); + const loopSpy = spyOn(webSearch, "runWithWebSearch").mockImplementation(async deps => { + const refusal = new Response(structuredRefusal, { status: 403 }); + const replacement = await deps.on429?.(null, refusal.headers, deps.parsed, refusal); + expect(replacement?.recoveryKind).toBe("oauth-account-403"); + expect(sendBudget.used).toBe(1); + expect(sendBudget.remainingBaseSends(2)).toBe(1); + deps.onIterationEnd?.(); + expect(sendBudget.used).toBe(0); + expect(sendBudget.remainingBaseSends(2)).toBe(2); + return refusal; + }); + try { + const response = await handleResponses(request(), cfg, route, { sendBudget }); + await response.text(); + } finally { loopSpy.mockRestore(); } +}); + +test("web-search 429 recovery still uses the sibling's identity and charges exactly two physical sends", async () => { + const accounts = await seedAntigravityAccounts(2); + const cfg = webSearchConfig(); + const sends: Array<{ auth: string; project: string }> = []; + installAntigravityFetchMock(({ auth, project, sendIndex }) => { + sends.push({ auth, project }); + return sendIndex === 1 ? Response.json({ error: { + status: "RESOURCE_EXHAUSTED", message: "Quota exceeded for quota metric", + } }, { status: 429 }) : new Response(sseSuccessBody("429 sibling answered"), { + headers: { "content-type": "text/event-stream" }, + }); + }); + const sendBudget = budget(2); + const response = await handleResponses(request(), cfg, route, { sendBudget }); + expect(response.status).toBe(200); + expect(await response.text()).toContain("429 sibling answered"); + expect(sends).toEqual(accounts.map(({ auth, project }) => ({ auth, project }))); + expect(sendBudget.used).toBe(2); +}); diff --git a/tests/server/server-google-antigravity-oauth-429-budget.test.ts b/tests/server/server-google-antigravity-oauth-429-budget.test.ts index b4a7c790646..05cc7afc92c 100644 --- a/tests/server/server-google-antigravity-oauth-429-budget.test.ts +++ b/tests/server/server-google-antigravity-oauth-429-budget.test.ts @@ -88,10 +88,6 @@ function jsonSuccessBody(text: string): Record { }; } -function sseSuccessBody(text: string): string { - return `data: ${JSON.stringify(jsonSuccessBody(text))}\n\n`; -} - function transient429ErrorBody(): Record { return { error: { @@ -216,188 +212,6 @@ describe("Google Antigravity OAuth 429 retry and multi-account budget (#5880)", expect(sendBudget.used).toBe(1); }); - test.each([ - { error: { status: "PERMISSION_DENIED", message: "validate", details: [{ reason: "VALIDATION_REQUIRED" }] } }, - { error: { status: "PERMISSION_DENIED", message: "Verify your account to continue." } }, - { error: { status: "PERMISSION_DENIED", message: "Please verify your account to continue using Antigravity." } }, - ])("web_search rotates only a recognized verification 403 with the sibling identity and project", async refusal => { - const accounts = await seedAntigravityAccounts(2); - const cfg = antigravityConfig(); - cfg.webSearchSidecar = { backend: "exa", exaApiKey: "synthetic-exa-key" }; - saveConfig(cfg); - const observedSends: Array<{ auth: string; project: string }> = []; - installAntigravityFetchMock(({ auth, project, sendIndex }) => { - observedSends.push({ auth, project }); - if (sendIndex === 1) return new Response(JSON.stringify(refusal), { status: 403 }); - return new Response(sseSuccessBody("sibling answered"), { - status: 200, headers: { "content-type": "text/event-stream" }, - }); - }); - const sendBudget = createRequestExecutionBudget({ - maxTotalModelSends: 2, baseSendAllowance: 2, finalRecoveryAllowance: 0, - maxAlternateTargetSends: 0, maxTargetTransitions: 0, - }); - - const response = await handleResponses( - createResponsesRequest({ tools: [{ type: "web_search" }] }), - cfg, - { model: "gemini-3.8-flash", provider: "google-antigravity" }, - { sendBudget }, - ); - - expect(response.status).toBe(200); - expect(await response.text()).toContain("sibling answered"); - expect(observedSends).toEqual([ - { auth: accounts[0]!.auth, project: accounts[0]!.project }, - { auth: accounts[1]!.auth, project: accounts[1]!.project }, - ]); - expect(getAccountSet("google-antigravity")!.accounts.find(row => row.id === accounts[0]!.id)) - .toMatchObject({ needsReauth: true, needsReauthReason: "verify_account" }); - expect(sendBudget.used).toBe(2); - }); - - test("web_search leaves an unrelated 403 intact and does not quarantine the account", async () => { - const accounts = await seedAntigravityAccounts(2); - const cfg = antigravityConfig(); - cfg.webSearchSidecar = { backend: "exa", exaApiKey: "synthetic-exa-key" }; - saveConfig(cfg); - const observedSends: Array<{ auth: string; project: string }> = []; - installAntigravityFetchMock(({ auth, project }) => { - observedSends.push({ auth, project }); - return new Response(JSON.stringify({ error: { - status: "PERMISSION_DENIED", message: "The requested feature is not available to this account.", - } }), { status: 403 }); - }); - - const response = await handleResponses( - createResponsesRequest({ tools: [{ type: "web_search" }] }), - cfg, - { model: "gemini-3.8-flash", provider: "google-antigravity" }, - ); - - expect(response.status).toBe(403); - expect(await response.text()).toContain("access denied"); - expect(observedSends).toEqual([{ auth: accounts[0]!.auth, project: accounts[0]!.project }]); - expect(getAccountSet("google-antigravity")!.accounts.find(row => row.id === accounts[0]!.id)?.needsReauth).toBeFalsy(); - }); - - test("web_search cannot spend a second send when the caller budget is exhausted", async () => { - const accounts = await seedAntigravityAccounts(2); - const cfg = antigravityConfig(); - cfg.webSearchSidecar = { backend: "exa", exaApiKey: "synthetic-exa-key" }; - saveConfig(cfg); - const observedSends: Array<{ auth: string; project: string }> = []; - installAntigravityFetchMock(({ auth, project }) => { - observedSends.push({ auth, project }); - return new Response(JSON.stringify({ error: { status: "PERMISSION_DENIED", message: "validate", - details: [{ reason: "VALIDATION_REQUIRED" }] } }), { status: 403 }); - }); - const sendBudget = createRequestExecutionBudget({ - maxTotalModelSends: 1, baseSendAllowance: 1, finalRecoveryAllowance: 0, - maxAlternateTargetSends: 0, maxTargetTransitions: 0, - }); - - const response = await handleResponses( - createResponsesRequest({ tools: [{ type: "web_search" }] }), - cfg, - { model: "gemini-3.8-flash", provider: "google-antigravity" }, - { sendBudget }, - ); - - expect(response.status).toBe(403); - expect(await response.text()).toContain("Antigravity account validation required"); - expect(observedSends).toEqual([{ auth: accounts[0]!.auth, project: accounts[0]!.project }]); - expect(sendBudget.used).toBe(1); - }); - - test("web_search cancellation after the refusal sends no sibling request", async () => { - await seedAntigravityAccounts(2); - const cfg = antigravityConfig(); - cfg.webSearchSidecar = { backend: "exa", exaApiKey: "synthetic-exa-key" }; - saveConfig(cfg); - const abort = new AbortController(); - let sends = 0; - installAntigravityFetchMock(() => { - sends += 1; - abort.abort(); - return new Response(JSON.stringify({ error: { status: "PERMISSION_DENIED", message: "validate", - details: [{ reason: "VALIDATION_REQUIRED" }] } }), { status: 403 }); - }); - - const response = await handleResponses( - createResponsesRequest({ tools: [{ type: "web_search" }] }), - cfg, - { model: "gemini-3.8-flash", provider: "google-antigravity" }, - { abortSignal: abort.signal }, - ); - - expect(sends).toBe(1); - await response.text(); - }); - - test("a stale credential generation is not quarantined or replayed from the web_search path", async () => { - const accounts = await seedAntigravityAccounts(2); - const cfg = antigravityConfig(); - cfg.webSearchSidecar = { backend: "exa", exaApiKey: "synthetic-exa-key" }; - saveConfig(cfg); - const observedSends: Array<{ auth: string; project: string }> = []; - installAntigravityFetchMock(async ({ auth, project, sendIndex }) => { - observedSends.push({ auth, project }); - if (sendIndex === 1) { - await saveCredential("google-antigravity", { - access: "relogin-token", refresh: "relogin-refresh", expires: Date.now() + 3_600_000, - accountId: "account-1", projectId: "project-1", - }); - return new Response(JSON.stringify({ error: { status: "PERMISSION_DENIED", message: "validate", - details: [{ reason: "VALIDATION_REQUIRED" }] } }), { status: 403 }); - } - return new Response(JSON.stringify(jsonSuccessBody("unexpected sibling")), { status: 200 }); - }); - - const response = await handleResponses( - createResponsesRequest({ tools: [{ type: "web_search" }] }), - cfg, - { model: "gemini-3.8-flash", provider: "google-antigravity" }, - ); - - expect(response.status).toBe(403); - await response.text(); - expect(observedSends).toHaveLength(1); - const failed = getAccountSet("google-antigravity")!.accounts.find(row => row.id === accounts[0]!.id)!; - expect(failed.needsReauth).toBeFalsy(); - expect(failed.needsReauthReason).toBeUndefined(); - }); - - test("web_search marks the refused account but preserves the 403 when no sibling has a project", async () => { - const accounts = await seedAntigravityAccounts(1); - const cfg = antigravityConfig(); - cfg.webSearchSidecar = { backend: "exa", exaApiKey: "synthetic-exa-key" }; - saveConfig(cfg); - await saveCredential("google-antigravity", { - access: "no-project-token", refresh: "no-project-refresh", expires: Date.now() + 3_600_000, - accountId: "no-project-account", - }, { addAccount: true }); - await setActiveAccount("google-antigravity", accounts[0]!.id); - const observedSends: Array<{ auth: string; project: string }> = []; - installAntigravityFetchMock(({ auth, project }) => { - observedSends.push({ auth, project }); - return new Response(JSON.stringify({ error: { status: "PERMISSION_DENIED", message: "validate", - details: [{ reason: "VALIDATION_REQUIRED" }] } }), { status: 403 }); - }); - - const response = await handleResponses( - createResponsesRequest({ tools: [{ type: "web_search" }] }), - cfg, - { model: "gemini-3.8-flash", provider: "google-antigravity" }, - ); - - expect(response.status).toBe(403); - expect(await response.text()).toContain("Antigravity account validation required"); - expect(observedSends).toEqual([{ auth: accounts[0]!.auth, project: accounts[0]!.project }]); - expect(getAccountSet("google-antigravity")!.accounts.find(row => row.id === accounts[0]!.id)) - .toMatchObject({ needsReauth: true, needsReauthReason: "verify_account" }); - }); - test.each([4, 5])("%i accounts each receive three transient sends before terminal 429", async accountCount => { const accounts = await seedAntigravityAccounts(accountCount); const cfg = antigravityConfig(); From ff790ffee7a14b124169cc44445aef1e4827a1a4 Mon Sep 17 00:00:00 2001 From: JUN Date: Wed, 7 Oct 2026 23:14:39 +0900 Subject: [PATCH 5/5] fix: preserve Antigravity search refusals and refund unsent hops --- .../docs/reference/configuration/providers.md | 4 +- src/server/responses/request-send-budget.ts | 63 +++++++++-------- src/server/responses/sidecar-execution.ts | 4 +- src/web-search/loop.ts | 22 ++++-- structure/transports/responses-failover.md | 2 +- structure/transports/responses-spend.md | 6 +- tests/adapters/physical-send.test.ts | 28 +++++++- tests/server/inference-send-budget.test.ts | 59 ++++++++++++++++ ...-antigravity-web-search-validation.test.ts | 69 +++++++++++++------ 9 files changed, 195 insertions(+), 62 deletions(-) diff --git a/docs-site/src/content/docs/reference/configuration/providers.md b/docs-site/src/content/docs/reference/configuration/providers.md index 1545bda5f0a..bfc084cf0ae 100644 --- a/docs-site/src/content/docs/reference/configuration/providers.md +++ b/docs-site/src/content/docs/reference/configuration/providers.md @@ -1024,7 +1024,9 @@ OAuth token and matching Cloud Code Assist project. It accepts only complete, bo envelopes with `error.details[].reason === "VALIDATION_REQUIRED"`; verification wording alone does not authorize rotation, and this path does not persist a reauthentication mark. Each physical send, including a 429 retry, consumes the same request budget exactly once. A disabled pool, cancellation, -exhausted budget or unavailable sibling preserves the failure. +exhausted budget or unavailable sibling preserves the failure. If sibling request construction or +dispatch fails, the original bounded 403 remains the error; a replay rejected before physical +dispatch returns its unused send allowance. Continuations, native Responses passthrough, image sidecars, and output already sent to the client do not use this rotation. diff --git a/src/server/responses/request-send-budget.ts b/src/server/responses/request-send-budget.ts index b8bc18e4fd9..d56836b7135 100644 --- a/src/server/responses/request-send-budget.ts +++ b/src/server/responses/request-send-budget.ts @@ -171,19 +171,21 @@ export function createResponsesSendBudget( * The hop hands its reservation down through `pendingHopPermit`, the same seam the * passthrough ladder already uses, and this view spends it on the adapter's FIRST * reservation. Every later send in that ladder is a new physical send and is charged - * normally. A permit the adapter takes but never sends under is released through the same - * call it would have used for a reservation of its own, so an abandoned replay is refunded - * rather than left charged. + * normally. The refundable view below keeps an unused hop open until executor invocation; + * its adapter release hands back an abandoned replay. The ordinary view retains early + * settlement for existing callers. */ + const claimHopPermit = (): SingleUseDispatchPermit | undefined => { + const permit = pendingHopPermit; + pendingHopPermit = undefined; + return permit; + }; const adapterDispatchBudget: RequestExecutionBudget | undefined = adapterSendBudget === undefined - ? undefined - : adapterDispatchBudgetView(adapterSendBudget, { - claimHopPermit: () => { - const permit = pendingHopPermit; - pendingHopPermit = undefined; - return permit; - }, - }); + ? undefined : adapterDispatchBudgetView(adapterSendBudget, { claimHopPermit }); + // Opt in only where the adapter's executor is the confirmation boundary. Ordinary callers + // keep the existing early settlement contract, including already-settled permit fallback. + const refundableAdapterDispatchBudget: RequestExecutionBudget | undefined = adapterSendBudget === undefined + ? undefined : adapterDispatchBudgetView(adapterSendBudget, { claimHopPermit }, true); /** * How many sends a recovery leg may make, and the permit that authorises the last one. * @@ -279,6 +281,7 @@ export function createResponsesSendBudget( get sendsUsed(): number { return sendBudget.used; }, adapterSendBudget, adapterDispatchBudget, + refundableAdapterDispatchBudget, noteAdapterPhysicalSend, noteAdapterRecoveryWithheld, sendBudgetExhausted, @@ -312,6 +315,7 @@ export type ResponsesSendBudget = Exclude SingleUseDispatchPermit | undefined }, + refundableHop = false, ): RequestExecutionBudget { return { get used(): number { return budget.used; }, @@ -338,30 +342,29 @@ function adapterDispatchBudgetView( hopPermit.release(); return budget.reserveDispatch({ ...intent, sendClass: hopPermit.sendClass }); } - // Confirmed here rather than in `use()`: the adapter reserves immediately before it - // opens the transport, which is the same boundary the hop's own confirmation uses. - // A permit some other leg already settled returns false, and this falls through to a - // real reservation rather than handing the adapter a dead permit -- an adapter whose - // `use()` fails treats the request as exhausted and stops sending entirely. - if (hopPermit !== undefined && hopPermit.assumeCharge()) { - let spent = false; + if (hopPermit !== undefined && (refundableHop || hopPermit.assumeCharge())) { + let settled = false; + const confirm = (): boolean => { + if (settled) return false; + settled = true; + // Admission can still reject this replay. Close its external booking only when + // the adapter invokes its executor, leaving release() able to refund it until then. + if (!refundableHop || hopPermit.assumeCharge()) return true; + // Preserve the old fallback for a permit another leg already settled. + const fresh = budget.reserveDispatch(intent); + return fresh.allowed && fresh.permit.assumeCharge(); + }; return { allowed: true, permit: { sendClass: hopPermit.sendClass, - use: (): boolean => { - if (spent) return false; - spent = true; - return true; - }, - assumeCharge: (): boolean => { - if (spent) return false; - spent = true; - return true; + use: confirm, + assumeCharge: confirm, + release: (): void => { + if (!refundableHop || settled) return; + settled = true; + hopPermit.release(); }, - // The hop's charge is already settled and belongs to the leg that asked for it, - // so there is nothing here to refund. - release: (): void => {}, }, }; } diff --git a/src/server/responses/sidecar-execution.ts b/src/server/responses/sidecar-execution.ts index 65d8872aaa4..d19f8516726 100644 --- a/src/server/responses/sidecar-execution.ts +++ b/src/server/responses/sidecar-execution.ts @@ -88,7 +88,7 @@ export async function executeResponsesSidecars( | "notifyResponseComplete" | "cancelResponseCompletion" >, - sendBudgetState: Pick, + sendBudgetState: Pick, ) { const { config, options, logCtx } = requestContext; const antigravityPoolActivated = requestState.route.providerName === "google-antigravity" @@ -548,7 +548,7 @@ export async function executeResponsesSidecars( translatorBudget, providerFetch: routedProviderFetch, ...(route.providerName === "google-antigravity" ? { - sendBudget: sendBudgetState.adapterDispatchBudget, + sendBudget: sendBudgetState.refundableAdapterDispatchBudget, onPhysicalSend: (send: { ordinal: number; recovery?: AttemptRecoveryKind }) => sendBudgetState.noteAdapterPhysicalSend(logCtx.usageLogInputTokens, send), } : {}), diff --git a/src/web-search/loop.ts b/src/web-search/loop.ts index 5bb2c017e62..0cbf89fbd79 100644 --- a/src/web-search/loop.ts +++ b/src/web-search/loop.ts @@ -582,13 +582,27 @@ export async function runWithWebSearch(deps: WebSearchLoopDeps): Promise {}); } catch { /* already closed */ } + const retainRefusal = prepared.response.status === 403 + && deps.incomingMeta?.providerName === "google-antigravity" && rotated.recoveryKind === "oauth-account-403"; + // Keep the bounded refusal readable until the sibling actually returns headers. + // Other rotation paths retain their existing cancellation boundary. + if (!retainRefusal) { + try { void prepared.response.body?.cancel().catch(() => {}); } catch { /* already closed */ } + } adapter = rotated.adapter; // Stall-watchdog seam between bounded retry fetches (audit 011 B3). yield { type: "heartbeat" }; - prepared = await fetchOnce(adapter, rotated.recoveryKind); + let replacement: IterationResponse; + try { + replacement = await fetchOnce(adapter, rotated.recoveryKind); + } catch (error) { + if (!retainRefusal || signal.aborted || headerDeadline.didExpire() || isTranslatorBudgetExceededError(error)) throw error; + break; // Format the original 403 safely; replacement diagnostics are not client errors. + } + if (retainRefusal) { + try { void prepared.response.body?.cancel().catch(() => {}); } catch { /* already closed */ } + } + prepared = replacement; } // Final headers have arrived. Clear only the deadline timer before ANY body read. diff --git a/structure/transports/responses-failover.md b/structure/transports/responses-failover.md index f63e3998aa4..f0a337e3a41 100644 --- a/structure/transports/responses-failover.md +++ b/structure/transports/responses-failover.md @@ -575,7 +575,7 @@ stable ceiling without making a cooled account eligible. Same-provider auth reco physical target, rather than a diagnostic key, and a real send is charged once even when recovery rebuilds the request. -Antigravity main Google adapter dispatch: after same-account refresh, a second pre-output 401 or terminal refresh failure may switch once to a live sibling within existing budgets. A pre-output 403 switches once only when the bounded Google adapter normalization found a complete structured `VALIDATION_REQUIRED` reason and `src/server/responses/antigravity-validation-refusal.ts` recognizes its exact marker. The 401 and 403 paths share one per-request sibling-attempt guard and carry the sibling's full token/project snapshot; an ineligible sibling, cancellation or budget refusal preserves the 403. The fetch web-search sidecar captures pool activation before dispatch and applies the same Google-adapter, replayability, cancellation, failover-limit and pre-output gates to a complete 4 KiB-bounded raw envelope normalized through the existing Google error normalizer. Only structured `error.details[].reason === "VALIDATION_REQUIRED"` authorizes one sibling rotation with recovery kind `oauth-account-403`; sentence-only errors do not rotate. It carries the sibling's full token/project snapshot without persisting a needs-reauth mark. Its 403 and 429 replays hand the reserved permit to the adapter physical-send owner, charging each physical send once and releasing unused permits when the iteration ends. Other sidecar paths, continuations and passthrough do not use this rotation (contract: `docs-site/src/content/docs/reference/configuration/providers.md`, `rotateAntigravityAccountOnAuthRefusal` in `src/oauth/generic-account-failover.ts`). +Antigravity main Google adapter dispatch: after same-account refresh, a second pre-output 401 or terminal refresh failure may switch once to a live sibling within existing budgets. A pre-output 403 switches once only when the bounded Google adapter normalization found a complete structured `VALIDATION_REQUIRED` reason and `src/server/responses/antigravity-validation-refusal.ts` recognizes its exact marker. The 401 and 403 paths share one per-request sibling-attempt guard and carry the sibling's full token/project snapshot; an ineligible sibling, cancellation or budget refusal preserves the 403. The fetch web-search sidecar captures pool activation before dispatch and applies the same Google-adapter, replayability, cancellation, failover-limit and pre-output gates to a complete 4 KiB-bounded raw envelope normalized through the existing Google error normalizer. Only structured `error.details[].reason === "VALIDATION_REQUIRED"` authorizes one sibling rotation with recovery kind `oauth-account-403`; sentence-only errors do not rotate. It carries the sibling's full token/project snapshot without persisting a needs-reauth mark. Its 403 and 429 replays hand the reserved permit to the adapter physical-send owner, charging each physical send once. The loop retains the bounded 403 until the sibling returns headers; construction or dispatch failure uses the original refusal's safe formatter, while cancellation and header deadlines retain lifecycle ownership. Unused permits are refunded before dispatch: iteration cleanup owns unclaimed permits, and the adapter physical-send owner releases claimed permits after failed admission. Other sidecar paths, continuations and passthrough do not use this rotation (contract: `docs-site/src/content/docs/reference/configuration/providers.md`, `rotateAntigravityAccountOnAuthRefusal` in `src/oauth/generic-account-failover.ts`). Precommit Codex model refusals use bounded account recovery for HTTP `detail` and WebSocket-projected `error.message` bodies. Only an exact HTTP 400 refusal naming the requested or wire model establishes diff --git a/structure/transports/responses-spend.md b/structure/transports/responses-spend.md index 63b04dbf076..4984f42ad44 100644 --- a/structure/transports/responses-spend.md +++ b/structure/transports/responses-spend.md @@ -42,8 +42,10 @@ pre-output stated-reset replay — reserves once per physical send instead, so n arrives. Those ladders are handed `adapterDispatchBudget`, a live delegating view of the same budget that spends a permit passed down through `pendingHopPermit` on the adapter's first reservation and closes the booking through -`permit.assumeCharge()`. Letting both charge is how one physical send became two charges, and how a -spent allowance answered a 429 with a synthetic error instead of the rate limit it was recovering +`permit.assumeCharge()`. The Antigravity fetch web-search path opts into `refundableAdapterDispatchBudget`: +its booking stays refundable until executor invocation confirms it through `permit.use()` or +`permit.assumeCharge()`, and failed admission releases it. Other callers keep early settlement. +Letting both charge is how one physical send became two charges, and how a spent allowance answered a 429 with a synthetic error instead of the rate limit it was recovering from (#4709). `run-turn-execution.ts` passes the same physical-send and recovery-withheld observers used by the diff --git a/tests/adapters/physical-send.test.ts b/tests/adapters/physical-send.test.ts index 236f83ce055..027afd405d1 100644 --- a/tests/adapters/physical-send.test.ts +++ b/tests/adapters/physical-send.test.ts @@ -14,14 +14,14 @@ afterEach(() => resetProviderRequestPacingForTest()); * A credential hop has already reserved the replay it hands to the adapter, so the adapter's * first send spends that permit through the dispatch view instead of reserving again. */ -function prepaid() { +function prepaid(refundable = false) { const parent = createRequestExecutionBudget(); parent.used = 3; const { owner, dispose } = budgetOwner(parent); const hop = owner.reserveCredentialHop("auth-recovery", url, true); if (!hop.allowed || !hop.permit) throw new Error("Expected prepaid final send"); owner.pendingHopPermit = hop.permit; - const scope = owner.adapterDispatchBudget; + const scope = refundable ? owner.refundableAdapterDispatchBudget : owner.adapterDispatchBudget; if (!scope) throw new Error("Expected an adapter dispatch budget"); return { parent, scope, dispose }; } @@ -112,6 +112,30 @@ describe("adapter physical inference admission", () => { } finally { dispose(); } }); + test.each(["pacing", "backoff", "abort", "adapter"] as const)("an opted-in hop refunds when the %s leg never dispatches", async phase => { + const { parent, scope, dispose } = prepaid(true); + let sends = 0; + const controller = new AbortController(); + const failure = new Error(`fixture ${phase} refusal`); + const executor = Object.assign(async () => { sends += 1; return new Response("unexpected"); }, { + waitForPacing: async () => { if (phase === "pacing") throw failure; }, + }) as typeof fetch; + try { + const send = createAdapterPhysicalSend({ sendBudget: scope, abortSignal: controller.signal }, executor); + await expect(send({ url, beforeDispatch: () => { + if (phase === "backoff") throw failure; + if (phase === "abort") controller.abort(failure); + }, dispatch: physical => { + if (phase === "adapter") throw failure; + return physical(url); + } })).rejects.toBe(failure); + // The refundable view leaves the hop open until actual executor invocation. + expect(parent.used).toBe(3); + expect(parent.reserveSpent).toBe(false); + expect(sends).toBe(0); + } finally { dispose(); } + }); + test("an exhausted initial send performs no inference or retry preparation", async () => { const budget = createRequestExecutionBudget(); budget.used = 4; diff --git a/tests/server/inference-send-budget.test.ts b/tests/server/inference-send-budget.test.ts index 0e3ca1597a9..ba32b9722df 100644 --- a/tests/server/inference-send-budget.test.ts +++ b/tests/server/inference-send-budget.test.ts @@ -93,6 +93,65 @@ describe("createInferenceSendBudget", () => { expect(budget.used).toBe(3); } finally { dispose(); } }); + test.each([false, true])("an undispatched adapter hop remains refundable (externally counted=%s)", external => { + const budget = createRequestExecutionBudget(); + const { owner, dispose } = budgetOwner(budget); + try { + const targetKey = "https://upstream.example/model"; + const initial = owner.refundableAdapterDispatchBudget!.reserveDispatch({ sendClass: "initial", targetKey }); + if (!initial.allowed) throw new Error("Expected initial permit"); + initial.permit.use(); + const hop = owner.reserveCredentialHop("auth-recovery", targetKey, external); + owner.pendingHopPermit = hop.permit; + const replay = owner.refundableAdapterDispatchBudget!.reserveDispatch({ sendClass: "transient", targetKey }); + if (!replay.allowed) throw new Error("Expected replay permit"); + expect(budget.used).toBe(2); + replay.permit.release(); + replay.permit.release(); + expect(budget.used).toBe(1); + expect(budget.remainingBaseSends(3)).toBe(2); + expect(replay.permit.use()).toBe(false); + expect(replay.permit.assumeCharge()).toBe(false); + owner.noteTransientSends(1); + expect(budget.used).toBe(2); // No abandoned external booking can swallow a later send. + } finally { dispose(); } + }); + + test.each(["use", "assumeCharge"] as const)("adapter hop %s settles an external booking once and cannot refund a send", confirm => { + const budget = createRequestExecutionBudget(); + const { owner, dispose } = budgetOwner(budget); + try { + const targetKey = "https://upstream.example/model"; + const hop = owner.reserveCredentialHop("auth-recovery", targetKey, true); + owner.pendingHopPermit = hop.permit; + const replay = owner.refundableAdapterDispatchBudget!.reserveDispatch({ sendClass: "transient", targetKey }); + if (!replay.allowed) throw new Error("Expected replay permit"); + expect(replay.permit[confirm]()).toBe(true); + expect(replay.permit.use()).toBe(false); + expect(replay.permit.assumeCharge()).toBe(false); + replay.permit.release(); + expect(budget.used).toBe(1); + owner.noteTransientSends(1); + expect(budget.used).toBe(2); + } finally { dispose(); } + }); + + test("a previously settled adapter hop falls back to a fresh reservation", () => { + const budget = createRequestExecutionBudget(); + const { owner, dispose } = budgetOwner(budget); + try { + const targetKey = "https://upstream.example/model"; + const hop = owner.reserveCredentialHop("auth-recovery", targetKey, true); + expect(hop.permit?.assumeCharge()).toBe(true); + owner.pendingHopPermit = hop.permit; + const replay = owner.refundableAdapterDispatchBudget!.reserveDispatch({ sendClass: "transient", targetKey }); + if (!replay.allowed) throw new Error("Expected replay permit"); + expect(replay.permit.use()).toBe(true); + expect(budget.used).toBe(2); + replay.permit.release(); + expect(budget.used).toBe(2); + } finally { dispose(); } + }); test("mints a default-policy holder and parks this request's spend tracker on the log", () => { const logCtx: RequestLogContext = { model: "m", provider: "p" }; const req = new Request("http://localhost/v1/responses", { method: "POST" }); diff --git a/tests/server/server-antigravity-web-search-validation.test.ts b/tests/server/server-antigravity-web-search-validation.test.ts index 32bb4928272..80d1e8cfff1 100644 --- a/tests/server/server-antigravity-web-search-validation.test.ts +++ b/tests/server/server-antigravity-web-search-validation.test.ts @@ -12,6 +12,8 @@ import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; import { removeTreeWithRetry } from "../helpers/remove-tree"; import * as retry from "../../src/lib/upstream-retry"; import * as webSearch from "../../src/web-search"; +import * as adapterResolve from "../../src/server/adapter-resolve"; +import * as pacing from "../../src/providers/request-pacing"; import { createRequestExecutionBudget } from "../../src/lib/request-execution-budget"; const DAILY_API_BASE = "https://daily-cloudcode-pa.googleapis.com"; @@ -373,26 +375,53 @@ test.each(["non-replayable", "committed output", "non-Google adapter"])( }, ); -test("iteration cleanup releases a sibling permit if construction ends before physical dispatch", async () => { - await seedAntigravityAccounts(2); - const cfg = webSearchConfig(); - const sendBudget = budget(2); - const loopSpy = spyOn(webSearch, "runWithWebSearch").mockImplementation(async deps => { - const refusal = new Response(structuredRefusal, { status: 403 }); - const replacement = await deps.on429?.(null, refusal.headers, deps.parsed, refusal); - expect(replacement?.recoveryKind).toBe("oauth-account-403"); - expect(sendBudget.used).toBe(1); - expect(sendBudget.remainingBaseSends(2)).toBe(1); - deps.onIterationEnd?.(); - expect(sendBudget.used).toBe(0); - expect(sendBudget.remainingBaseSends(2)).toBe(2); - return refusal; - }); - try { - const response = await handleResponses(request(), cfg, route, { sendBudget }); - await response.text(); - } finally { loopSpy.mockRestore(); } -}); +test.each(["construction", "admission", "admission cancellation"])( + "real web-search sibling %s failure refunds the unused send and preserves error ownership", + async failure => { + const accounts = await seedAntigravityAccounts(2); + const cfg = webSearchConfig(); + const before = durableHealth(); + const sendBudget = budget(2); + const abort = new AbortController(); + const sends: Array<{ auth: string; project: string }> = []; + let rejected = 0; + installAntigravityFetchMock(({ auth, project }) => { + sends.push({ auth, project }); + return new Response(structuredRefusal, { status: 403 }); + }); + const resolve = adapterResolve.resolveAdapter; + const wait = pacing.waitForProviderRequestSlot; + const resolveSpy = spyOn(adapterResolve, "resolveAdapter").mockImplementation((provider, ...args) => { + const adapter = resolve(provider, ...args); + if (failure === "construction" && provider.project === accounts[1]!.project) { + adapter.buildRequest = async () => { rejected++; throw new Error("replacement-build-canary"); }; + } + return adapter; + }); + const pacingSpy = spyOn(pacing, "waitForProviderRequestSlot").mockImplementation(async (name, provider, ...args) => { + if (failure.startsWith("admission") && provider.project === accounts[1]!.project) { + rejected++; + if (failure === "admission cancellation") abort.abort(); + throw new Error("replacement-admission-canary"); + } + return wait(name, provider, ...args); + }); + try { + const response = await handleResponses(request(), cfg, route, { sendBudget, abortSignal: abort.signal }); + const body = await response.text(); + expect(rejected).toBeGreaterThan(0); + expect(response.status).toBe(failure === "admission cancellation" ? 499 : 403); + if (failure !== "admission cancellation") { + expect(body).toContain("Antigravity account validation required (VALIDATION_REQUIRED): validate"); + } + expect(body).not.toContain("canary"); + expect(sends).toEqual([{ auth: accounts[0]!.auth, project: accounts[0]!.project }]); + expect(sendBudget.used).toBe(1); + expect(sendBudget.remainingBaseSends(2)).toBe(1); + expect(durableHealth()).toEqual(before); + } finally { resolveSpy.mockRestore(); pacingSpy.mockRestore(); } + }, +); test("web-search 429 recovery still uses the sibling's identity and charges exactly two physical sends", async () => { const accounts = await seedAntigravityAccounts(2);