diff --git a/docs-site/src/content/docs/reference/proxy-formats.md b/docs-site/src/content/docs/reference/proxy-formats.md index cce9d5a92e4..8daa0f6333e 100644 --- a/docs-site/src/content/docs/reference/proxy-formats.md +++ b/docs-site/src/content/docs/reference/proxy-formats.md @@ -302,7 +302,20 @@ returns an error instead of partial JSON with HTTP 200. Streaming callers are un When a provider filters or truncates a response, an unfinished tool call remains `incomplete` in both JSON and SSE. Partial output is preserved, and the bridge does not emit an argument completion event for that open call. Calls already completed keep their status. This preserves -the provider outcome; client retry behavior for incomplete responses is unchanged. +the provider outcome. + +On the Responses streaming wire, an explicit non-retryable content-filter stop from +Anthropic (`refusal` / `content_filter`) or Kiro (`CONTENT_FILTERED` / +`GUARDRAIL_INTERVENED`) ends with `response.failed`, `error.code: "invalid_prompt"`, +and `retryable: false`. This prevents clients that retry incomplete streams from +resending the refused context. Usage and partial output are preserved. Web-search +and image sidecars stop without executing calls from that refused iteration or +asking the model again, on every inbound wire. + +Buffered Responses JSON retains `status: "incomplete"`; Chat Completions retains +`finish_reason: "content_filter"`, and Messages retains `stop_reason: "refusal"`. +A content filter without an explicit non-retryable verdict, token limits, stalls, +and EOFs keep their existing incomplete behavior. On the pending `dev` implementation for #4112, a final upstream HTTP 413 on this surface is classified as `invalid_request_error` / `context_length_exceeded`. Non-streaming callers diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index 470fba1b3a6..e165f3e5bf8 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -3,6 +3,11 @@ "root": "tests", "explicitShards": ["layout-clients.json"], "explicit": { "service-log.test.ts": "service", "messages-native-empty-inline.test.ts": "claude-integration", + "bridge-terminal-refusal.test.ts": "adapters", + "loop-terminal-refusal.test.ts": "images", + "web-search-terminal-refusal.test.ts": "web-search", + "combo-terminal-refusal.test.ts": "routing", + "server-terminal-refusal-wiring.test.ts": "server", "owner-registry-isolation.test.ts": "config", "codex-history-pending-wal.test.ts": "codex-integration", "responses-anthropic-billing-send-budget.test.ts": "responses", "anthropic-billing-ownership.test.ts": "adapters/anthropic", "server-anthropic-billing.test.ts": "server", "server-anthropic-billing-boundaries.test.ts": "server", "router-anthropic-billing-failover.test.ts": "routing", "anthropic-billing-refusal.test.ts": "adapters/anthropic", "messages-request-id-headers.test.ts": "claude-integration", "bounded-file-read.test.ts": "lib", "responses-undeclared-tool-guard-inbound.test.ts": "responses", "explicit-empty-tool-catalog.test.ts": "responses", "explicit-empty-tool-catalog-history.test.ts": "responses", "messages-request-id-endpoint.test.ts": "claude-integration", diff --git a/src/adapters/anthropic.ts b/src/adapters/anthropic.ts index 0211aef03fc..187a400d1fb 100644 --- a/src/adapters/anthropic.ts +++ b/src/adapters/anthropic.ts @@ -312,11 +312,12 @@ function isAnthropicContentFilterStopReason( } /** - * Anthropic `refusal` / `content_filter` is a permanent sampling decision, not a disconnect. - * Emitting `done` with that stopReason used to surface as `response.incomplete` without - * `retryable`, which Codex treats as a dropped stream and retries five times (#4312). - * The explicit incomplete event is what the bridge already forwards into - * `incomplete_details.retryable`. Usage is preserved: a filtered turn still consumed tokens. + * Anthropic `refusal` / `content_filter` is a permanent sampling decision, not a disconnect + * (#4312). Codex does not read `incomplete_details.retryable`: it retries every + * `response.incomplete`. The explicit `retryable: false` verdict here is the marker the bridge + * uses to deliver the turn to a Responses client as a terminal `response.failed` instead + * (`src/bridge/terminal-refusal.ts`); a `done` with that stopReason carries no verdict and + * stays a plain incomplete. Usage is preserved: a filtered turn still consumed tokens. * `max_tokens` stays a `done` so the client can continue from a legitimate truncation. */ function anthropicContentFilterIncomplete( @@ -1222,8 +1223,8 @@ export function createAnthropicAdapter(provider: OcxProviderConfig, cacheRetenti return; } // Refusal / content_filter must not look like a dropped stream. `done` with that - // stopReason becomes `response.incomplete` without `retryable`, and Codex retries - // the same refusal five times (#4312). + // stopReason carries no verdict, so the bridge could not tell it from a retryable + // incomplete and Codex would re-send the same refusal (#4312). if (isAnthropicContentFilterStopReason(pendingStopReason)) { yield anthropicContentFilterIncomplete(pendingStopReason, usageFromAnthropic(pendingUsage)); return; @@ -1418,7 +1419,7 @@ export function createAnthropicAdapter(provider: OcxProviderConfig, cacheRetenti } // Same rule as emitDone: refusal / content_filter is a permanent decision, not a // disconnect. This branch bypasses emitDone, so the check has to be repeated here - // or the EOF route still emits `done` and Codex retries the refusal (#4312). + // or the EOF route emits a verdict-less `done` the bridge cannot make terminal (#4312). if (isAnthropicContentFilterStopReason(pendingStopReason)) { emittedDone = true; yield anthropicContentFilterIncomplete(pendingStopReason, usageFromAnthropic(pendingUsage)); diff --git a/src/bridge/sse.ts b/src/bridge/sse.ts index e34c3bebdf6..5f4659e8f8c 100644 --- a/src/bridge/sse.ts +++ b/src/bridge/sse.ts @@ -12,7 +12,7 @@ import { classifyError, cyberPolicyErrorType, CYBER_POLICY_ERROR_CODE, - isCyberPolicyCode, + isTerminalRefusalCode, type OcxErrorPayload, } from "../lib/errors"; import { redactSecretString } from "../lib/redact"; @@ -50,6 +50,7 @@ import { } from "../lib/translator-budget"; import { adapterFailureFromEvent, emptyChunks, joinChunks, ownedBudgetAbandonedMs, responsesUsage, toolCallArgumentsCouldBeJson, toolCallArgumentsUsable, uuid, webSearchAction } from "./internal"; import type { OutputItem, StringChunks } from "./internal"; +import { terminalRefusalFailure } from "./terminal-refusal"; function sseEvent(name: string, data: Record): string { return `event: ${name}\ndata: ${JSON.stringify(data)}\n\n`; @@ -120,6 +121,12 @@ export function bridgeToResponsesSSE( * would reach the client under the invented name instead of the normalized one. */ enforceDeclaredToolNames?: boolean; + /** + * Deliver a non-retryable upstream refusal as `response.failed` with a terminal-refusal code + * instead of `response.incomplete` (see `./terminal-refusal`). Set only for the Responses + * inbound wire: the chat and Anthropic encoders read the incomplete to report a filter stop. + */ + terminalRefusalAsFailed?: boolean; /** Declared parameter schema per tool name; repairs integral-float integer args (#1611). */ toolParameterSchemas?: ReadonlyMap>; /** @@ -1287,6 +1294,19 @@ export function bridgeToResponsesSSE( flushHiddenReasoningEnvelope(); options?.onUsage?.(event.usage); await awaitThoughtSignatureDurability(); + const refusal = options?.terminalRefusalAsFailed ? terminalRefusalFailure(event) : undefined; + if (refusal) { + emit("response.failed", { + response: { + ...responseSnapshot("failed", finishedItems, event.endTurn), + usage: responsesUsage(event.usage), + ...refusal, + }, + }); + reportTerminal("failed"); + terminalEvent = true; + break; + } emit("response.incomplete", { response: { ...responseSnapshot("incomplete", finishedItems, event.endTurn), @@ -1325,7 +1345,7 @@ export function bridgeToResponsesSSE( ...(event.usage ? { usage: responsesUsage(event.usage) } : {}), error: failure.error, last_error: failure.error, - ...(isCyberPolicyCode(failure.error.code) + ...(isTerminalRefusalCode(failure.error.code) ? { retryable: false } : event.retryable !== undefined ? { retryable: event.retryable } : {}), }, @@ -1366,7 +1386,7 @@ export function bridgeToResponsesSSE( ...responseSnapshot("failed", finishedItems), error: failure, last_error: failure, - ...(isCyberPolicyCode(failure.code) ? { retryable: false } : {}), + ...(isTerminalRefusalCode(failure.code) ? { retryable: false } : {}), }, }); reportTerminal("failed"); diff --git a/src/bridge/terminal-refusal.ts b/src/bridge/terminal-refusal.ts new file mode 100644 index 00000000000..e7586e26cd2 --- /dev/null +++ b/src/bridge/terminal-refusal.ts @@ -0,0 +1,59 @@ +import type { AdapterEvent } from "../types"; +import type { OcxErrorPayload } from "../lib/errors"; +import { redactSecretString } from "../lib/redact"; + +/** + * The terminal-refusal code Codex ends a turn on. It is a member of `TERMINAL_REFUSAL_CODES` + * in `src/lib/errors.ts`, and the one that says "this prompt, as sent, is refused" without + * claiming a specific policy family the upstream never named. + */ +const TERMINAL_REFUSAL_FAILED_CODE = "invalid_prompt"; + +interface TerminalRefusalFailure { + error: OcxErrorPayload; + last_error: OcxErrorPayload; + retryable: false; +} + +/** + * Client-visible copy: what happened, and that resending the same context cannot help. It points + * at a new conversation rather than a new turn, because a later turn replays the refused context. + */ +function terminalRefusalMessage(detail: string | undefined): string { + const reason = detail?.trim() ? ` (${redactSecretString(detail.trim())})` : ""; + return `The upstream model refused this request${reason}. ` + + "Retrying with the same conversation context will not succeed; change the request or start a new conversation."; +} + +/** + * An adapter `incomplete` that is a non-retryable upstream refusal: a content filter carrying an + * explicit `retryable: false` verdict. A filter without the verdict, an output cap, a stall and an + * adapter EOF do not qualify. The sidecar loops read this too: a refused iteration ends the turn, + * so a sidecar call it carried is neither run nor answered with another model request. + */ +export function isTerminalRefusalIncomplete(event: AdapterEvent | undefined): boolean { + return event?.type === "incomplete" && event.reason === "content_filter" && event.retryable === false; +} + +/** + * `response.failed` fields for an adapter `incomplete` that is a non-retryable upstream refusal + * (#4312), or `undefined` when the event is anything else. The Anthropic and Kiro adapters are + * the ones that emit it today. + * + * codex-rs maps every `response.incomplete` to a retryable stream error and never reads + * `incomplete_details.retryable`, so a refusal delivered that way is re-sent through the whole + * stream retry budget. It only ends the turn on `response.failed` carrying a terminal-refusal + * `error.code`. Only an explicit `retryable: false` content filter qualifies: a filter without + * the verdict, an output cap, a stall and an adapter EOF stay `response.incomplete`. + */ +export function terminalRefusalFailure( + event: Extract, +): TerminalRefusalFailure | undefined { + if (!isTerminalRefusalIncomplete(event)) return undefined; + const error: OcxErrorPayload = { + type: "invalid_request_error", + code: TERMINAL_REFUSAL_FAILED_CODE, + message: terminalRefusalMessage(event.message), + }; + return { error, last_error: error, retryable: false }; +} diff --git a/src/images/loop.ts b/src/images/loop.ts index 4eb6ee68d40..307ee2dd295 100644 --- a/src/images/loop.ts +++ b/src/images/loop.ts @@ -20,6 +20,7 @@ import { isAnthropicInstanceId } from "../providers/anthropic-instance-id"; import { cloneProviderOpaqueToolCallMetadata } from "../responses/provider-opaque-metadata"; import type { AttemptRecoveryKind } from "../usage/log"; import { bridgeToResponsesSSE } from "../bridge"; +import { isTerminalRefusalIncomplete } from "../bridge/terminal-refusal"; import { clearableDeadline, idleDeadline } from "../lib/abort"; import { readBoundedResponseBody } from "../lib/bounded-body"; import { applyUpstreamRecoveryInit, fetchWithResetRetry, prepareSameTarget429Wait } from "../lib/upstream-retry"; @@ -355,6 +356,8 @@ export interface ImageBridgeDeps { onCompletedResponse?: (response: Record, providerState?: OcxProviderContinuationState) => void; /** WebSocket Responses path only — leave response id empty for protocol compatibility. */ forceEmptyResponseId?: boolean; + /** Responses inbound only: deliver a non-retryable upstream refusal as `response.failed`. */ + terminalRefusalAsFailed?: boolean; } /** @@ -865,7 +868,10 @@ export async function runWithImageBridge(deps: ImageBridgeDeps): Promise 0 && !split.hasRealToolCall && !forceFinal; + // A refused iteration also finalizes: fulfilling its media call would re-ask the model + // with the refused context (#7148). + const shouldLoop = split.calls.length > 0 && !split.hasRealToolCall && !forceFinal + && !split.passthrough.some(isTerminalRefusalIncomplete); if (!shouldLoop) { if (hiddenUsage) { for (let i = split.passthrough.length - 1; i >= 0; i--) { @@ -1089,6 +1095,7 @@ export async function runWithImageBridge(deps: ImageBridgeDeps): Promise { const served = transportState.replayOAuthCredentialSnapshot; if (route.providerName === "kiro" && response.status === "completed" && served @@ -636,6 +637,7 @@ export async function executeResponsesSidecars( settings: wsPlan.settings, maxSearches: wsPlan.maxSearches, forceEmptyResponseId: true, + terminalRefusalAsFailed: inboundWire !== "chat" && inboundWire !== "anthropic", abortSignal: options.abortSignal, onFirstOutput: noteSidecarOutput, onRequestBuilt: request => { diff --git a/src/web-search/loop.ts b/src/web-search/loop.ts index ddb95398206..01eb61d5881 100644 --- a/src/web-search/loop.ts +++ b/src/web-search/loop.ts @@ -6,6 +6,7 @@ import { cloneProviderOpaqueToolCallMetadata } from "../responses/provider-opaqu import type { AttemptRecoveryKind } from "../usage/log"; import { isTruncatedStopReason } from "../responses/truncated-stop-reason"; import { bridgeToResponsesSSE } from "../bridge"; +import { isTerminalRefusalIncomplete } from "../bridge/terminal-refusal"; import { guardEmptyToolCatalogStream, type ClientToolCatalogRestriction } from "../lib/empty-tool-catalog"; import { runWebSearch, type SidecarOutcome, type SidecarOutcomeRecorder, type SidecarSettings } from "./executor"; import { runAnthropicWebSearch } from "./anthropic-executor"; @@ -299,6 +300,8 @@ export interface WebSearchLoopDeps { settings: SidecarSettings; maxSearches: number; forceEmptyResponseId?: boolean; + /** Responses inbound only: deliver a non-retryable upstream refusal as `response.failed`. */ + terminalRefusalAsFailed?: boolean; abortSignal?: AbortSignal; recordSidecarOutcome?: SidecarOutcomeRecorder; beforeSidecarDispatch?: () => void; @@ -916,7 +919,10 @@ export async function runWithWebSearch(deps: WebSearchLoopDeps): Promise 0 && !split.hasRealToolCall; + // A refused iteration ends the turn even when it carried a search call: running the + // search would send the refused context back to the model (#7148). + const shouldLoop = split.calls.length > 0 && !split.hasRealToolCall + && !split.passthrough.some(isTerminalRefusalIncomplete); if (!shouldLoop) { // #1001: a forced-answer pass that ends `done` must have produced // usable output — never a malformed tool call, and never silence. @@ -1008,6 +1014,7 @@ export async function runWithWebSearch(deps: WebSearchLoopDeps): Promise.fastEnabled` is `true`. An off switch is provider capability `false`, applied in the FastPolicy authority (`service-tier.ts`), `resolveModelPolicy`, and router registry enrichment, so no model-level Fast toggle, `--fast` row, or proxy-generated `speed` field is produced. Native Claude Messages passthrough still forwards a `speed` field the caller sends itself, outside the proxy Fast policy. `tests/adapters/anthropic/anthropic-fast-opt-in.test.ts` pins the default, the switch, and the management PATCH/GET. | +| `src/adapters/anthropic.ts` | Anthropic Messages bridge. A `refusal` or `content_filter` stop reason yields an explicit `incomplete` event with `retryable: false` rather than `done` with that stopReason (#4312); `max_tokens` remains `done`. For a Responses client the bridge delivers that event, and the same one from the Kiro adapter, as a terminal `response.failed` ([Adapter-to-Responses bridge](transports/responses.md#adapter-to-responses-bridge)). It is the wire that defines `tools[*].strict` and `tools[*].allowed_callers`, so a rebuilt declaration carries both: an explicit `strict: true` and any `allowed_callers` the caller declared. An absent `strict` stays absent, because the Messages inbound records it as `false` and a `false` on the wire would read as an opt-out nobody asked for. Anthropic Fast uses the native `anthropic-speed` FastWire: a set decision sends `speed: "fast"` with `fast-mode-2026-02-01` in one case-insensitively merged, deduplicated `anthropic-beta` header that preserves OAuth betas. Stream and buffered `usage.speed` echoes confirm fast or downgrade to standard; no echo leaves the request assumed. `tests/adapters/anthropic/anthropic-fast-speed.test.ts` pins the wire and echoes. Anthropic Fast is opt-in: the registry marks both Anthropic entries `fastOptIn`, and `src/providers/fast-opt-in.ts` (`providerFastSwitchOff`) keeps Fast off until `providers..fastEnabled` is `true`. An off switch is provider capability `false`, applied in the FastPolicy authority (`service-tier.ts`), `resolveModelPolicy`, and router registry enrichment, so no model-level Fast toggle, `--fast` row, or proxy-generated `speed` field is produced. Native Claude Messages passthrough still forwards a `speed` field the caller sends itself, outside the proxy Fast policy. `tests/adapters/anthropic/anthropic-fast-opt-in.test.ts` pins the default, the switch, and the management PATCH/GET. | | `src/adapters/anthropic-model-contract.ts` | Per-family Anthropic Messages wire rules, shared by the adapter and the web-search and vision sidecars, measured live on 2026-09-29, with Haiku 5.5 documented on 2026-10-08: adaptive vs budget thinking (Haiku 5.5+ is adaptive); explicit `thinking: disabled` (Sonnet 5.0 up to but excluding 5.5, Haiku 5.5+ without effort); the `between_tools` floor that replaces it on Sonnet 5.5+ (sent without an effort, since xhigh/max reject it); forced `tool_choice` downgraded to `auto` on Opus 5.5, Fable 5.1+ and Sonnet 5.5+; `temperature`/`top_p` dropped on Opus 4.7+, Sonnet 5+, Haiku 5.5+ and every Fable, which 400 on any non-default value; `top_p` dropped when sent with `temperature` on the 4.5/4.6 families, which take either alone but not both; and sidecar thinking-off fields, a low effort with no `thinking` for Opus 5.5 and Fable, which reject both off switches. Dotted Bedrock ids do not parse as a family, and the Messages-native passthrough forwards caller fields unchanged. `tests/adapters/anthropic/anthropic-sonnet-5-5-contract.test.ts` and `tests/adapters/anthropic/anthropic-haiku-5-5-contract.test.ts` pin the family table. | | `src/adapters/google.ts` | Gemini bridge. The final wire compiler owns [endpoint-scoped tool-schema loss policy](providers/google.md#google-tool-schema-loss-reporting): compatible mode changes no request bytes, strict initial loss creates no physical send, and strict non-direct repair creates no changed repair send. A caller-declared strict tool selects `functionCallingConfig.mode: "VALIDATED"` in place of the absent-choice default; `NONE`, `ANY` and a forced-name choice are stronger constraints the caller asked for and are never overwritten. | | `src/adapters/unique-tool-call-ids.ts` | Request-scoped tool-call-id uniqueness for every `openai-chat` provider. An upstream that mints an id from the call's position in its response repeats `call-0-0` on every turn; a Messages client has already paired that id, drops the duplicate, and is left with a call that has no result, so the turn reads as empty and the model re-issues it indefinitely. Only a **repeat** is rewritten — the first occurrence stays byte-identical, leaving prompt-cache keys, reasoning-replay lookups and already-unique upstreams untouched. The ids to avoid come from the caller's history, captured in `buildRequest` (the only point that sees it) and applied at emission, never at ingestion: ingestion matches streamed deltas against the id upstream sent, so rewriting there would strip a pending call of its identity mid-stream. A repeat takes a `-` suffix, never `_`, because `_` reads as a batch sub-call of ``; occupied-set search resumes by suffix width and retained base prefix, including when siblings converge as `-9` becomes `-10`. Covered by `tests/adapters/openai/openai-chat-tool-call-id-remint.test.ts`. | diff --git a/structure/transports/responses.md b/structure/transports/responses.md index e14ce57c703..3bfbea7d975 100644 --- a/structure/transports/responses.md +++ b/structure/transports/responses.md @@ -589,12 +589,12 @@ keeps only allowlisted transport verdict codes and the explicit local `spend_led `output_tokens_details` are always emitted, with zero defaults, because strict Responses clients deserialize them as required fields. - -Active-turn admission owns workflow admission, so both remain held until a streaming body finishes or -is cancelled. +Active-turn admission owns workflow admission, so both remain held until a streaming body finishes or is cancelled. The shared endpoint path also preserves the cyber-policy stop when a malformed UTF-8 5xx body is replacement-decoded; other malformed-body usage, quota, reset evidence and classification retain the status-only fallback. Rebuilt failures remain non-replayable and cyber-policy failures carry neither `Retry-After` nor quota-reset metadata. The [Responses failover contract](responses-failover.md) owns the bounded recovery and replay decisions. + +On the `responses` inbound wire only (`terminalRefusalAsFailed`, set by the same test as `enforceDeclaredToolNames` in `src/server/responses/adapter-delivery.ts`, `src/server/responses/run-turn-execution.ts` and the sidecar loops started from `src/server/responses/sidecar-execution.ts`), an adapter `incomplete` with `reason: "content_filter"` and `retryable: false` is delivered as `response.failed` whose `error` and `last_error` carry `invalid_request_error` / `invalid_prompt` with `retryable: false` and the turn's usage, built by `src/bridge/terminal-refusal.ts`: Codex retries every `response.incomplete` without reading `incomplete_details.retryable` and ends a turn only on a terminal-refusal `error.code` (#4312). The Anthropic and Kiro adapters emit that event today. Inside a combo a refusal with no output never reaches the client as a stream: `src/server/responses/combo-stream-preflight.ts` converts the zero-output `response.failed` into an HTTP 400 JSON error with the same `invalid_prompt` code, and `comboFailureDecision` stops the chain on it instead of sending the refused prompt to the next target (`tests/routing/combo-terminal-refusal.test.ts`). The web-search and image sidecar loops (`src/web-search/loop.ts`, `src/images/loop.ts`) check `isTerminalRefusalIncomplete` before looping, on every inbound wire: a refused iteration that carried a sidecar call ends the turn instead of running the call and re-asking the model with the refused context (`tests/web-search/web-search-terminal-refusal.test.ts`, `tests/images/loop-terminal-refusal.test.ts`); `src/web-search/run-turn-loop.ts` already stops on any incomplete. Every other incomplete, the chat and Anthropic inbound wires, and `buildResponseJSON` keep the incomplete shape. A `response.failed` built from an adapter error carries `retryable: false` for every code in `isTerminalRefusalCode` (`src/lib/errors.ts`), not only cyber policy. `tests/adapters/bridge-terminal-refusal.test.ts` pins both, `tests/server/server-terminal-refusal-wiring.test.ts` pins which inbound wire opts in, and `tests/server/server-kiro-refusal-e2e.test.ts` pins the adapter, runTurn and both sidecar-loop deliveries. diff --git a/tests/adapters/bridge-terminal-refusal.test.ts b/tests/adapters/bridge-terminal-refusal.test.ts new file mode 100644 index 00000000000..6ad059dcb89 --- /dev/null +++ b/tests/adapters/bridge-terminal-refusal.test.ts @@ -0,0 +1,231 @@ +import { describe, expect, test } from "bun:test"; +import { bridgeToResponsesSSE } from "../../src/bridge"; +import { isTerminalRefusalCode } from "../../src/lib/errors"; +import type { AdapterEvent, OcxUsage } from "../../src/types"; + +/** + * #4312: codex-rs maps every `response.incomplete` to a retryable stream error and never reads + * `incomplete_details.retryable`, so an upstream refusal delivered that way is re-sent through the + * whole stream retry budget. Only `response.failed` with a terminal-refusal `error.code` ends the + * turn, and only the Responses inbound wire opts into that delivery. + */ +type BridgeOptions = NonNullable[7]>; +type Frame = { name: string; data: Record }; + +const TERMINALS = new Set(["response.completed", "response.incomplete", "response.failed"]); +const USAGE: OcxUsage = { inputTokens: 120, outputTokens: 7, totalTokens: 127 }; +const REFUSAL: AdapterEvent = { + type: "incomplete", + reason: "content_filter", + retryable: false, + message: 'upstream ended the turn with stop_reason "refusal"', + usage: USAGE, +}; + +async function frames(events: AdapterEvent[], options?: BridgeOptions): Promise { + async function* source(): AsyncGenerator { + for (const event of events) yield event; + } + const text = await new Response( + bridgeToResponsesSSE(source(), "routed/model", undefined, undefined, undefined, undefined, 2_000, options), + ).text(); + return text.split("\n\n").flatMap(frame => { + const lines = frame.trim().split("\n"); + const name = lines.find(line => line.startsWith("event: "))?.slice(7); + const data = lines.find(line => line.startsWith("data: "))?.slice(6); + return name && data ? [{ name, data: JSON.parse(data) as Record }] : []; + }); +} + +async function terminal(events: AdapterEvent[], options?: BridgeOptions): Promise { + const terminals = (await frames(events, options)).filter(frame => TERMINALS.has(frame.name)); + expect(terminals).toHaveLength(1); + return terminals[0]!; +} + +describe("Responses bridge: terminal upstream refusal", () => { + test("a non-retryable content filter becomes response.failed with a terminal-refusal code", async () => { + const statuses: string[] = []; + const usages: Array = []; + const frame = await terminal([REFUSAL], { + terminalRefusalAsFailed: true, + onTerminal: status => { statuses.push(status); }, + onUsage: usage => { usages.push(usage); }, + }); + + expect(frame.name).toBe("response.failed"); + const response = frame.data.response; + expect(response.status).toBe("failed"); + expect(response.error).toEqual(response.last_error); + expect(response.error.type).toBe("invalid_request_error"); + expect(response.error.code).toBe("invalid_prompt"); + // The code is what ends the Codex turn, so it must stay inside the set Codex treats as final. + expect(isTerminalRefusalCode(response.error.code)).toBe(true); + expect(response.error.message).toContain("refused"); + expect(response.error.message).toContain("will not succeed"); + expect(response.error.message).toContain('stop_reason "refusal"'); + expect(response.error.message).toContain("start a new conversation"); + expect(response.retryable).toBe(false); + expect(response.incomplete_details).toBeUndefined(); + // A refused turn still consumed tokens. + expect(response.usage.input_tokens).toBe(120); + expect(response.usage.output_tokens).toBe(7); + expect(usages).toEqual([USAGE]); + expect(statuses).toEqual(["failed"]); + }); + + test("output opened before the refusal is closed the same way as on the incomplete path", async () => { + const events: AdapterEvent[] = [ + { type: "text_delta", text: "I can" }, + { type: "tool_call_start", id: "call_1", name: "lookup" }, + { type: "tool_call_delta", arguments: '{"q":"par' }, + REFUSAL, + ]; + const done = (all: Frame[]) => all + .filter(frame => frame.name === "response.output_item.done") + .map(frame => [frame.data.item.type, frame.data.item.status]); + + const failed = await frames(events, { terminalRefusalAsFailed: true }); + const incomplete = await frames(events); + + expect(failed.at(-1)?.name).toBe("response.failed"); + expect(incomplete.at(-1)?.name).toBe("response.incomplete"); + expect(done(failed)).toEqual(done(incomplete)); + expect(done(failed)).toEqual([["message", "completed"], ["function_call", "incomplete"]]); + expect(failed.at(-1)!.data.response.output.map((item: { type: string }) => item.type)) + .toEqual(incomplete.at(-1)!.data.response.output.map((item: { type: string }) => item.type)); + }); + + test("a Kiro-shaped refusal is promoted and keeps its end_turn verdict", async () => { + // src/adapters/kiro/stream.ts: CONTENT_FILTERED / GUARDRAIL_INTERVENED, endTurn false, with + // continuation state. A failed turn is never remembered, so the state is not handed out. + const completed: unknown[] = []; + const frame = await terminal([{ + type: "incomplete", + reason: "content_filter", + retryable: false, + message: "Kiro stopped with CONTENT_FILTERED before an explicit final answer", + usage: USAGE, + endTurn: false, + providerState: { kiro: { conversationId: "conv_1" } }, + }], { terminalRefusalAsFailed: true, onCompletedResponse: response => { completed.push(response); } }); + + expect(frame.name).toBe("response.failed"); + expect(frame.data.response.error.code).toBe("invalid_prompt"); + expect(frame.data.response.error.message).toContain("CONTENT_FILTERED"); + expect(frame.data.response.retryable).toBe(false); + expect(frame.data.response.end_turn).toBe(false); + expect(completed).toEqual([]); + }); + + test("end_turn is carried only when the adapter set it", async () => { + const options = { terminalRefusalAsFailed: true }; + + expect((await terminal([{ ...REFUSAL, endTurn: true }], options)).data.response.end_turn).toBe(true); + expect((await terminal([{ ...REFUSAL, endTurn: false }], options)).data.response.end_turn).toBe(false); + expect("end_turn" in (await terminal([REFUSAL], options)).data.response).toBe(false); + }); + + test("a refusal without an adapter message still says what happened", async () => { + for (const message of [undefined, "", " "]) { + const frame = await terminal( + [{ type: "incomplete", reason: "content_filter", retryable: false, ...(message !== undefined ? { message } : {}) }], + { terminalRefusalAsFailed: true }, + ); + + expect(frame.name).toBe("response.failed"); + expect(frame.data.response.error.message).toBe( + "The upstream model refused this request. Retrying with the same conversation context will not succeed; " + + "change the request or start a new conversation.", + ); + } + }); + + test("a credential echoed in the adapter message is masked before it reaches the client", async () => { + // Assembled at runtime so the fixture is not itself a credential-shaped literal. + const bearer = ["tok", "en", "0123456789", "abcdef"].join(""); + const apiKey = ["sk", "refusal0123456789"].join("-"); + const frame = await terminal([{ + type: "incomplete", reason: "content_filter", retryable: false, + message: `upstream refused; request carried Authorization: Bearer ${bearer} and api_key=${apiKey}`, + }], { terminalRefusalAsFailed: true }); + + const message: string = frame.data.response.error.message; + expect(frame.name).toBe("response.failed"); + expect(message).toContain("upstream refused; request carried"); + expect(message).not.toContain(bearer); + expect(message).not.toContain(apiKey); + expect(frame.data.response.last_error.message).toBe(message); + }); + + test("without the option the refusal stays response.incomplete", async () => { + for (const options of [undefined, {}, { terminalRefusalAsFailed: false }] as const) { + const statuses: string[] = []; + const frame = await terminal([REFUSAL], { ...options, onTerminal: status => { statuses.push(status); } }); + + expect(frame.name).toBe("response.incomplete"); + expect(frame.data.response.status).toBe("incomplete"); + expect(frame.data.response.error).toBeUndefined(); + expect(frame.data.response.incomplete_details).toEqual({ + reason: "content_filter", + message: 'upstream ended the turn with stop_reason "refusal"', + retryable: false, + }); + expect(frame.data.response.usage.input_tokens).toBe(120); + expect(statuses).toEqual(["incomplete"]); + } + }); + + test("only an explicit non-retryable content filter is promoted", async () => { + const stillIncomplete: AdapterEvent[] = [ + { type: "incomplete", reason: "content_filter", usage: USAGE }, + { type: "incomplete", reason: "content_filter", retryable: true, usage: USAGE }, + { type: "incomplete", reason: "max_output_tokens", usage: USAGE }, + // A cap is a legitimate truncation the client may continue, whatever the verdict says. + { type: "incomplete", reason: "max_output_tokens", retryable: false, usage: USAGE }, + { type: "incomplete", reason: "upstream_stall_timeout", retryable: false }, + ]; + for (const event of stillIncomplete) { + const frame = await terminal([event], { terminalRefusalAsFailed: true }); + + expect(frame.name).toBe("response.incomplete"); + expect(frame.data.response.incomplete_details.reason).toBe((event as { reason: string }).reason); + expect(frame.data.response.error).toBeUndefined(); + } + }); + + test("a done carrying a filter stop reason is not promoted", async () => { + const frame = await terminal( + [{ type: "done", stopReason: "content_filter", usage: USAGE }], + { terminalRefusalAsFailed: true }, + ); + + expect(frame.name).toBe("response.incomplete"); + expect(frame.data.response.incomplete_details.reason).toBe("content_filter"); + }); + + test("an adapter error with a terminal-refusal code is marked non-retryable", async () => { + for (const code of ["invalid_prompt", "misalignment_policy_violation", "bio_policy", "cyber_policy"]) { + // The verdict belongs to the code: an adapter that claims the refusal is retryable is wrong. + for (const retryable of [undefined, true]) { + const frame = await terminal([{ + type: "error", message: "upstream refused the request", status: 400, + errorType: "invalid_request_error", code, + ...(retryable !== undefined ? { retryable } : {}), + }]); + + expect(frame.name).toBe("response.failed"); + expect(frame.data.response.error.code).toBe(code); + expect(frame.data.response.retryable).toBe(false); + } + } + }); + + test("an adapter error with an ordinary code keeps the adapter's own verdict", async () => { + const base = { type: "error", message: "upstream overloaded", status: 503, errorType: "server_error", code: "server_is_overloaded" } as const; + + expect((await terminal([base])).data.response.retryable).toBeUndefined(); + expect((await terminal([{ ...base, retryable: true }])).data.response.retryable).toBe(true); + expect((await terminal([{ ...base, retryable: false }])).data.response.retryable).toBe(false); + }); +}); diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index 44d37a1e5b3..6af3b9a28dd 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -1,4 +1,9 @@ { + "bridge-terminal-refusal.test.ts": "adapters", + "loop-terminal-refusal.test.ts": "images", + "web-search-terminal-refusal.test.ts": "web-search", + "combo-terminal-refusal.test.ts": "routing", + "server-terminal-refusal-wiring.test.ts": "server", "messages-native-empty-inline.test.ts": "claude-integration", "explicit-empty-tool-catalog.test.ts": "responses", "explicit-empty-tool-catalog-history.test.ts": "responses", diff --git a/tests/images/loop-terminal-refusal.test.ts b/tests/images/loop-terminal-refusal.test.ts new file mode 100644 index 00000000000..b43a94f4ea7 --- /dev/null +++ b/tests/images/loop-terminal-refusal.test.ts @@ -0,0 +1,74 @@ +import { afterEach, expect, test } from "bun:test"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { ProviderAdapter } from "../../src/adapters/base"; +import type { AdapterEvent, OcxParsedRequest } from "../../src/types"; +import type { ImageBridgePlan } from "../../src/images/types"; +import { runWithImageBridge } from "../../src/images/loop"; +import { createTestTranslatorBudget } from "../helpers/translator-budget"; +import { removeTreeWithRetry } from "../helpers/remove-tree"; + +/** + * #7148: a refusal that ends an iteration carrying a complete image_gen call must end the turn. + * Fulfilling the call and re-asking would send the refused context back to the model. + */ +const originalFetch = globalThis.fetch; +const previousHome = process.env.OPENCODEX_HOME; +let home = ""; +afterEach(() => { + globalThis.fetch = originalFetch; + if (previousHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previousHome; + if (home) removeTreeWithRetry(home); +}); + +test.each([true, false, undefined])("a refused image_gen iteration stops with terminalRefusalAsFailed=%s", async terminalRefusalAsFailed => { + home = mkdtempSync(join(tmpdir(), "ocx-image-refusal-")); + process.env.OPENCODEX_HOME = home; + let mediaFetches = 0; + globalThis.fetch = (async () => { mediaFetches++; return new Response("unavailable", { status: 503 }); }) as typeof fetch; + let routedRequests = 0; + const adapter: ProviderAdapter = { + name: "mock-refusal", + buildRequest: async () => { + routedRequests++; + return { url: "https://routed.test/v1", method: "POST", headers: {}, body: "{}" }; + }, + fetchResponse: async () => new Response("ok", { status: 200 }), + async *parseStream() { + yield { type: "tool_call_start", id: "call_1", name: "image_gen" } satisfies AdapterEvent; + yield { type: "tool_call_delta", arguments: '{"prompt":"a cat"}' } satisfies AdapterEvent; + yield { type: "tool_call_end" } satisfies AdapterEvent; + yield { + type: "incomplete", reason: "content_filter", retryable: false, + message: 'upstream ended the turn with stop_reason "refusal"', + usage: { inputTokens: 10, outputTokens: 3, totalTokens: 13 }, + } satisfies AdapterEvent; + }, + async parseResponse() { throw new Error("parseResponse must be unreachable"); }, + }; + const plan = { + provider: {} as never, + auth: { baseUrl: "https://api.x.ai", token: "test-token" }, + model: "grok-imagine-image-quality", + toolNames: new Set(["image_gen"]), + } as ImageBridgePlan; + + const response = await runWithImageBridge({ + parsed: { modelId: "test-model", context: { messages: [], tools: [] }, stream: true, options: {} } as OcxParsedRequest, + adapter, + plan, + incomingMeta: { headers: new Headers(), translatorBudget: createTestTranslatorBudget() }, + terminalRefusalAsFailed, + }); + const text = await response.text(); + + expect(routedRequests).toBe(1); + expect(mediaFetches).toBe(0); + expect(text).toContain(`event: response.${terminalRefusalAsFailed ? "failed" : "incomplete"}`); + if (terminalRefusalAsFailed) expect(text).toContain('"code":"invalid_prompt"'); + else expect(text).toContain('"reason":"content_filter"'); + expect(text).not.toContain("event: response.completed"); +}); + diff --git a/tests/routing/combo-terminal-refusal.test.ts b/tests/routing/combo-terminal-refusal.test.ts new file mode 100644 index 00000000000..068b9044b12 --- /dev/null +++ b/tests/routing/combo-terminal-refusal.test.ts @@ -0,0 +1,71 @@ +import { describe, expect, test } from "bun:test"; +import { bridgeToResponsesSSE } from "../../src/bridge"; +import { comboFailureDecision } from "../../src/combos"; +import type { RequestLogContext } from "../../src/server/request-log"; +import { preflightComboStreamResponse } from "../../src/server/responses/combo-stream-preflight"; +import { consumeComboFailure } from "../../src/server/responses/core-combo-failure"; +import type { AdapterEvent } from "../../src/types"; + +/** + * #4312 inside a combo. A refusal with no output is a zero-output `response.failed`, which the + * combo preflight converts into an HTTP error before any byte reaches the client. That must end + * the chain rather than send the refused prompt to the next target, and once output has been + * relayed the same terminal has to stay a streamed `response.failed`. + */ +const REFUSAL: AdapterEvent = { + type: "incomplete", + reason: "content_filter", + retryable: false, + message: 'upstream ended the turn with stop_reason "refusal"', + usage: { inputTokens: 10, outputTokens: 0, totalTokens: 10 }, +}; + +function bridged(events: AdapterEvent[]): Response { + async function* source(): AsyncGenerator { + for (const event of events) yield event; + } + return new Response( + bridgeToResponsesSSE(source(), "routed/model", undefined, undefined, undefined, undefined, 2_000, { + terminalRefusalAsFailed: true, + }), + { headers: { "content-type": "text/event-stream" } }, + ); +} + +describe("combo: terminal upstream refusal", () => { + test("a zero-output refusal becomes an HTTP 400 that stops the chain", async () => { + const logCtx = {} as RequestLogContext; + const preflight = await preflightComboStreamResponse(bridged([REFUSAL]), logCtx); + + if (preflight.kind !== "failed") throw new Error(`expected a failed preflight, got ${preflight.kind}`); + expect(preflight.response.status).toBe(400); + expect(preflight.response.headers.get("content-type")).toContain("application/json"); + const body = await preflight.response.clone().json() as { error: { type: string; code: string; message: string } }; + expect(body.error.type).toBe("invalid_request_error"); + expect(body.error.code).toBe("invalid_prompt"); + expect(body.error.message).toContain("refused"); + // The steps `src/server/responses/core-combo.ts` takes with a failed preflight: consume the + // failure body, carry the preflight's model-refusal evidence over, then decide. The loop's + // other stop conditions (non-replayable marker, spent replacement) are not in play here. + const failure = await consumeComboFailure(preflight.response); + expect(failure.response.status).toBe(400); + expect(failure.upstreamCode).toBe("invalid_prompt"); + expect(comboFailureDecision(failure.response.status, failure.classificationText, { + code: failure.upstreamCode, + codexModelRefusal: preflight.codexModelRefusal ?? failure.codexModelRefusal, + })).toBe("stop"); + }); + + test("a refusal after output is relayed as the streamed response.failed", async () => { + const preflight = await preflightComboStreamResponse( + bridged([{ type: "text_delta", text: "I can" }, REFUSAL]), + {} as RequestLogContext, + ); + + expect(preflight.kind).toBe("accepted"); + const text = await preflight.response.text(); + expect(text).toContain("event: response.failed"); + expect(text).toContain('"code":"invalid_prompt"'); + expect(text).not.toContain("event: response.incomplete"); + }); +}); diff --git a/tests/server/server-kiro-refusal-e2e.test.ts b/tests/server/server-kiro-refusal-e2e.test.ts index 74d17cc9514..eb0b42b5450 100644 --- a/tests/server/server-kiro-refusal-e2e.test.ts +++ b/tests/server/server-kiro-refusal-e2e.test.ts @@ -522,3 +522,84 @@ describe("Kiro refusal recovery through Responses", () => { }); } }); + +/** + * #4312: Kiro's CONTENT_FILTERED / GUARDRAIL_INTERVENED stop is an `incomplete` with + * `retryable: false`. A Responses client must see it as a terminal `response.failed` on every + * delivery that owns a bridge call: the adapter stream, both sidecar loops, and runTurn. No + * runTurn adapter emits this verdict today, so that case drives the same event through a + * `runTurn` stand-in on the Kiro registry entry. + */ +describe("Kiro content-filter stop through Responses streaming", () => { + const frame = (eventType: string, payload: Record) => encodeMessage( + { ":message-type": "event", ":event-type": eventType }, + new TextEncoder().encode(JSON.stringify(payload))); + const filtered = (stopReason: string) => new Response(Buffer.concat([ + frame("assistantResponseEvent", { content: "Partial." }), + frame("metadataEvent", { stopReason }), + ]), { headers: { "content-type": "application/vnd.amazon.eventstream" } }); + const lookupTool = { type: "function", name: "lookup", description: "Look something up.", + parameters: { type: "object", properties: { q: { type: "string" } }, required: ["q"] } }; + const xai = { adapter: "openai-chat", baseUrl: "https://api.x.ai/v1", authMode: "key", + apiKey: "synthetic", models: ["grok-4"] }; + + for (const [label, stopReason, extra, tools] of [ + ["adapter delivery", "CONTENT_FILTERED", {}, [lookupTool]], + ["adapter delivery", "GUARDRAIL_INTERVENED", {}, [lookupTool]], + ["web-search sidecar loop", "CONTENT_FILTERED", + { webSearchSidecar: { backend: "exa", exaApiKey: "synthetic" } }, + [{ type: "web_search" }]], + ["image sidecar loop", "CONTENT_FILTERED", + { images: { bridgeEnabled: true }, providers: { ...config().providers, xai } }, [{ type: "image_generation" }]], + ["runTurn delivery", "CONTENT_FILTERED", {}, [lookupTool]], + ] as const) { + test(`${label}: ${stopReason} ends with one response.failed carrying invalid_prompt`, async () => { + await seed(); + saveConfig({ ...config(), ...extra } as OcxConfig); + let sends = 0; + globalThis.fetch = (async (input, init) => { + const url = input instanceof Request ? input.url : String(input); + if (url.startsWith("https://runtime.") && url.endsWith(".kiro.dev/")) { + sends++; + return filtered(stopReason); + } + return realFetch(input, init); + }) as typeof fetch; + const originalCreate = ADAPTER_REGISTRY.kiro.create; + if (label === "runTurn delivery") { + ADAPTER_REGISTRY.kiro.create = (provider, context) => ({ + ...originalCreate(provider, context), + async runTurn(_parsed, _incoming, emit) { + sends++; + emit({ type: "text_delta", text: "Partial." }); + emit({ type: "incomplete", reason: "content_filter", retryable: false, endTurn: false, + message: `Kiro stopped with ${stopReason} before an explicit final answer` }); + }, + }); + } + const server = startServer(0); + try { + const response = await realFetch(new URL("/v1/responses", server.url), { + method: "POST", headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "kiro/claude-sonnet-4.5", input: "hello", stream: true, tools }), + }); + expect(response.status).toBe(200); + const text = await response.text(); + const terminals = text.split("\n\n").flatMap(block => { + const name = block.split("\n").find(line => line.startsWith("event: "))?.slice(7) ?? ""; + const data = block.split("\n").find(line => line.startsWith("data: "))?.slice(6); + return /^response\.(completed|incomplete|failed)$/.test(name) && data + ? [{ name, response: JSON.parse(data).response as Record }] : []; + }); + expect(terminals.map(terminal => terminal.name)).toEqual(["response.failed"]); + const failed = terminals[0]!.response; + expect(failed.error.type).toBe("invalid_request_error"); + expect(failed.error.code).toBe("invalid_prompt"); + expect(failed.error.message).toContain(stopReason); + expect(failed.retryable).toBe(false); + // The refusal is neither replayed on the same account nor rotated to the other one. + expect(sends).toBe(1); + } finally { ADAPTER_REGISTRY.kiro.create = originalCreate; await server.stop(true); } + }); + } +}); diff --git a/tests/server/server-terminal-refusal-wiring.test.ts b/tests/server/server-terminal-refusal-wiring.test.ts new file mode 100644 index 00000000000..8aaf07d0755 --- /dev/null +++ b/tests/server/server-terminal-refusal-wiring.test.ts @@ -0,0 +1,190 @@ +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { waitForNativeMainStartupGate } from "../../src/codex/native-profile-startup"; +import { saveConfig } from "../../src/config"; +import { startServer } from "../../src/server"; +import type { OcxConfig } from "../../src/types"; +import { installIsolatedCodexHome, type IsolatedCodexHome } from "../helpers/isolated-codex-home"; +import { removeTreeWithRetry } from "../helpers/remove-tree"; + +/** + * #4312, end to end: which inbound wire sees an upstream refusal as what. + * + * `tests/adapters/bridge-terminal-refusal.test.ts` pins the bridge; this pins the wiring — that + * only the Responses inbound wire asks the bridge for `response.failed`, through the real server + * and a real Anthropic-shaped upstream that ends the turn with `stop_reason: "refusal"`. + */ +type Frame = { name?: string; data: Record }; + +const MODEL = "claude-refusal/claude-test"; +const RESPONSES_TERMINALS = new Set(["response.completed", "response.incomplete", "response.failed"]); + +function anthropicSse(stopReason: string): string { + const frame = (name: string, data: Record) => + `event: ${name}\ndata: ${JSON.stringify({ type: name, ...data })}\n\n`; + return [ + frame("message_start", { + message: { id: "msg_1", model: "claude-test", usage: { input_tokens: 10, output_tokens: 1 } }, + }), + frame("content_block_start", { index: 0, content_block: { type: "text", text: "" } }), + frame("content_block_delta", { index: 0, delta: { type: "text_delta", text: "I can" } }), + frame("content_block_stop", { index: 0 }), + frame("message_delta", { delta: { stop_reason: stopReason }, usage: { output_tokens: 3 } }), + frame("message_stop", {}), + ].join(""); +} + +function frames(text: string): Frame[] { + return text.split("\n\n").flatMap(block => { + const lines = block.trim().split("\n"); + const name = lines.find(line => line.startsWith("event: "))?.slice(7); + const data = lines.find(line => line.startsWith("data: "))?.slice(6); + if (!data || data === "[DONE]") return []; + return [{ name, data: JSON.parse(data) as Record }]; + }); +} + +const previousHome = process.env.OPENCODEX_HOME; +let home = ""; +let isolated: IsolatedCodexHome | undefined; +let upstream: ReturnType | undefined; +let server: ReturnType | undefined; +let stopReason = "refusal"; +let upstreamRequests = 0; + +beforeAll(async () => { + upstream = Bun.serve({ + port: 0, + hostname: "127.0.0.1", + async fetch(request) { + upstreamRequests += 1; + const body = await request.json() as { stream?: boolean }; + if (body.stream !== true) { + return Response.json({ + id: "msg_1", type: "message", role: "assistant", model: "claude-test", + content: [{ type: "text", text: "I can" }], + stop_reason: stopReason, stop_sequence: null, + usage: { input_tokens: 10, output_tokens: 3 }, + }); + } + return new Response(anthropicSse(stopReason), { headers: { "content-type": "text/event-stream" } }); + }, + }); + home = mkdtempSync(join(tmpdir(), "ocx-refusal-wiring-")); + isolated = installIsolatedCodexHome("ocx-refusal-wiring-codex-"); + process.env.OPENCODEX_HOME = home; + saveConfig({ + port: 0, + hostname: "127.0.0.1", + defaultProvider: "claude-refusal", + providers: { + "claude-refusal": { + adapter: "anthropic", + baseUrl: `http://127.0.0.1:${upstream.port}`, + apiKey: "sk-test", + // The upstream is a loopback fixture, and nothing here may reach a real catalog. + allowPrivateNetwork: true, + liveModels: false, + defaultModel: "claude-test", + models: ["claude-test"], + }, + }, + } as OcxConfig); + server = startServer(0); + // Requests admitted before the native-main startup gate settles are held, not served. + await waitForNativeMainStartupGate(); +}); + +afterAll(async () => { + await server?.stop(true); + upstream?.stop(true); + // Assigning `undefined` to an env var stores the string "undefined", which then becomes a + // directory name under the working tree. Delete the key when there was nothing to restore. + if (previousHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previousHome; + isolated?.restore(); + if (home) removeTreeWithRetry(home); +}); + +async function post(path: string, body: Record): Promise { + return fetch(new URL(path, server!.url), { + method: "POST", + headers: { "content-type": "application/json", "anthropic-version": "2023-06-01" }, + body: JSON.stringify(body), + }); +} + +describe("upstream refusal, per inbound wire", () => { + test("/v1/responses streaming ends with exactly one response.failed carrying invalid_prompt", async () => { + stopReason = "refusal"; + const before = upstreamRequests; + const response = await post("/v1/responses", { model: MODEL, input: "hi", stream: true }); + const all = frames(await response.text()); + + expect(response.status).toBe(200); + expect(all.map(frame => frame.name).filter(name => RESPONSES_TERMINALS.has(name ?? ""))) + .toEqual(["response.failed"]); + const failed = all.find(frame => frame.name === "response.failed")!.data.response; + expect(failed.status).toBe("failed"); + expect(failed.error).toEqual(failed.last_error); + expect(failed.error.type).toBe("invalid_request_error"); + expect(failed.error.code).toBe("invalid_prompt"); + expect(failed.error.message).toContain('stop_reason "refusal"'); + expect(failed.retryable).toBe(false); + expect(failed.usage.input_tokens).toBe(10); + // The proxy itself must not replay a refusal either. + expect(upstreamRequests - before).toBe(1); + }); + + test("/v1/responses streaming still completes an ordinary turn", async () => { + stopReason = "end_turn"; + const response = await post("/v1/responses", { model: MODEL, input: "hi", stream: true }); + const names = frames(await response.text()).map(frame => frame.name); + + expect(names.filter(name => RESPONSES_TERMINALS.has(name ?? ""))).toEqual(["response.completed"]); + }); + + test("/v1/responses non-streaming keeps the incomplete body", async () => { + stopReason = "refusal"; + const response = await post("/v1/responses", { model: MODEL, input: "hi", stream: false }); + const body = await response.json() as Record; + + expect(response.status).toBe(200); + expect(body.status).toBe("incomplete"); + expect(body.incomplete_details.reason).toBe("content_filter"); + expect(body.error ?? null).toBeNull(); + expect(JSON.stringify(body)).not.toContain("invalid_prompt"); + }); + + test("/v1/chat/completions streaming reports finish_reason content_filter", async () => { + stopReason = "refusal"; + const response = await post("/v1/chat/completions", { + model: MODEL, messages: [{ role: "user", content: "hi" }], stream: true, + }); + const text = await response.text(); + const finishes = frames(text) + .flatMap(frame => (frame.data.choices ?? []) as Array<{ finish_reason?: string | null }>) + .map(choice => choice.finish_reason) + .filter(Boolean); + + expect(response.status).toBe(200); + expect(finishes).toEqual(["content_filter"]); + expect(text).not.toContain("invalid_prompt"); + }); + + test("/v1/messages streaming reports stop_reason refusal", async () => { + stopReason = "refusal"; + const response = await post("/v1/messages", { + model: MODEL, max_tokens: 64, messages: [{ role: "user", content: "hi" }], stream: true, + }); + const text = await response.text(); + const delta = frames(text).find(frame => frame.name === "message_delta"); + + expect(response.status).toBe(200); + expect(delta?.data.delta.stop_reason).toBe("refusal"); + expect(text).not.toContain("invalid_prompt"); + expect(text).not.toContain("event: error"); + }); +}); diff --git a/tests/web-search/web-search-terminal-refusal.test.ts b/tests/web-search/web-search-terminal-refusal.test.ts new file mode 100644 index 00000000000..8f39c61a714 --- /dev/null +++ b/tests/web-search/web-search-terminal-refusal.test.ts @@ -0,0 +1,61 @@ +import { afterEach, expect, test } from "bun:test"; +import type { AdapterEvent } from "../../src/types"; +import type { ProviderAdapter } from "../../src/adapters/base"; +import { parseRequest } from "../../src/responses/parser"; +import { runWithWebSearch } from "../../src/web-search/loop"; +import { createTestTranslatorBudget } from "../helpers/translator-budget"; + +/** + * #7148: a refusal that ends an iteration carrying a complete web_search call must end the turn. + * Running the search and re-asking would send the refused context back to the model, the same + * re-send the Responses bridge's terminal `response.failed` exists to prevent. + */ +const originalFetch = globalThis.fetch; +afterEach(() => { globalThis.fetch = originalFetch; }); + +test.each([true, false, undefined])("a refused web_search iteration stops with terminalRefusalAsFailed=%s", async terminalRefusalAsFailed => { + let searchFetches = 0; + globalThis.fetch = (async () => { + searchFetches++; + return Response.json({ results: [{ title: "Fixture", url: "https://example.test/r", content: "result" }] }); + }) as typeof fetch; + let routedSends = 0; + const adapter: ProviderAdapter = { + name: "mock-refusal", + buildRequest: () => ({ url: "https://routed.test/v1", method: "POST", headers: {}, body: "{}" }), + fetchResponse: async () => { routedSends++; return new Response("ok", { status: 200 }); }, + async *parseStream() { + yield { type: "tool_call_start", id: "search-1", name: "web_search" } satisfies AdapterEvent; + yield { type: "tool_call_delta", arguments: JSON.stringify({ query: "fixture query" }) } satisfies AdapterEvent; + yield { type: "tool_call_end" } satisfies AdapterEvent; + yield { + type: "incomplete", reason: "content_filter", retryable: false, + message: 'upstream ended the turn with stop_reason "refusal"', + usage: { inputTokens: 10, outputTokens: 3, totalTokens: 13 }, + } satisfies AdapterEvent; + }, + async parseResponse() { throw new Error("parseResponse must be unreachable"); }, + }; + + const response = await runWithWebSearch({ + parsed: parseRequest({ model: "routed/model", input: "hi", stream: true, tools: [{ type: "web_search" }] }), + adapter, + incomingMeta: { headers: new Headers(), translatorBudget: createTestTranslatorBudget() }, + backend: "exa", + exaApiKey: "synthetic-exa-key", + hostedTool: { type: "web_search" }, + selectedForwardHeaders: new Headers(), + settings: { model: "exa-fixture-model", reasoning: "low", timeoutMs: 30_000 }, + maxSearches: 2, + terminalRefusalAsFailed, + }); + const text = await response.text(); + + expect(routedSends).toBe(1); + expect(searchFetches).toBe(0); + expect(text).toContain(`event: response.${terminalRefusalAsFailed ? "failed" : "incomplete"}`); + if (terminalRefusalAsFailed) expect(text).toContain('"code":"invalid_prompt"'); + else expect(text).toContain('"reason":"content_filter"'); + expect(text).not.toContain("event: response.completed"); +}); +