diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts index 84c78b91453c2..b6ff070c4bbf7 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts @@ -5611,6 +5611,37 @@ describe("CodexAdapterV2 post-settle continuation", () => { notification: false, expectedClass: "provider_error", }, + ...([false, true] as const).map( + (notification) => + ({ + name: `disconnected-${notification ? "notification" : "completion"}`, + code: "responseStreamDisconnected", + errorInfo: { responseStreamDisconnected: { httpStatusCode: null } }, + notification, + expectedClass: "transport_error", + }) as const, + ), + { + name: "connection-failed", + code: "responseStreamConnectionFailed", + errorInfo: { responseStreamConnectionFailed: { httpStatusCode: null } }, + notification: false, + expectedClass: "transport_error", + }, + { + name: "http-connection-failed", + code: "httpConnectionFailed", + errorInfo: { httpConnectionFailed: { httpStatusCode: null } }, + notification: false, + expectedClass: "transport_error", + }, + { + name: "http-provider-failure", + code: "httpConnectionFailed", + errorInfo: { httpConnectionFailed: { httpStatusCode: 503 } }, + notification: false, + expectedClass: "provider_error", + }, { name: "chatgpt-workspace-limit", code: "other", @@ -5651,7 +5682,15 @@ describe("CodexAdapterV2 post-settle continuation", () => { errorInfo: { responseStreamDisconnected: { httpStatusCode: null } }, message: "Transport unavailable; this is not subscription_sharing_usage_limit_exceeded.", notification: false, - expectedClass: "provider_error", + expectedClass: "transport_error", + }, + { + name: "terminal-disconnection", + code: "responseStreamDisconnected", + notification: false, + terminalMessage: "Connection lost.", + terminalErrorInfo: { responseStreamDisconnected: { httpStatusCode: null } }, + expectedClass: "transport_error", }, { name: "sharing-code-mention-retry", @@ -5662,6 +5701,71 @@ describe("CodexAdapterV2 post-settle continuation", () => { willRetry: true, expectedClass: "transport_error", }, + ...( + [ + { + name: "generic-disconnection", + status: null, + terminalMessage: "Connection lost.", + expectedClass: "transport_error", + }, + { + name: "generic-replacement", + status: null, + terminalMessage: "A different failure.", + expectedClass: "provider_error", + }, + { + name: "generic-http-status", + status: 503, + terminalMessage: "Connection lost.", + expectedClass: "provider_error", + }, + { + name: "generic-truncated-replacement", + status: null, + details: `${"Connection lost. ".repeat(300)}Network cause.`, + terminalMessage: `${"Connection lost. ".repeat(300)}Different cause.`, + expectedClass: "provider_error", + }, + { + name: "generic-terminal-status", + status: null, + terminalMessage: "Connection lost.", + terminalErrorInfo: { responseStreamDisconnected: { httpStatusCode: 503 } }, + expectedClass: "provider_error", + }, + { + name: "generic-exhaustion", + status: null, + terminalMessage: "Connection lost.", + terminalErrorInfo: { responseTooManyFailedAttempts: { httpStatusCode: null } }, + completionErrorInfo: { responseTooManyFailedAttempts: { httpStatusCode: null } }, + expectedClass: "transport_error", + }, + { + name: "generic-blank-details", + status: null, + details: " ", + terminalMessage: "Provider stopped this request.", + expectedClass: "transport_error", + }, + { + name: "generic-exhaustion-status", + status: null, + terminalMessage: "Connection lost.", + terminalErrorInfo: { responseTooManyFailedAttempts: { httpStatusCode: 503 } }, + completionErrorInfo: { responseTooManyFailedAttempts: { httpStatusCode: null } }, + expectedClass: "provider_error", + }, + ] as const + ).map((scenario) => ({ + ...scenario, + code: "responseStreamDisconnected" as const, + errorInfo: { responseStreamDisconnected: { httpStatusCode: scenario.status } }, + notification: true, + details: "details" in scenario ? scenario.details : "Connection lost.", + })), ] as const)("classifies Codex terminal failures from $name evidence", (scenario) => Effect.scoped( Effect.gen(function* () { @@ -5754,7 +5858,10 @@ describe("CodexAdapterV2 post-settle continuation", () => { params: { threadId: nativeThreadId, turnId: nativeTurnId, - willRetry: scenario.name === "retry" || "willRetry" in scenario, + willRetry: + scenario.name === "retry" || + "willRetry" in scenario || + "terminalMessage" in scenario, error: { message, codexErrorInfo: errorInfo, @@ -5770,6 +5877,27 @@ describe("CodexAdapterV2 post-settle continuation", () => { }, ] : []), + ...("terminalMessage" in scenario + ? [ + { + type: "emit_inbound" as const, + label: "error/terminal", + frame: { + method: "error", + params: { + threadId: nativeThreadId, + turnId: nativeTurnId, + willRetry: false, + error: { + message: scenario.terminalMessage, + codexErrorInfo: + "terminalErrorInfo" in scenario ? scenario.terminalErrorInfo : "other", + }, + }, + }, + }, + ] + : []), { type: "emit_inbound", label: "turn/completed", @@ -5780,12 +5908,24 @@ describe("CodexAdapterV2 post-settle continuation", () => { turn: { ...makeCodexReplayTurn({ id: nativeTurnId, status: "failed" }), error: { - message: scenario.name === "replacement" ? "A different failure." : message, - ...(scenario.notification && - scenario.name !== "matching-details" && - !("details" in scenario) - ? {} - : { codexErrorInfo: errorInfo }), + message: + "terminalMessage" in scenario + ? scenario.terminalMessage + : scenario.name === "replacement" + ? "A different failure." + : message, + ...("terminalMessage" in scenario + ? { + codexErrorInfo: + "completionErrorInfo" in scenario + ? scenario.completionErrorInfo + : "other", + } + : scenario.notification && + scenario.name !== "matching-details" && + !("details" in scenario) + ? {} + : { codexErrorInfo: errorInfo }), }, }, }, @@ -5905,35 +6045,78 @@ describe("CodexAdapterV2 post-settle continuation", () => { // regression: #17501 Codex 429 retry exhaustion classifies as a usage limit it.effect.each([ { + code: "responseTooManyFailedAttempts", notificationStatus: 500, completionStatus: 429, willRetry: false, expectedClass: "usage_limit", }, { + code: "responseTooManyFailedAttempts", notificationStatus: 429, completionStatus: 500, willRetry: true, expectedClass: "provider_error", }, { + code: "responseTooManyFailedAttempts", notificationStatus: 429, completionStatus: null, willRetry: false, expectedClass: "usage_limit", }, { + code: "responseTooManyFailedAttempts", notificationStatus: 429, completionStatus: null, willRetry: true, expectedClass: "usage_limit", }, + ...( + [ + "responseStreamDisconnected", + "responseStreamConnectionFailed", + "httpConnectionFailed", + ] as const + ).flatMap( + (code) => + [ + { + code, + notificationStatus: null, + completionStatus: 503, + willRetry: false, + expectedClass: "provider_error", + }, + { + code, + notificationStatus: 503, + completionStatus: null, + willRetry: false, + expectedClass: "transport_error", + }, + ] as const, + ), + { + code: "responseStreamDisconnected", + notificationStatus: null, + completionStatus: 503, + willRetry: true, + expectedClass: "provider_error", + }, + { + code: "responseStreamDisconnected", + notificationStatus: null, + completionStatus: null, + willRetry: false, + expectedClass: "transport_error", + }, ] as const)( - "honors completion HTTP status from $notificationStatus to $completionStatus (retry: $willRetry)", - ({ notificationStatus, completionStatus, expectedClass, willRetry }) => + "honors $code completion HTTP status from $notificationStatus to $completionStatus (retry: $willRetry)", + ({ code, notificationStatus, completionStatus, expectedClass, willRetry }) => Effect.scoped( Effect.gen(function* () { - const name = `${notificationStatus}-${completionStatus}-${willRetry}`; + const name = `${code}-${notificationStatus}-${completionStatus}-${willRetry}`; const nativeThreadId = `native-exhaustion-status-${name}`; const nativeTurnId = `turn-exhaustion-status-${name}`; const message = "Provider stopped this request."; @@ -5954,7 +6137,7 @@ describe("CodexAdapterV2 post-settle continuation", () => { error: { message, codexErrorInfo: { - responseTooManyFailedAttempts: { httpStatusCode: notificationStatus }, + [code]: { httpStatusCode: notificationStatus }, }, additionalDetails, }, @@ -5973,7 +6156,7 @@ describe("CodexAdapterV2 post-settle continuation", () => { error: { message, codexErrorInfo: { - responseTooManyFailedAttempts: { httpStatusCode: completionStatus }, + [code]: { httpStatusCode: completionStatus }, }, }, }, @@ -5997,9 +6180,16 @@ describe("CodexAdapterV2 post-settle continuation", () => { assert.equal(terminal?.status, "failed"); if (terminal?.status !== "failed") return; assert.equal(terminal.failure.class, expectedClass); - assert.equal(terminal.failure.code, "responseTooManyFailedAttempts"); + assert.equal(terminal.failure.code, code); assert.equal(terminal.threadDisposition, "reusable"); - const notificationClass = notificationStatus === 429 ? "usage_limit" : "provider_error"; + const notificationClass = + code === "responseTooManyFailedAttempts" + ? notificationStatus === 429 + ? "usage_limit" + : "provider_error" + : notificationStatus === null + ? "transport_error" + : "provider_error"; assert.equal( terminal.failure.message, expectedClass === notificationClass ? additionalDetails : message, diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts index 07c4e4e72eca7..26ab4a31ee4ce 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts @@ -1146,6 +1146,35 @@ function codexUsageLimitError( ); } +function codexTerminalFailureClass( + errorInfo: CodexErrorInfo | null | undefined, + code: string | null, +) { + if (codexUsageLimitError(code, errorInfo)) return "usage_limit"; + if (typeof errorInfo === "object" && errorInfo !== null) { + const transport = + "responseStreamDisconnected" in errorInfo + ? errorInfo.responseStreamDisconnected + : "responseStreamConnectionFailed" in errorInfo + ? errorInfo.responseStreamConnectionFailed + : "httpConnectionFailed" in errorInfo + ? errorInfo.httpConnectionFailed + : undefined; + if (transport !== undefined && transport.httpStatusCode == null) return "transport_error"; + } + return "provider_error"; +} + +function codexTerminalErrorNeedsEvidence( + errorInfo: CodexErrorInfo | null | undefined, + code: string | null, +) { + return ( + code === "other" || + (code === "responseTooManyFailedAttempts" && codexExhaustionHttpStatus(errorInfo) === null) + ); +} + /** * ChatGPT sharing reports its own reason inside Codex's generic turn error, so * the code it names replaces Codex's, and clients pick a recovery from it. Its @@ -1184,6 +1213,7 @@ interface ActiveCodexTurnContext { latestProviderFailure?: { readonly nativeMessage: string; readonly nativeCode: string | null; + readonly errorInfo?: CodexErrorInfo | null; readonly failure: OrchestrationV2ProviderFailure; }; readonly nativeStartReady?: Deferred.Deferred; @@ -1214,6 +1244,8 @@ interface ActiveCodexProviderRetry { readonly context?: ActiveCodexTurnContext; readonly nativeMessage: string; readonly nativeCode: string | null; + readonly nativeDetails?: string; + readonly errorInfo?: CodexErrorInfo | null; readonly retry: OrchestrationV2ProviderRetry; readonly failure: OrchestrationV2ProviderFailure; readonly startedAt: DateTime.Utc; @@ -4708,13 +4740,17 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( context.latestProviderFailure = { nativeMessage: payload.error.message, nativeCode: notificationCode, + ...(payload.error.codexErrorInfo === undefined + ? {} + : { errorInfo: payload.error.codexErrorInfo }), failure: withCodexManagedFailure( makeProviderFailure({ message: payload.error.additionalDetails?.trim() || payload.error.message, code: notificationCode, - class: codexUsageLimitError(notificationCode, payload.error.codexErrorInfo) - ? "usage_limit" - : "provider_error", + class: codexTerminalFailureClass( + payload.error.codexErrorInfo, + notificationCode, + ), }), payload.error, ), @@ -4754,6 +4790,12 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( ...(previous?.context === undefined ? {} : { context: previous.context }), nativeMessage: payload.error.message, nativeCode: code, + nativeDetails: additionalDetails + ? (payload.error.additionalDetails ?? additionalDetails) + : payload.error.message, + ...(payload.error.codexErrorInfo === undefined + ? {} + : { errorInfo: payload.error.codexErrorInfo }), retry, failure, startedAt: previous?.startedAt ?? updatedAt, @@ -5806,19 +5848,20 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( input.failureErrorInfo === undefined ? undefined : codexErrorInfoCode(input.failureErrorInfo); - const failureClass = codexUsageLimitError(failureCode ?? null, input.failureErrorInfo) - ? "usage_limit" - : "provider_error"; - // The notification carries the richer message, so it wins when the - // completion repeats it. An explicit exhaustion status on the - // completion still decides the class; a missing one is not evidence. - const exhaustionStatus = codexExhaustionHttpStatus(input.failureErrorInfo); - const failure = + const failureClass = codexTerminalFailureClass( + input.failureErrorInfo, + failureCode ?? null, + ); + // Retry exhaustion without a status cannot replace the notification's evidence. + const completionHasClass = + failureCode != null && + !codexTerminalErrorNeedsEvidence(input.failureErrorInfo, failureCode); + let failure = previousFailure !== undefined && (input.failureMessage === undefined || input.failureMessage === previousFailure.nativeMessage) && (failureCode === undefined || failureCode === previousFailure.nativeCode) && - (exhaustionStatus === null || previousFailure.failure.class === failureClass) + (!completionHasClass || previousFailure.failure.class === failureClass) ? previousFailure.failure : withCodexManagedFailure( makeProviderFailure({ @@ -5828,6 +5871,28 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( }), input.failureMessage, ); + const retry = input.providerRetry; + const notification = input.context.latestProviderFailure; + // Codex 0.164 reports an exhausted stream error as `other` with the same details. + if ( + codexTerminalErrorNeedsEvidence(input.failureErrorInfo, failure.code) && + failure.class === "provider_error" && + (notification === undefined || + notification.failure.class === "transport_error" || + codexTerminalErrorNeedsEvidence( + notification.errorInfo, + notification.nativeCode, + )) && + ((retry !== undefined && + input.failureMessage !== undefined && + input.failureMessage === retry.nativeDetails && + codexTerminalFailureClass(retry.errorInfo, retry.nativeCode) === + "transport_error") || + (notification?.failure.class === "transport_error" && + input.failureMessage === notification.nativeMessage)) + ) { + failure = { ...failure, class: "transport_error" }; + } return { type: "turn.terminal", driver: CODEX_PROVIDER,