From 1de4c8152dc9fdbe605f809e98da5173d57119ad Mon Sep 17 00:00:00 2001 From: haswysa <1344672204@qq.com> Date: Wed, 26 Aug 2026 06:33:06 +0800 Subject: [PATCH 1/3] fix(gateway): failover account on reasoning recovery failure and strip unportable reasoning In multi-account pools, server-side encrypted compaction blobs and reasoning states can fail to decode when a session drifts across accounts or upstream nodes, causing xAI to return HTTP 400 ("Could not decode the compaction blob"). Previously, isRetryable(400) evaluated to false in the gateway service, treating all 400 responses as non-retryable client errors. When provider reasoning recovery failed on the active account, the gateway immediately propagated the 400 to the client instead of attempting other healthy accounts in the candidate pool. Fixes: 1. Gateway failover: Recognize reasoning_recovery_failed in isRetryableResponse so the gateway excludes the failing account and attempts the next candidate within the request attempt loop. 2. Reasoning recovery: Completely strip all reasoning items and convert compaction items to compatibility boundary messages during downgrade retries. 3. History normalization: Convert unencrypted reasoning items without portable content to developer boundary messages. 4. Error markers: Add compaction decode patterns to reasoningDecodeFailureMarkers. Tested: - Added TestRecoverReasoningDecodeFailureCompactionBlob - Updated TestReasoningWithoutEncryptedContentBecomesBoundary - All backend packages pass (go test ./...) --- .../internal/application/gateway/service.go | 29 +++++++++++- .../cli/responses_codex_tools_test.go | 10 ++-- .../infra/provider/cli/responses_history.go | 30 ++++-------- .../cli/responses_reasoning_recovery.go | 47 ++++++++----------- .../cli/responses_reasoning_recovery_test.go | 38 +++++++++++++-- 5 files changed, 94 insertions(+), 60 deletions(-) diff --git a/backend/internal/application/gateway/service.go b/backend/internal/application/gateway/service.go index e6b99685a..bed3e7807 100644 --- a/backend/internal/application/gateway/service.go +++ b/backend/internal/application/gateway/service.go @@ -2092,8 +2092,35 @@ func isRetryable(status int) bool { return status == 402 || status == 403 || status == 429 || status >= 500 } +func isReasoningRecoveryFailedResponse(response *provider.Response) bool { + if response == nil { + return false + } + warnings := response.Header.Get("X-Grok2api-Compatibility-Warnings") + if strings.Contains(warnings, "reasoning_recovery_failed") { + return true + } + if response.Diagnostic != nil { + lower := strings.ToLower(string(response.Diagnostic.Body)) + if strings.Contains(lower, "compaction blob") || + strings.Contains(lower, "encrypted_content") || + strings.Contains(lower, "could not decrypt") || + strings.Contains(lower, "could not decode") || + strings.Contains(lower, "invalid_encrypted_content") { + return true + } + } + return false +} + func isRetryableResponse(response *provider.Response, upstreamProvider accountdomain.Provider) bool { - if response == nil || !isRetryable(response.StatusCode) { + if response == nil { + return false + } + if response.StatusCode == http.StatusBadRequest && isReasoningRecoveryFailedResponse(response) { + return true + } + if !isRetryable(response.StatusCode) { return false } // Account-scoped payment failures must always rotate accounts. diff --git a/backend/internal/infra/provider/cli/responses_codex_tools_test.go b/backend/internal/infra/provider/cli/responses_codex_tools_test.go index 235733020..6c370fc1f 100644 --- a/backend/internal/infra/provider/cli/responses_codex_tools_test.go +++ b/backend/internal/infra/provider/cli/responses_codex_tools_test.go @@ -404,7 +404,7 @@ func TestNativeBuildHistoryItemsArePreservedAndSanitized(t *testing.T) { } } -func TestReasoningWithoutEncryptedContentRemainsNative(t *testing.T) { +func TestReasoningWithoutEncryptedContentBecomesBoundary(t *testing.T) { normalized, _, err := normalizeResponsesRequest([]byte(`{ "model":"public","input":[ {"type":"reasoning","id":"rs_1","status":"completed","summary":[{"type":"summary_text","text":"who am I"}],"content":null,"encrypted_content":null,"internal_chat_message_metadata_passthrough":{"turn_id":"t1"}}, @@ -421,12 +421,8 @@ func TestReasoningWithoutEncryptedContentRemainsNative(t *testing.T) { } items := request["input"].([]any) first := items[0].(map[string]any) - if first["type"] != "reasoning" || first["status"] != nil || first["encrypted_content"] != nil || first["content"] != nil { - t.Fatalf("reasoning history = %#v", first) - } - summary := first["summary"].([]any)[0].(map[string]any) - if summary["text"] != "who am I" { - t.Fatalf("reasoning summary = %#v", first["summary"]) + if first["type"] != "message" || first["role"] != "developer" { + t.Fatalf("unencrypted reasoning should become boundary, got %#v", first) } if items[1].(map[string]any)["content"] != "hi" { t.Fatalf("assistant history = %#v", items[1]) diff --git a/backend/internal/infra/provider/cli/responses_history.go b/backend/internal/infra/provider/cli/responses_history.go index c5675decb..6de683c0f 100644 --- a/backend/internal/infra/provider/cli/responses_history.go +++ b/backend/internal/infra/provider/cli/responses_history.go @@ -259,31 +259,19 @@ func (c *responsesToolCompatibility) normalizeCustomToolCallInput(item map[strin } func sanitizeReasoningInput(item map[string]any) map[string]any { - // 官方 Grok Build 回放 reasoning 时会删除 output-only status,但会保留 - // id、summary、content 和可选 encrypted_content。密文不是回放的前置条件。 - converted := copyNonNullHistoryFields(item, "id", "summary", "content", "encrypted_content") - converted["type"] = "reasoning" - if encrypted, ok := converted["encrypted_content"].(string); ok && strings.TrimSpace(encrypted) != "" { + // Grok Build Responses API 中 type: "reasoning" 严格用于不透明密文回放, + // 上游强制要求携带合法的 encrypted_content。 + // 若缺少 encrypted_content,则降级为 developer 兼容边界消息,杜绝向 Grok 发送裸 reasoning。 + encrypted, ok := item["encrypted_content"].(string) + if ok && strings.TrimSpace(encrypted) != "" { + converted := copyNonNullHistoryFields(item, "id", "summary", "content", "encrypted_content") + converted["type"] = "reasoning" if _, exists := converted["summary"]; !exists { converted["summary"] = []any{} } + return converted } - if !hasPortableReasoningContent(converted) { - return compatibilityBoundaryMessage("A prior model reasoning item was omitted because it has no portable content for Grok Build.") - } - return converted -} - -func hasPortableReasoningContent(item map[string]any) bool { - if encrypted, ok := item["encrypted_content"].(string); ok && strings.TrimSpace(encrypted) != "" { - return true - } - for _, key := range []string{"summary", "content"} { - if values, ok := item[key].([]any); ok && len(values) > 0 { - return true - } - } - return false + return compatibilityBoundaryMessage("A prior model reasoning item was omitted because it has no portable content for Grok Build.") } // sanitizeNativeHistoryInput rebuilds history from native Grok Build InputItem fields diff --git a/backend/internal/infra/provider/cli/responses_reasoning_recovery.go b/backend/internal/infra/provider/cli/responses_reasoning_recovery.go index e0527dced..6fcda885a 100644 --- a/backend/internal/infra/provider/cli/responses_reasoning_recovery.go +++ b/backend/internal/infra/provider/cli/responses_reasoning_recovery.go @@ -15,6 +15,11 @@ import ( var reasoningDecodeFailureMarkers = [][]byte{ []byte("could not decode the compaction blob"), []byte("could not decrypt the provided encrypted_content"), + []byte("invalid_encrypted_content"), + []byte("could not be decrypted or parsed"), + []byte("could not decrypt"), + []byte("decode the compaction blob"), + []byte("compaction blob"), } type reasoningRecoveryOutcome struct { @@ -239,9 +244,12 @@ func isReasoningDecodeFailure(body []byte) bool { return false } -// stripReasoningEncryptedContent removes opaque reasoning state while -// preserving any readable summary/content. An encrypted-only reasoning item -// becomes empty after stripping and is removed entirely. +// stripReasoningEncryptedContent removes undecodable opaque reasoning and compaction +// states so Grok Build does not fail on server-side decryption. +// In Grok Build Responses API, type: "reasoning" is exclusively used for opaque encrypted replay; +// sending type: "reasoning" without encrypted_content is rejected by the upstream. +// Any reasoning item (whether it contains encrypted_content or bare summary) is removed, and any +// compaction item is converted to a compatibility boundary message. func stripReasoningEncryptedContent(body []byte) ([]byte, bool) { var payload map[string]any if json.Unmarshal(body, &payload) != nil { @@ -255,23 +263,21 @@ func stripReasoningEncryptedContent(body []byte) ([]byte, bool) { rebuilt := make([]any, 0, len(input)) for _, raw := range input { item, ok := raw.(map[string]any) - if !ok || stringField(item, "type") != "reasoning" { + if !ok { rebuilt = append(rebuilt, raw) continue } - encrypted, ok := item["encrypted_content"].(string) - if !ok || strings.TrimSpace(encrypted) == "" { - rebuilt = append(rebuilt, raw) + itemType := stringField(item, "type") + if itemType == "reasoning" { + changed = true continue } - cleaned := cloneJSONObject(item) - delete(cleaned, "encrypted_content") - delete(cleaned, "id") - delete(cleaned, "status") - changed = true - if hasReadableReasoningContent(cleaned) { - rebuilt = append(rebuilt, cleaned) + if itemType == "compaction" { + changed = true + rebuilt = append(rebuilt, compatibilityBoundaryMessage("A prior compacted context could not be decoded by upstream. Continue from the retained conversation messages.")) + continue } + rebuilt = append(rebuilt, raw) } if !changed { return body, false @@ -284,19 +290,6 @@ func stripReasoningEncryptedContent(body []byte) ([]byte, bool) { return encoded, true } -func hasReadableReasoningContent(item map[string]any) bool { - for _, field := range []string{"summary", "content"} { - parts, _ := item[field].([]any) - for _, raw := range parts { - part, _ := raw.(map[string]any) - if strings.TrimSpace(stringField(part, "text")) != "" { - return true - } - } - } - return false -} - func appendCompatibilityWarning(header http.Header, warning string) { if header == nil || strings.TrimSpace(warning) == "" { return diff --git a/backend/internal/infra/provider/cli/responses_reasoning_recovery_test.go b/backend/internal/infra/provider/cli/responses_reasoning_recovery_test.go index d5580c973..e7cff4622 100644 --- a/backend/internal/infra/provider/cli/responses_reasoning_recovery_test.go +++ b/backend/internal/infra/provider/cli/responses_reasoning_recovery_test.go @@ -22,6 +22,7 @@ func TestStripReasoningEncryptedContentPreservesOnlyPortableHistory(t *testing.T {"type":"reasoning","id":"rs_empty","status":"completed","summary":[],"encrypted_content":"opaque-empty"}, {"type":"reasoning","summary":[{"type":"summary_text","text":""}],"encrypted_content":"opaque-blank"}, {"type":"reasoning","id":"rs_summary","status":"completed","summary":[{"type":"summary_text","text":"readable"}],"encrypted_content":"opaque-summary"}, + {"type":"compaction","id":"cmp_1","encrypted_content":"opaque-compaction"}, {"type":"message","role":"assistant","content":"answer","encrypted_content":"message-value"}, {"type":"message","role":"user","content":"continue"} ] @@ -34,15 +35,17 @@ func TestStripReasoningEncryptedContentPreservesOnlyPortableHistory(t *testing.T Input []map[string]any `json:"input"` } if json.Unmarshal(downgraded, &payload) != nil || len(payload.Input) != 3 { - t.Fatalf("downgraded = %s", downgraded) + t.Fatalf("downgraded = %s, len=%d", downgraded, len(payload.Input)) } - reasoning := payload.Input[0] - if reasoning["type"] != "reasoning" || reasoning["id"] != nil || reasoning["status"] != nil || reasoning["encrypted_content"] != nil { - t.Fatalf("reasoning = %#v", reasoning) + if payload.Input[0]["type"] != "message" || payload.Input[0]["role"] != "developer" { + t.Fatalf("compaction boundary item = %#v", payload.Input[0]) } if payload.Input[1]["encrypted_content"] != "message-value" { t.Fatalf("non-reasoning encrypted content changed: %#v", payload.Input[1]) } + if payload.Input[2]["role"] != "user" { + t.Fatalf("user message changed: %#v", payload.Input[2]) + } } func TestRecoverReasoningDecodeFailureRetriesSameUpstreamOnce(t *testing.T) { @@ -466,3 +469,30 @@ type reasoningRecoveryFallbackMarker struct{} func (reasoningRecoveryFallbackMarker) MarkBuildAPIFallback(context.Context, uint64, bool) error { return nil } + +func TestRecoverReasoningDecodeFailureCompactionBlob(t *testing.T) { + adapter, encrypted := newReasoningRecoveryTestAdapter(t) + var calls atomic.Int32 + adapter.http.Transport = roundTripFunc(func(request *http.Request) (*http.Response, error) { + call := calls.Add(1) + if call == 1 { + return jsonHTTPResponse(request, http.StatusBadRequest, `{"code":"invalid-argument","error":"Could not decode the compaction blob. Ensure it is unmodified from the compact response."}`), nil + } + return jsonHTTPResponse(request, http.StatusOK, `{"id":"resp_ok","status":"completed","output":[]}`), nil + }) + + body := []byte(`{"model":"grok-4.6","messages":[{"role":"user","content":"hello"}]}`) + resp, err := adapter.ForwardResponse(context.Background(), provider.ResponseResourceRequest{ + Credential: account.Credential{ID: 131, Provider: account.ProviderBuild, EncryptedAccessToken: encrypted}, + Method: http.MethodPost, Path: "/responses", Model: "grok-4.6", PromptCacheKey: "session-compaction-test", + NormalizeBody: true, Operation: conversation.OperationChat, Body: body, + }) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("expected status 200, got %d", resp.StatusCode) + } +} + From 2408b67ddb6e1228979f3d393ee01ac8cb7af3ee Mon Sep 17 00:00:00 2001 From: haswysa <1344672204@qq.com> Date: Wed, 26 Aug 2026 08:05:12 +0800 Subject: [PATCH 2/3] fix(gateway): keep portable reasoning summaries during compaction recovery Same-account recovery now strips ciphertext while rewriting readable summaries as developer messages, continues to session reset when the stripped retry still returns 400, and only rotates accounts after reasoning_recovery_failed. Drop Diagnostic substring matching so unrelated 400s do not scan the account pool. --- .../application/gateway/failure_test.go | 33 +++++++ .../internal/application/gateway/service.go | 13 +-- .../cli/responses_codex_tools_test.go | 8 +- .../infra/provider/cli/responses_history.go | 16 ++-- .../cli/responses_reasoning_recovery.go | 95 +++++++++++++----- .../cli/responses_reasoning_recovery_test.go | 96 +++++++++++++++++-- 6 files changed, 209 insertions(+), 52 deletions(-) diff --git a/backend/internal/application/gateway/failure_test.go b/backend/internal/application/gateway/failure_test.go index 74b6e912d..0a384d4f4 100644 --- a/backend/internal/application/gateway/failure_test.go +++ b/backend/internal/application/gateway/failure_test.go @@ -243,6 +243,39 @@ func TestHTTPUpstreamFailureLeavesPaymentRecoveryKindToBilling(t *testing.T) { } } +func TestRetryableResponseRotatesOnlyOnReasoningRecoveryFailedWarning(t *testing.T) { + failedHeader := make(http.Header) + failedHeader.Set("X-Grok2API-Compatibility-Warnings", "reasoning_encrypted_content_downgraded,reasoning_recovery_failed") + failed := &provider.Response{ + StatusCode: http.StatusBadRequest, + Header: failedHeader, + Body: io.NopCloser(strings.NewReader(`{"error":"Could not decode the compaction blob"}`)), + } + if !isRetryableResponse(failed, accountdomain.ProviderBuild) { + t.Fatal("reasoning_recovery_failed 400 must rotate accounts") + } + + plain := &provider.Response{ + StatusCode: http.StatusBadRequest, + Header: make(http.Header), + Body: io.NopCloser(strings.NewReader(`{"error":"Could not decode the compaction blob"}`)), + Diagnostic: &provider.DiagnosticResponse{Body: []byte(`{"error":"Could not decode the compaction blob. Ensure it is unmodified from the compact response."}`)}, + } + if isRetryableResponse(plain, accountdomain.ProviderBuild) { + t.Fatal("plain compaction 400 must not rotate accounts without reasoning_recovery_failed") + } + + unrelated := &provider.Response{ + StatusCode: http.StatusBadRequest, + Header: make(http.Header), + Body: io.NopCloser(strings.NewReader(`{"error":"invalid request history"}`)), + Diagnostic: &provider.DiagnosticResponse{Body: []byte(`{"error":"could not decode request json"}`)}, + } + if isRetryableResponse(unrelated, accountdomain.ProviderBuild) { + t.Fatal("unrelated 400 must not match encrypted_content/decode substrings") + } +} + func TestRetryableResponseHonorsUpstreamRetryVeto(t *testing.T) { response := &provider.Response{ StatusCode: http.StatusInternalServerError, diff --git a/backend/internal/application/gateway/service.go b/backend/internal/application/gateway/service.go index bed3e7807..1224b3f25 100644 --- a/backend/internal/application/gateway/service.go +++ b/backend/internal/application/gateway/service.go @@ -2096,17 +2096,8 @@ func isReasoningRecoveryFailedResponse(response *provider.Response) bool { if response == nil { return false } - warnings := response.Header.Get("X-Grok2api-Compatibility-Warnings") - if strings.Contains(warnings, "reasoning_recovery_failed") { - return true - } - if response.Diagnostic != nil { - lower := strings.ToLower(string(response.Diagnostic.Body)) - if strings.Contains(lower, "compaction blob") || - strings.Contains(lower, "encrypted_content") || - strings.Contains(lower, "could not decrypt") || - strings.Contains(lower, "could not decode") || - strings.Contains(lower, "invalid_encrypted_content") { + for _, value := range strings.Split(response.Header.Get("X-Grok2API-Compatibility-Warnings"), ",") { + if strings.TrimSpace(value) == "reasoning_recovery_failed" { return true } } diff --git a/backend/internal/infra/provider/cli/responses_codex_tools_test.go b/backend/internal/infra/provider/cli/responses_codex_tools_test.go index 6c370fc1f..44186ee37 100644 --- a/backend/internal/infra/provider/cli/responses_codex_tools_test.go +++ b/backend/internal/infra/provider/cli/responses_codex_tools_test.go @@ -404,7 +404,7 @@ func TestNativeBuildHistoryItemsArePreservedAndSanitized(t *testing.T) { } } -func TestReasoningWithoutEncryptedContentBecomesBoundary(t *testing.T) { +func TestReasoningWithoutEncryptedContentBecomesPortableSummary(t *testing.T) { normalized, _, err := normalizeResponsesRequest([]byte(`{ "model":"public","input":[ {"type":"reasoning","id":"rs_1","status":"completed","summary":[{"type":"summary_text","text":"who am I"}],"content":null,"encrypted_content":null,"internal_chat_message_metadata_passthrough":{"turn_id":"t1"}}, @@ -422,7 +422,11 @@ func TestReasoningWithoutEncryptedContentBecomesBoundary(t *testing.T) { items := request["input"].([]any) first := items[0].(map[string]any) if first["type"] != "message" || first["role"] != "developer" { - t.Fatalf("unencrypted reasoning should become boundary, got %#v", first) + t.Fatalf("unencrypted reasoning should become portable summary, got %#v", first) + } + text := first["content"].([]any)[0].(map[string]any)["text"].(string) + if !strings.Contains(text, "who am I") || strings.Contains(text, "omitted") { + t.Fatalf("portable reasoning text = %q", text) } if items[1].(map[string]any)["content"] != "hi" { t.Fatalf("assistant history = %#v", items[1]) diff --git a/backend/internal/infra/provider/cli/responses_history.go b/backend/internal/infra/provider/cli/responses_history.go index 6de683c0f..070bdbf69 100644 --- a/backend/internal/infra/provider/cli/responses_history.go +++ b/backend/internal/infra/provider/cli/responses_history.go @@ -259,18 +259,20 @@ func (c *responsesToolCompatibility) normalizeCustomToolCallInput(item map[strin } func sanitizeReasoningInput(item map[string]any) map[string]any { - // Grok Build Responses API 中 type: "reasoning" 严格用于不透明密文回放, - // 上游强制要求携带合法的 encrypted_content。 - // 若缺少 encrypted_content,则降级为 developer 兼容边界消息,杜绝向 Grok 发送裸 reasoning。 - encrypted, ok := item["encrypted_content"].(string) - if ok && strings.TrimSpace(encrypted) != "" { - converted := copyNonNullHistoryFields(item, "id", "summary", "content", "encrypted_content") - converted["type"] = "reasoning" + // Keep ciphertext-backed reasoning native. Summary-only items are rewritten + // as developer messages so the readable plan survives without sending a + // bare type=reasoning item that Grok Build may reject. + converted := copyNonNullHistoryFields(item, "id", "summary", "content", "encrypted_content") + converted["type"] = "reasoning" + if encrypted, ok := converted["encrypted_content"].(string); ok && strings.TrimSpace(encrypted) != "" { if _, exists := converted["summary"]; !exists { converted["summary"] = []any{} } return converted } + if portable, ok := portableReasoningSummaryMessage(converted); ok { + return portable + } return compatibilityBoundaryMessage("A prior model reasoning item was omitted because it has no portable content for Grok Build.") } diff --git a/backend/internal/infra/provider/cli/responses_reasoning_recovery.go b/backend/internal/infra/provider/cli/responses_reasoning_recovery.go index 6fcda885a..427a99143 100644 --- a/backend/internal/infra/provider/cli/responses_reasoning_recovery.go +++ b/backend/internal/infra/provider/cli/responses_reasoning_recovery.go @@ -16,10 +16,6 @@ var reasoningDecodeFailureMarkers = [][]byte{ []byte("could not decode the compaction blob"), []byte("could not decrypt the provided encrypted_content"), []byte("invalid_encrypted_content"), - []byte("could not be decrypted or parsed"), - []byte("could not decrypt"), - []byte("decode the compaction blob"), - []byte("compaction blob"), } type reasoningRecoveryOutcome struct { @@ -51,12 +47,14 @@ func (o reasoningRecoveryOutcome) appendWarnings(header http.Header) { // recoverReasoningDecodeFailure handles only the upstream's explicit // pre-generation opaque-reasoning decode rejection. Recovery never changes // credential or Build/XAI plane: -// 1. remove replayed encrypted_content and retry in the same session; -// 2. when the same decode error remains (or no opaque item exists), clear the -// server-side session identity and retry once with the full portable input. +// 1. remove replayed encrypted_content, keep any readable summary as a +// portable developer message, and retry in the same session; +// 2. when a 400 remains (or no opaque item exists), clear the server-side +// session identity and retry once with the full portable input. // -// If recovery is unsuccessful, the original 400 is returned so the Gateway -// does not rotate accounts or obscure the first failure. +// If recovery is unsuccessful, the original 400 is returned with +// reasoning_recovery_failed so the Gateway can rotate accounts. The Provider +// itself never changes credential. func (a *Adapter) recoverReasoningDecodeFailure( ctx context.Context, request provider.ResponseResourceRequest, @@ -109,12 +107,23 @@ func (a *Adapter) recoverReasoningDecodeFailure( a.logReasoningRecovery(request, base, "encrypted_content", "rate_limited", retry.StatusCode, nil) return retry, retryURL, reasoningRecoveryOutcome{encryptedContentDowngraded: true} } + retryStatus := retry.StatusCode sameDecodeFailure, inspectErr := responseHasReasoningDecodeFailure(retry) - if inspectErr != nil || !sameDecodeFailure { - a.logReasoningRecovery(request, base, "encrypted_content", "retry_rejected", retry.StatusCode, inspectErr) + if inspectErr != nil { + a.logReasoningRecovery(request, base, "encrypted_content", "retry_rejected", retryStatus, inspectErr) return original, requestURL, reasoningRecoveryOutcome{failed: true} } - a.logReasoningRecovery(request, base, "encrypted_content", "decode_error_persisted", retry.StatusCode, nil) + if retryStatus != http.StatusBadRequest { + a.logReasoningRecovery(request, base, "encrypted_content", "retry_rejected", retryStatus, nil) + return original, requestURL, reasoningRecoveryOutcome{failed: true} + } + if sameDecodeFailure { + a.logReasoningRecovery(request, base, "encrypted_content", "decode_error_persisted", retryStatus, nil) + } else { + // Stripping ciphertext can reword the 400 (for example a bare + // reasoning item). Keep going to session reset instead of aborting. + a.logReasoningRecovery(request, base, "encrypted_content", "retry_still_400", retryStatus, nil) + } } if !canResetReasoningSession(request, portableBody) { @@ -244,12 +253,11 @@ func isReasoningDecodeFailure(body []byte) bool { return false } -// stripReasoningEncryptedContent removes undecodable opaque reasoning and compaction -// states so Grok Build does not fail on server-side decryption. -// In Grok Build Responses API, type: "reasoning" is exclusively used for opaque encrypted replay; -// sending type: "reasoning" without encrypted_content is rejected by the upstream. -// Any reasoning item (whether it contains encrypted_content or bare summary) is removed, and any -// compaction item is converted to a compatibility boundary message. +// stripReasoningEncryptedContent removes undecodable opaque reasoning and +// compaction ciphertext so Grok Build does not fail on server-side decryption. +// Readable reasoning summaries are kept as portable developer messages; empty +// encrypted-only reasoning items are dropped. Foreign compaction blobs become +// a boundary note because this gateway cannot decrypt them. func stripReasoningEncryptedContent(body []byte) ([]byte, bool) { var payload map[string]any if json.Unmarshal(body, &payload) != nil { @@ -267,17 +275,32 @@ func stripReasoningEncryptedContent(body []byte) ([]byte, bool) { rebuilt = append(rebuilt, raw) continue } - itemType := stringField(item, "type") - if itemType == "reasoning" { + switch stringField(item, "type") { + case "reasoning": + encrypted, hasEncrypted := item["encrypted_content"].(string) + if !hasEncrypted || strings.TrimSpace(encrypted) == "" { + if portable, ok := portableReasoningSummaryMessage(item); ok { + changed = true + rebuilt = append(rebuilt, portable) + continue + } + rebuilt = append(rebuilt, raw) + continue + } changed = true - continue - } - if itemType == "compaction" { + if portable, ok := portableReasoningSummaryMessage(item); ok { + rebuilt = append(rebuilt, portable) + } + case "compaction": changed = true + if portable, ok := portableReasoningSummaryMessage(item); ok { + rebuilt = append(rebuilt, portable) + continue + } rebuilt = append(rebuilt, compatibilityBoundaryMessage("A prior compacted context could not be decoded by upstream. Continue from the retained conversation messages.")) - continue + default: + rebuilt = append(rebuilt, raw) } - rebuilt = append(rebuilt, raw) } if !changed { return body, false @@ -290,6 +313,28 @@ func stripReasoningEncryptedContent(body []byte) ([]byte, bool) { return encoded, true } +func portableReasoningSummaryMessage(item map[string]any) (map[string]any, bool) { + text := reasoningPortableText(item) + if text == "" { + return nil, false + } + return compatibilityBoundaryMessage("Prior model reasoning summary:\n" + text), true +} + +func reasoningPortableText(item map[string]any) string { + var parts []string + for _, field := range []string{"summary", "content"} { + values, _ := item[field].([]any) + for _, raw := range values { + part, _ := raw.(map[string]any) + if text := strings.TrimSpace(stringField(part, "text")); text != "" { + parts = append(parts, text) + } + } + } + return strings.Join(parts, "\n") +} + func appendCompatibilityWarning(header http.Header, warning string) { if header == nil || strings.TrimSpace(warning) == "" { return diff --git a/backend/internal/infra/provider/cli/responses_reasoning_recovery_test.go b/backend/internal/infra/provider/cli/responses_reasoning_recovery_test.go index e7cff4622..a11d63847 100644 --- a/backend/internal/infra/provider/cli/responses_reasoning_recovery_test.go +++ b/backend/internal/infra/provider/cli/responses_reasoning_recovery_test.go @@ -34,17 +34,26 @@ func TestStripReasoningEncryptedContentPreservesOnlyPortableHistory(t *testing.T var payload struct { Input []map[string]any `json:"input"` } - if json.Unmarshal(downgraded, &payload) != nil || len(payload.Input) != 3 { + if json.Unmarshal(downgraded, &payload) != nil || len(payload.Input) != 4 { t.Fatalf("downgraded = %s, len=%d", downgraded, len(payload.Input)) } - if payload.Input[0]["type"] != "message" || payload.Input[0]["role"] != "developer" { - t.Fatalf("compaction boundary item = %#v", payload.Input[0]) + summary := payload.Input[0] + if summary["type"] != "message" || summary["role"] != "developer" { + t.Fatalf("readable reasoning = %#v", summary) } - if payload.Input[1]["encrypted_content"] != "message-value" { - t.Fatalf("non-reasoning encrypted content changed: %#v", payload.Input[1]) + summaryText, _ := summary["content"].([]any)[0].(map[string]any)["text"].(string) + if !strings.Contains(summaryText, "readable") || strings.Contains(summaryText, "omitted") { + t.Fatalf("readable reasoning text = %q", summaryText) } - if payload.Input[2]["role"] != "user" { - t.Fatalf("user message changed: %#v", payload.Input[2]) + compaction := payload.Input[1] + if compaction["type"] != "message" || compaction["role"] != "developer" { + t.Fatalf("compaction boundary item = %#v", compaction) + } + if payload.Input[2]["encrypted_content"] != "message-value" { + t.Fatalf("non-reasoning encrypted content changed: %#v", payload.Input[2]) + } + if payload.Input[3]["role"] != "user" { + t.Fatalf("user message changed: %#v", payload.Input[3]) } } @@ -470,6 +479,79 @@ func (reasoningRecoveryFallbackMarker) MarkBuildAPIFallback(context.Context, uin return nil } +func TestRecoverReasoningDecodeFailureKeepsReadableSummary(t *testing.T) { + adapter, encrypted := newReasoningRecoveryTestAdapter(t) + var calls atomic.Int32 + adapter.http.Transport = roundTripFunc(func(request *http.Request) (*http.Response, error) { + call := calls.Add(1) + data, _ := io.ReadAll(request.Body) + if call == 1 { + if !strings.Contains(string(data), `"encrypted_content":"opaque"`) { + t.Fatalf("first body = %s", data) + } + return jsonHTTPResponse(request, http.StatusBadRequest, `{"error":"Could not decrypt the provided encrypted_content. Ensure the value is unmodified."}`), nil + } + if strings.Contains(string(data), `"encrypted_content"`) || strings.Contains(string(data), `"type":"reasoning"`) { + t.Fatalf("retry still has opaque reasoning: %s", data) + } + if !strings.Contains(string(data), "do not touch Y") { + t.Fatalf("retry dropped readable summary: %s", data) + } + return jsonHTTPResponse(request, http.StatusOK, `{"id":"resp_ok","status":"completed","output":[]}`), nil + }) + response, err := adapter.ForwardResponse(t.Context(), provider.ResponseResourceRequest{ + Credential: account.Credential{ID: 1, Provider: account.ProviderBuild, EncryptedAccessToken: encrypted}, + Method: http.MethodPost, Path: "/responses", Model: "grok-4.5", PromptCacheKey: "session-1", + Body: []byte(`{"model":"grok-4.5","input":[{"type":"reasoning","summary":[{"type":"summary_text","text":"do not touch Y"}],"encrypted_content":"opaque"},{"role":"user","content":"continue"}]}`), + }) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + if calls.Load() != 2 || response.StatusCode != http.StatusOK || !strings.Contains(response.Header.Get("X-Grok2API-Compatibility-Warnings"), "reasoning_encrypted_content_downgraded") { + t.Fatalf("calls=%d status=%d warnings=%q", calls.Load(), response.StatusCode, response.Header.Get("X-Grok2API-Compatibility-Warnings")) + } +} + +func TestRecoverReasoningDecodeFailureContinuesToSessionResetWhenStripRewords400(t *testing.T) { + adapter, encrypted := newReasoningRecoveryTestAdapter(t) + var calls atomic.Int32 + adapter.http.Transport = roundTripFunc(func(request *http.Request) (*http.Response, error) { + call := calls.Add(1) + data, _ := io.ReadAll(request.Body) + switch call { + case 1: + return jsonHTTPResponse(request, http.StatusBadRequest, `{"error":"Could not decrypt the provided encrypted_content. Ensure the value is unmodified."}`), nil + case 2: + if strings.Contains(string(data), `"encrypted_content"`) || request.Header.Get("x-grok-session-id") == "" { + t.Fatalf("opaque downgrade body=%s headers=%#v", data, request.Header) + } + return jsonHTTPResponse(request, http.StatusBadRequest, `{"error":"invalid request history"}`), nil + case 3: + if strings.Contains(string(data), `"encrypted_content"`) || strings.Contains(string(data), `"prompt_cache_key"`) || request.Header.Get("x-grok-session-id") != "" { + t.Fatalf("session reset body=%s headers=%#v", data, request.Header) + } + return jsonHTTPResponse(request, http.StatusOK, `{"id":"resp_ok","status":"completed","output":[]}`), nil + default: + t.Fatalf("unexpected call %d", call) + return nil, nil + } + }) + response, err := adapter.ForwardResponse(t.Context(), provider.ResponseResourceRequest{ + Credential: account.Credential{ID: 1, Provider: account.ProviderBuild, EncryptedAccessToken: encrypted}, + Method: http.MethodPost, Path: "/responses", Model: "grok-4.5", PromptCacheKey: "session-1", + Body: []byte(`{"model":"grok-4.5","input":[{"type":"reasoning","summary":[],"encrypted_content":"opaque"},{"role":"user","content":"continue"}]}`), + }) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + warnings := response.Header.Get("X-Grok2API-Compatibility-Warnings") + if calls.Load() != 3 || response.StatusCode != http.StatusOK || !strings.Contains(warnings, "reasoning_encrypted_content_downgraded") || !strings.Contains(warnings, "reasoning_session_reset") { + t.Fatalf("calls=%d status=%d warnings=%q", calls.Load(), response.StatusCode, warnings) + } +} + func TestRecoverReasoningDecodeFailureCompactionBlob(t *testing.T) { adapter, encrypted := newReasoningRecoveryTestAdapter(t) var calls atomic.Int32 From 7caa52c14fd80e3012356bb058ad0b45a6ad71c2 Mon Sep 17 00:00:00 2001 From: Chenyme <118253778+chenyme@users.noreply.github.com> Date: Mon, 31 Aug 2026 09:40:39 +0800 Subject: [PATCH 3/3] fix: preserve upstream outcomes during reasoning recovery --- .../application/gateway/failure_test.go | 21 ++- .../internal/application/gateway/service.go | 14 +- .../internal/infra/provider/cli/adapter.go | 14 +- .../cli/responses_codex_tools_test.go | 4 +- .../infra/provider/cli/responses_history.go | 2 +- .../cli/responses_reasoning_recovery.go | 80 +++++------ .../cli/responses_reasoning_recovery_test.go | 129 ++++++++++++++++-- backend/internal/infra/provider/provider.go | 4 + 8 files changed, 190 insertions(+), 78 deletions(-) diff --git a/backend/internal/application/gateway/failure_test.go b/backend/internal/application/gateway/failure_test.go index 0a384d4f4..b93aff5d1 100644 --- a/backend/internal/application/gateway/failure_test.go +++ b/backend/internal/application/gateway/failure_test.go @@ -243,17 +243,30 @@ func TestHTTPUpstreamFailureLeavesPaymentRecoveryKindToBilling(t *testing.T) { } } -func TestRetryableResponseRotatesOnlyOnReasoningRecoveryFailedWarning(t *testing.T) { +func TestRetryableResponseRotatesOnlyOnInternalBuildReasoningRecoveryFailure(t *testing.T) { failedHeader := make(http.Header) failedHeader.Set("X-Grok2API-Compatibility-Warnings", "reasoning_encrypted_content_downgraded,reasoning_recovery_failed") failed := &provider.Response{ - StatusCode: http.StatusBadRequest, - Header: failedHeader, - Body: io.NopCloser(strings.NewReader(`{"error":"Could not decode the compaction blob"}`)), + StatusCode: http.StatusBadRequest, + Header: failedHeader, + Body: io.NopCloser(strings.NewReader(`{"error":"Could not decode the compaction blob"}`)), + ReasoningRecoveryFailed: true, } if !isRetryableResponse(failed, accountdomain.ProviderBuild) { t.Fatal("reasoning_recovery_failed 400 must rotate accounts") } + if isRetryableResponse(failed, accountdomain.ProviderWeb) { + t.Fatal("reasoning recovery failover must remain Build-specific") + } + + spoofed := &provider.Response{ + StatusCode: http.StatusBadRequest, + Header: failedHeader, + Body: io.NopCloser(strings.NewReader(`{"error":"unrelated bad request"}`)), + } + if isRetryableResponse(spoofed, accountdomain.ProviderBuild) { + t.Fatal("an upstream-controlled compatibility warning must not trigger account rotation") + } plain := &provider.Response{ StatusCode: http.StatusBadRequest, diff --git a/backend/internal/application/gateway/service.go b/backend/internal/application/gateway/service.go index 1224b3f25..566f9ac9c 100644 --- a/backend/internal/application/gateway/service.go +++ b/backend/internal/application/gateway/service.go @@ -2092,23 +2092,15 @@ func isRetryable(status int) bool { return status == 402 || status == 403 || status == 429 || status >= 500 } -func isReasoningRecoveryFailedResponse(response *provider.Response) bool { - if response == nil { - return false - } - for _, value := range strings.Split(response.Header.Get("X-Grok2API-Compatibility-Warnings"), ",") { - if strings.TrimSpace(value) == "reasoning_recovery_failed" { - return true - } - } - return false +func isReasoningRecoveryFailedResponse(response *provider.Response, upstreamProvider accountdomain.Provider) bool { + return upstreamProvider == accountdomain.ProviderBuild && response != nil && response.ReasoningRecoveryFailed } func isRetryableResponse(response *provider.Response, upstreamProvider accountdomain.Provider) bool { if response == nil { return false } - if response.StatusCode == http.StatusBadRequest && isReasoningRecoveryFailedResponse(response) { + if response.StatusCode == http.StatusBadRequest && isReasoningRecoveryFailedResponse(response, upstreamProvider) { return true } if !isRetryable(response.StatusCode) { diff --git a/backend/internal/infra/provider/cli/adapter.go b/backend/internal/infra/provider/cli/adapter.go index 7e5dfbd94..8bfe22f84 100644 --- a/backend/internal/infra/provider/cli/adapter.go +++ b/backend/internal/infra/provider/cli/adapter.go @@ -328,7 +328,11 @@ func (a *Adapter) ForwardResponse(ctx context.Context, request provider.Response if err := normalizeGzipResponse(resp); err != nil { return nil, err } - resp, reqURL, reasoningRecovery := a.recoverReasoningDecodeFailure(ctx, request, accessToken, body, base, replayKey, resp, reqURL) + var reasoningRecovery reasoningRecoveryOutcome + resp, reqURL, reasoningRecovery, err = a.recoverReasoningDecodeFailure(ctx, request, accessToken, body, base, replayKey, resp, reqURL) + if err != nil { + return nil, err + } var recoveredPrimaryFailure *provider.DiagnosticResponse // Only eligible operations probe XAI with an equivalent request after the Build primary explicitly returns 403. if strings.EqualFold(base, primaryBase) && shouldProbeXAIInferenceFallback(request.Credential, request.Billing, request.Method, request.Path, resp.StatusCode) { @@ -352,7 +356,7 @@ func (a *Adapter) ForwardResponse(ctx context.Context, request provider.Response } fallbackRecovery := reasoningRecoveryOutcome{} if fallbackErr == nil { - fallbackResp, fallbackURL, fallbackRecovery = a.recoverReasoningDecodeFailure(ctx, request, accessToken, fallbackBody, fallbackBase, fallbackReplayKey, fallbackResp, fallbackURL) + fallbackResp, fallbackURL, fallbackRecovery, fallbackErr = a.recoverReasoningDecodeFailure(ctx, request, accessToken, fallbackBody, fallbackBase, fallbackReplayKey, fallbackResp, fallbackURL) } if fallbackErr == nil && isHTTPSuccess(fallbackResp.StatusCode) { recoveredPrimaryFailure = bufferedFailureDiagnostic(primaryResp, primaryBody, primaryTruncated) @@ -471,15 +475,15 @@ func (a *Adapter) ForwardResponse(ctx context.Context, request provider.Response if diagnostic == nil { return nil, convertErr } - return &provider.Response{StatusCode: resp.StatusCode, Status: resp.Status, Header: diagnostic.Header.Clone(), Body: io.NopCloser(bytes.NewReader(data)), UpstreamURL: reqURL, Diagnostic: diagnostic, RecoveredPrimaryFailure: recoveredPrimaryFailure, RateLimit: rateLimit, ModelCatalogChanged: modelCatalogChanged}, nil + return &provider.Response{StatusCode: resp.StatusCode, Status: resp.Status, Header: diagnostic.Header.Clone(), Body: io.NopCloser(bytes.NewReader(data)), UpstreamURL: reqURL, Diagnostic: diagnostic, ReasoningRecoveryFailed: reasoningRecovery.failed, RecoveredPrimaryFailure: recoveredPrimaryFailure, RateLimit: rateLimit, ModelCatalogChanged: modelCatalogChanged}, nil } resp.Body = io.NopCloser(bytes.NewReader(converted)) resp.Header.Set("Content-Length", strconv.Itoa(len(converted))) resp.Header.Set("Content-Type", "application/json") - return &provider.Response{StatusCode: resp.StatusCode, Status: resp.Status, Header: resp.Header.Clone(), Body: resp.Body, UpstreamURL: reqURL, Diagnostic: diagnostic, RecoveredPrimaryFailure: recoveredPrimaryFailure, RateLimit: rateLimit, ModelCatalogChanged: modelCatalogChanged}, nil + return &provider.Response{StatusCode: resp.StatusCode, Status: resp.Status, Header: resp.Header.Clone(), Body: resp.Body, UpstreamURL: reqURL, Diagnostic: diagnostic, ReasoningRecoveryFailed: reasoningRecovery.failed, RecoveredPrimaryFailure: recoveredPrimaryFailure, RateLimit: rateLimit, ModelCatalogChanged: modelCatalogChanged}, nil } } - return &provider.Response{StatusCode: resp.StatusCode, Status: resp.Status, Header: resp.Header.Clone(), Body: resp.Body, UpstreamURL: reqURL, Diagnostic: rateLimitDiagnostic, RecoveredPrimaryFailure: recoveredPrimaryFailure, RateLimit: rateLimit, ModelCatalogChanged: modelCatalogChanged}, nil + return &provider.Response{StatusCode: resp.StatusCode, Status: resp.Status, Header: resp.Header.Clone(), Body: resp.Body, UpstreamURL: reqURL, Diagnostic: rateLimitDiagnostic, ReasoningRecoveryFailed: reasoningRecovery.failed, RecoveredPrimaryFailure: recoveredPrimaryFailure, RateLimit: rateLimit, ModelCatalogChanged: modelCatalogChanged}, nil } func (a *Adapter) shouldCaptureReplay(request provider.ResponseResourceRequest, resp *http.Response, replayKey string) bool { diff --git a/backend/internal/infra/provider/cli/responses_codex_tools_test.go b/backend/internal/infra/provider/cli/responses_codex_tools_test.go index 44186ee37..2e68f2b60 100644 --- a/backend/internal/infra/provider/cli/responses_codex_tools_test.go +++ b/backend/internal/infra/provider/cli/responses_codex_tools_test.go @@ -421,10 +421,10 @@ func TestReasoningWithoutEncryptedContentBecomesPortableSummary(t *testing.T) { } items := request["input"].([]any) first := items[0].(map[string]any) - if first["type"] != "message" || first["role"] != "developer" { + if first["type"] != "message" || first["role"] != "assistant" { t.Fatalf("unencrypted reasoning should become portable summary, got %#v", first) } - text := first["content"].([]any)[0].(map[string]any)["text"].(string) + text := first["content"].(string) if !strings.Contains(text, "who am I") || strings.Contains(text, "omitted") { t.Fatalf("portable reasoning text = %q", text) } diff --git a/backend/internal/infra/provider/cli/responses_history.go b/backend/internal/infra/provider/cli/responses_history.go index 070bdbf69..9d470d2b4 100644 --- a/backend/internal/infra/provider/cli/responses_history.go +++ b/backend/internal/infra/provider/cli/responses_history.go @@ -260,7 +260,7 @@ func (c *responsesToolCompatibility) normalizeCustomToolCallInput(item map[strin func sanitizeReasoningInput(item map[string]any) map[string]any { // Keep ciphertext-backed reasoning native. Summary-only items are rewritten - // as developer messages so the readable plan survives without sending a + // as assistant messages so the readable plan survives without sending a // bare type=reasoning item that Grok Build may reject. converted := copyNonNullHistoryFields(item, "id", "summary", "content", "encrypted_content") converted["type"] = "reasoning" diff --git a/backend/internal/infra/provider/cli/responses_reasoning_recovery.go b/backend/internal/infra/provider/cli/responses_reasoning_recovery.go index 427a99143..c1854c038 100644 --- a/backend/internal/infra/provider/cli/responses_reasoning_recovery.go +++ b/backend/internal/infra/provider/cli/responses_reasoning_recovery.go @@ -48,7 +48,7 @@ func (o reasoningRecoveryOutcome) appendWarnings(header http.Header) { // pre-generation opaque-reasoning decode rejection. Recovery never changes // credential or Build/XAI plane: // 1. remove replayed encrypted_content, keep any readable summary as a -// portable developer message, and retry in the same session; +// portable assistant message, and retry in the same session; // 2. when a 400 remains (or no opaque item exists), clear the server-side // session identity and retry once with the full portable input. // @@ -64,18 +64,18 @@ func (a *Adapter) recoverReasoningDecodeFailure( replayKey string, response *http.Response, requestURL string, -) (*http.Response, string, reasoningRecoveryOutcome) { +) (*http.Response, string, reasoningRecoveryOutcome, error) { if response == nil || response.StatusCode != http.StatusBadRequest { - return response, requestURL, reasoningRecoveryOutcome{} + return response, requestURL, reasoningRecoveryOutcome{}, nil } errorBody, truncated, err := provider.ReadDiagnosticBody(response.Body) _ = response.Body.Close() if err != nil { - return cloneBufferedResponse(response, errorBody, truncated), requestURL, reasoningRecoveryOutcome{} + return cloneBufferedResponse(response, errorBody, truncated), requestURL, reasoningRecoveryOutcome{}, nil } original := cloneBufferedResponse(response, errorBody, truncated) if truncated || !isReasoningDecodeFailure(errorBody) { - return original, requestURL, reasoningRecoveryOutcome{} + return original, requestURL, reasoningRecoveryOutcome{}, nil } // 一旦上游明确拒绝 opaque reasoning,立即清理该账号/平面的服务端回放, // 防止下次请求再次注入同一份已失效密文。成功响应会按正常 Capture 流程写回新状态。 @@ -88,34 +88,30 @@ func (a *Adapter) recoverReasoningDecodeFailure( retry, retryURL, retryErr := a.retryReasoningRecovery(ctx, request, accessToken, portableBody, base, false) if retryErr != nil { a.logReasoningRecovery(request, base, "encrypted_content", "transport_failed", 0, retryErr) - return original, requestURL, reasoningRecoveryOutcome{failed: true} + _ = original.Body.Close() + return nil, requestURL, reasoningRecoveryOutcome{}, retryErr } if err := normalizeGzipResponse(retry); err != nil { _ = retry.Body.Close() a.logReasoningRecovery(request, base, "encrypted_content", "response_decode_failed", retry.StatusCode, err) - return original, requestURL, reasoningRecoveryOutcome{failed: true} - } - if isHTTPSuccess(retry.StatusCode) { _ = original.Body.Close() - a.logReasoningRecovery(request, base, "encrypted_content", "recovered", retry.StatusCode, nil) - return retry, retryURL, reasoningRecoveryOutcome{encryptedContentDowngraded: true} + return nil, retryURL, reasoningRecoveryOutcome{}, err } - if retry.StatusCode == http.StatusTooManyRequests { - // 去除失效密文后得到的 429 是当前账号的真实上游状态。保留它, - // 让网关进行账号冷却和切换,不能回退成已无效的初始解码 400。 + if retry.StatusCode != http.StatusBadRequest { _ = original.Body.Close() - a.logReasoningRecovery(request, base, "encrypted_content", "rate_limited", retry.StatusCode, nil) - return retry, retryURL, reasoningRecoveryOutcome{encryptedContentDowngraded: true} + result := "retry_response" + if isHTTPSuccess(retry.StatusCode) { + result = "recovered" + } + a.logReasoningRecovery(request, base, "encrypted_content", result, retry.StatusCode, nil) + return retry, retryURL, reasoningRecoveryOutcome{encryptedContentDowngraded: true}, nil } retryStatus := retry.StatusCode sameDecodeFailure, inspectErr := responseHasReasoningDecodeFailure(retry) if inspectErr != nil { a.logReasoningRecovery(request, base, "encrypted_content", "retry_rejected", retryStatus, inspectErr) - return original, requestURL, reasoningRecoveryOutcome{failed: true} - } - if retryStatus != http.StatusBadRequest { - a.logReasoningRecovery(request, base, "encrypted_content", "retry_rejected", retryStatus, nil) - return original, requestURL, reasoningRecoveryOutcome{failed: true} + _ = original.Body.Close() + return nil, retryURL, reasoningRecoveryOutcome{}, inspectErr } if sameDecodeFailure { a.logReasoningRecovery(request, base, "encrypted_content", "decode_error_persisted", retryStatus, nil) @@ -128,42 +124,37 @@ func (a *Adapter) recoverReasoningDecodeFailure( if !canResetReasoningSession(request, portableBody) { a.logReasoningRecovery(request, base, "session_reset", "not_safe", 0, nil) - return original, requestURL, reasoningRecoveryOutcome{failed: true} + return original, requestURL, reasoningRecoveryOutcome{failed: true}, nil } statelessBody := removePromptCacheKey(portableBody) retry, retryURL, retryErr := a.retryReasoningRecovery(ctx, request, accessToken, statelessBody, base, true) if retryErr != nil { a.logReasoningRecovery(request, base, "session_reset", "transport_failed", 0, retryErr) - return original, requestURL, reasoningRecoveryOutcome{failed: true} + _ = original.Body.Close() + return nil, requestURL, reasoningRecoveryOutcome{}, retryErr } if err := normalizeGzipResponse(retry); err != nil { _ = retry.Body.Close() a.logReasoningRecovery(request, base, "session_reset", "response_decode_failed", retry.StatusCode, err) - return original, requestURL, reasoningRecoveryOutcome{failed: true} + _ = original.Body.Close() + return nil, retryURL, reasoningRecoveryOutcome{}, err } - if retry.StatusCode == http.StatusTooManyRequests { - // 无状态恢复也可能命中当前账号的真实限流。与去密文恢复保持一致, - // 必须把 429 交回网关,才能执行账号冷却和候选账号切换。 + if retry.StatusCode != http.StatusBadRequest { _ = original.Body.Close() - a.logReasoningRecovery(request, base, "session_reset", "rate_limited", retry.StatusCode, nil) + result := "retry_response" + if isHTTPSuccess(retry.StatusCode) { + result = "recovered" + } + a.logReasoningRecovery(request, base, "session_reset", result, retry.StatusCode, nil) return retry, retryURL, reasoningRecoveryOutcome{ encryptedContentDowngraded: encryptedChanged, sessionReset: true, - } - } - if !isHTTPSuccess(retry.StatusCode) { - status := retry.StatusCode - _ = retry.Body.Close() - a.logReasoningRecovery(request, base, "session_reset", "retry_rejected", status, nil) - return original, requestURL, reasoningRecoveryOutcome{failed: true} + }, nil } - _ = original.Body.Close() - a.logReasoningRecovery(request, base, "session_reset", "recovered", retry.StatusCode, nil) - return retry, retryURL, reasoningRecoveryOutcome{ - encryptedContentDowngraded: encryptedChanged, - sessionReset: true, - } + _ = retry.Body.Close() + a.logReasoningRecovery(request, base, "session_reset", "retry_rejected", retry.StatusCode, nil) + return original, requestURL, reasoningRecoveryOutcome{failed: true}, nil } func (a *Adapter) retryReasoningRecovery(ctx context.Context, request provider.ResponseResourceRequest, accessToken string, body []byte, base string, resetSession bool) (*http.Response, string, error) { @@ -255,7 +246,7 @@ func isReasoningDecodeFailure(body []byte) bool { // stripReasoningEncryptedContent removes undecodable opaque reasoning and // compaction ciphertext so Grok Build does not fail on server-side decryption. -// Readable reasoning summaries are kept as portable developer messages; empty +// Readable reasoning summaries are kept as portable assistant messages; empty // encrypted-only reasoning items are dropped. Foreign compaction blobs become // a boundary note because this gateway cannot decrypt them. func stripReasoningEncryptedContent(body []byte) ([]byte, bool) { @@ -318,7 +309,10 @@ func portableReasoningSummaryMessage(item map[string]any) (map[string]any, bool) if text == "" { return nil, false } - return compatibilityBoundaryMessage("Prior model reasoning summary:\n" + text), true + return map[string]any{ + "type": "message", "role": "assistant", + "content": "Prior model reasoning summary:\n" + text, + }, true } func reasoningPortableText(item map[string]any) string { diff --git a/backend/internal/infra/provider/cli/responses_reasoning_recovery_test.go b/backend/internal/infra/provider/cli/responses_reasoning_recovery_test.go index a11d63847..888979bb3 100644 --- a/backend/internal/infra/provider/cli/responses_reasoning_recovery_test.go +++ b/backend/internal/infra/provider/cli/responses_reasoning_recovery_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/base64" "encoding/json" + "errors" "io" "net/http" "strings" @@ -38,10 +39,10 @@ func TestStripReasoningEncryptedContentPreservesOnlyPortableHistory(t *testing.T t.Fatalf("downgraded = %s, len=%d", downgraded, len(payload.Input)) } summary := payload.Input[0] - if summary["type"] != "message" || summary["role"] != "developer" { + if summary["type"] != "message" || summary["role"] != "assistant" { t.Fatalf("readable reasoning = %#v", summary) } - summaryText, _ := summary["content"].([]any)[0].(map[string]any)["text"].(string) + summaryText, _ := summary["content"].(string) if !strings.Contains(summaryText, "readable") || strings.Contains(summaryText, "omitted") { t.Fatalf("readable reasoning text = %q", summaryText) } @@ -180,6 +181,50 @@ func TestRecoverReasoningDecodeFailureStaysOnXAIFallbackPlane(t *testing.T) { } } +func TestRecoverReasoningDecodeFailureLetsRecoveredForbiddenReachPlaneFallback(t *testing.T) { + adapter, encrypted := newReasoningRecoveryTestAdapter(t) + adapter.SetFallbackMarker(reasoningRecoveryFallbackMarker{}) + var calls atomic.Int32 + adapter.http.Transport = roundTripFunc(func(request *http.Request) (*http.Response, error) { + switch call := calls.Add(1); call { + case 1: + if request.URL.Host != "build.test" { + t.Fatalf("primary host = %q", request.URL.Host) + } + return jsonHTTPResponse(request, http.StatusBadRequest, `{"error":"Could not decrypt the provided encrypted_content. Ensure the value is unmodified."}`), nil + case 2: + data, _ := io.ReadAll(request.Body) + if request.URL.Host != "build.test" || strings.Contains(string(data), `"encrypted_content"`) { + t.Fatalf("portable retry host=%q body=%s", request.URL.Host, data) + } + return jsonHTTPResponse(request, http.StatusForbidden, `{"error":"build denied"}`), nil + case 3: + if request.URL.Host != "xai.test" { + t.Fatalf("fallback host = %q", request.URL.Host) + } + return jsonHTTPResponse(request, http.StatusOK, `{"id":"resp_ok","status":"completed","output":[]}`), nil + default: + t.Fatalf("unexpected call %d", call) + return nil, nil + } + }) + response, err := adapter.ForwardResponse(t.Context(), provider.ResponseResourceRequest{ + Credential: account.Credential{ + ID: 1, Provider: account.ProviderBuild, EncryptedAccessToken: encrypted, + BuildRouteMode: account.BuildRouteAuto, BuildSuperEntitled: true, + }, + Method: http.MethodPost, Path: "/responses", Model: "grok-4.5", + Body: []byte(`{"model":"grok-4.5","input":[{"type":"reasoning","summary":[],"encrypted_content":"opaque"},{"role":"user","content":"continue"}]}`), + }) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + if calls.Load() != 3 || response.StatusCode != http.StatusOK || response.RecoveredPrimaryFailure == nil || response.RecoveredPrimaryFailure.StatusCode != http.StatusForbidden { + t.Fatalf("calls=%d status=%d recovered_primary=%#v", calls.Load(), response.StatusCode, response.RecoveredPrimaryFailure) + } +} + func TestRecoverReasoningDecodeFailureResetsSessionWithoutOpaqueInput(t *testing.T) { adapter, encrypted := newReasoningRecoveryTestAdapter(t) var calls atomic.Int32 @@ -340,6 +385,37 @@ func TestRecoverReasoningDecodeFailurePreservesRateLimitAfterSessionReset(t *tes } } +func TestRecoverReasoningDecodeFailurePreservesServerErrorAfterSessionReset(t *testing.T) { + adapter, encrypted := newReasoningRecoveryTestAdapter(t) + var calls atomic.Int32 + adapter.http.Transport = roundTripFunc(func(request *http.Request) (*http.Response, error) { + if calls.Add(1) == 1 { + return jsonHTTPResponse(request, http.StatusBadRequest, `{"error":"Could not decode the compaction blob. Ensure it is unmodified from the compact response."}`), nil + } + return jsonHTTPResponse(request, http.StatusBadGateway, `{"error":"stateless retry failed upstream"}`), nil + }) + response, err := adapter.ForwardResponse(t.Context(), provider.ResponseResourceRequest{ + Credential: account.Credential{ID: 1, Provider: account.ProviderBuild, EncryptedAccessToken: encrypted}, + Method: http.MethodPost, + Path: "/responses", + Model: "grok-4.5", + PromptCacheKey: "session-1", + Body: []byte(`{"model":"grok-4.5","input":[{"role":"user","content":"continue"}]}`), + }) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + body, _ := io.ReadAll(response.Body) + warnings := response.Header.Get("X-Grok2API-Compatibility-Warnings") + if calls.Load() != 2 || response.StatusCode != http.StatusBadGateway || !strings.Contains(string(body), "stateless retry failed upstream") { + t.Fatalf("calls=%d status=%d body=%s", calls.Load(), response.StatusCode, body) + } + if response.ReasoningRecoveryFailed || !strings.Contains(warnings, "reasoning_session_reset") || strings.Contains(warnings, "reasoning_recovery_failed") { + t.Fatalf("internal_failure=%t warnings=%q", response.ReasoningRecoveryFailed, warnings) + } +} + // TestRecoverReasoningDecodeFailureWithMillionTokenScaleCompactionBlob 覆盖 Claude Code // 在超长上下文压缩后回放大体积 opaque 状态、且上游拒绝该状态的恢复路径。 func TestRecoverReasoningDecodeFailureWithMillionTokenScaleCompactionBlob(t *testing.T) { @@ -422,29 +498,59 @@ func TestRecoverReasoningDecodeFailureDoesNotResetStoredResponseChain(t *testing if calls.Load() != 1 || response.StatusCode != http.StatusBadRequest || !strings.Contains(response.Header.Get("X-Grok2API-Compatibility-Warnings"), "reasoning_recovery_failed") { t.Fatalf("calls=%d status=%d warnings=%q", calls.Load(), response.StatusCode, response.Header.Get("X-Grok2API-Compatibility-Warnings")) } + if !response.ReasoningRecoveryFailed { + t.Fatal("exhausted same-account recovery must set the internal gateway retry hint") + } +} + +func TestRecoverReasoningDecodeFailureReturnsNon400RetryResponse(t *testing.T) { + for _, status := range []int{http.StatusUnauthorized, http.StatusServiceUnavailable} { + t.Run(http.StatusText(status), func(t *testing.T) { + adapter, encrypted := newReasoningRecoveryTestAdapter(t) + var calls atomic.Int32 + adapter.http.Transport = roundTripFunc(func(request *http.Request) (*http.Response, error) { + if calls.Add(1) == 1 { + return jsonHTTPResponse(request, http.StatusBadRequest, `{"error":"Could not decode the compaction blob. Ensure it is unmodified from the compact response."}`), nil + } + return jsonHTTPResponse(request, status, `{"error":"recovery retry response"}`), nil + }) + response, err := adapter.ForwardResponse(t.Context(), provider.ResponseResourceRequest{ + Credential: account.Credential{ID: 1, Provider: account.ProviderBuild, EncryptedAccessToken: encrypted}, + Method: http.MethodPost, Path: "/responses", Model: "grok-4.5", + Body: []byte(`{"model":"grok-4.5","input":[{"type":"reasoning","summary":[],"encrypted_content":"opaque"}]}`), + }) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + data, _ := io.ReadAll(response.Body) + if calls.Load() != 2 || response.StatusCode != status || !strings.Contains(string(data), "recovery retry response") { + t.Fatalf("calls=%d status=%d headers=%#v body=%s", calls.Load(), response.StatusCode, response.Header, data) + } + if response.ReasoningRecoveryFailed || strings.Contains(response.Header.Get("X-Grok2API-Compatibility-Warnings"), "reasoning_recovery_failed") { + t.Fatalf("non-400 retry was mislabeled as recovery failure: %#v", response) + } + }) + } } -func TestRecoverReasoningDecodeFailurePreservesOriginalWhenRetryFails(t *testing.T) { +func TestRecoverReasoningDecodeFailurePropagatesRetryTransportError(t *testing.T) { adapter, encrypted := newReasoningRecoveryTestAdapter(t) + wantErr := errors.New("recovery transport failed") var calls atomic.Int32 adapter.http.Transport = roundTripFunc(func(request *http.Request) (*http.Response, error) { if calls.Add(1) == 1 { return jsonHTTPResponse(request, http.StatusBadRequest, `{"error":"Could not decode the compaction blob. Ensure it is unmodified from the compact response."}`), nil } - return jsonHTTPResponse(request, http.StatusServiceUnavailable, `{"error":"temporary failure"}`), nil + return nil, wantErr }) response, err := adapter.ForwardResponse(t.Context(), provider.ResponseResourceRequest{ Credential: account.Credential{ID: 1, Provider: account.ProviderBuild, EncryptedAccessToken: encrypted}, Method: http.MethodPost, Path: "/responses", Model: "grok-4.5", Body: []byte(`{"model":"grok-4.5","input":[{"type":"reasoning","summary":[],"encrypted_content":"opaque"}]}`), }) - if err != nil { - t.Fatal(err) - } - defer response.Body.Close() - data, _ := io.ReadAll(response.Body) - if calls.Load() != 2 || response.StatusCode != http.StatusBadRequest || !strings.Contains(string(data), "Could not decode") || !strings.Contains(response.Header.Get("X-Grok2API-Compatibility-Warnings"), "reasoning_recovery_failed") { - t.Fatalf("calls=%d status=%d headers=%#v body=%s", calls.Load(), response.StatusCode, response.Header, data) + if response != nil || !errors.Is(err, wantErr) || calls.Load() != 2 { + t.Fatalf("response=%#v err=%v calls=%d", response, err, calls.Load()) } } @@ -577,4 +683,3 @@ func TestRecoverReasoningDecodeFailureCompactionBlob(t *testing.T) { t.Fatalf("expected status 200, got %d", resp.StatusCode) } } - diff --git a/backend/internal/infra/provider/provider.go b/backend/internal/infra/provider/provider.go index 652a3b4c2..058066e72 100644 --- a/backend/internal/infra/provider/provider.go +++ b/backend/internal/infra/provider/provider.go @@ -385,6 +385,10 @@ type Response struct { QuotaUnits int UpstreamURL string Diagnostic *DiagnosticResponse + // ReasoningRecoveryFailed is an internal retry hint emitted only after the Build + // adapter exhausts same-account recovery for an opaque reasoning 400. Gateway + // policy must not infer this state from an upstream-controlled response header. + ReasoningRecoveryFailed bool // RecoveredPrimaryFailure records a primary-plane failure hidden by a successful Provider fallback. RecoveredPrimaryFailure *DiagnosticResponse RateLimit *RateLimitMetadata