diff --git a/backend/internal/application/gateway/failure_test.go b/backend/internal/application/gateway/failure_test.go index 74b6e912d..b93aff5d1 100644 --- a/backend/internal/application/gateway/failure_test.go +++ b/backend/internal/application/gateway/failure_test.go @@ -243,6 +243,52 @@ func TestHTTPUpstreamFailureLeavesPaymentRecoveryKindToBilling(t *testing.T) { } } +func TestRetryableResponseRotatesOnlyOnInternalBuildReasoningRecoveryFailure(t *testing.T) { + failedHeader := make(http.Header) + failedHeader.Set("X-Grok2API-Compatibility-Warnings", "reasoning_encrypted_content_downgraded,reasoning_recovery_failed") + failed := &provider.Response{ + StatusCode: http.StatusBadRequest, + Header: failedHeader, + Body: io.NopCloser(strings.NewReader(`{"error":"Could not decode the compaction blob"}`)), + ReasoningRecoveryFailed: true, + } + if !isRetryableResponse(failed, accountdomain.ProviderBuild) { + t.Fatal("reasoning_recovery_failed 400 must rotate accounts") + } + if isRetryableResponse(failed, accountdomain.ProviderWeb) { + t.Fatal("reasoning recovery failover must remain Build-specific") + } + + spoofed := &provider.Response{ + StatusCode: http.StatusBadRequest, + Header: failedHeader, + Body: io.NopCloser(strings.NewReader(`{"error":"unrelated bad request"}`)), + } + if isRetryableResponse(spoofed, accountdomain.ProviderBuild) { + t.Fatal("an upstream-controlled compatibility warning must not trigger account rotation") + } + + plain := &provider.Response{ + StatusCode: http.StatusBadRequest, + Header: make(http.Header), + Body: io.NopCloser(strings.NewReader(`{"error":"Could not decode the compaction blob"}`)), + Diagnostic: &provider.DiagnosticResponse{Body: []byte(`{"error":"Could not decode the compaction blob. Ensure it is unmodified from the compact response."}`)}, + } + if isRetryableResponse(plain, accountdomain.ProviderBuild) { + t.Fatal("plain compaction 400 must not rotate accounts without reasoning_recovery_failed") + } + + unrelated := &provider.Response{ + StatusCode: http.StatusBadRequest, + Header: make(http.Header), + Body: io.NopCloser(strings.NewReader(`{"error":"invalid request history"}`)), + Diagnostic: &provider.DiagnosticResponse{Body: []byte(`{"error":"could not decode request json"}`)}, + } + if isRetryableResponse(unrelated, accountdomain.ProviderBuild) { + t.Fatal("unrelated 400 must not match encrypted_content/decode substrings") + } +} + func TestRetryableResponseHonorsUpstreamRetryVeto(t *testing.T) { response := &provider.Response{ StatusCode: http.StatusInternalServerError, diff --git a/backend/internal/application/gateway/service.go b/backend/internal/application/gateway/service.go index e6b99685a..566f9ac9c 100644 --- a/backend/internal/application/gateway/service.go +++ b/backend/internal/application/gateway/service.go @@ -2092,8 +2092,18 @@ func isRetryable(status int) bool { return status == 402 || status == 403 || status == 429 || status >= 500 } +func isReasoningRecoveryFailedResponse(response *provider.Response, upstreamProvider accountdomain.Provider) bool { + return upstreamProvider == accountdomain.ProviderBuild && response != nil && response.ReasoningRecoveryFailed +} + func isRetryableResponse(response *provider.Response, upstreamProvider accountdomain.Provider) bool { - if response == nil || !isRetryable(response.StatusCode) { + if response == nil { + return false + } + if response.StatusCode == http.StatusBadRequest && isReasoningRecoveryFailedResponse(response, upstreamProvider) { + return true + } + if !isRetryable(response.StatusCode) { return false } // Account-scoped payment failures must always rotate accounts. diff --git a/backend/internal/infra/provider/cli/adapter.go b/backend/internal/infra/provider/cli/adapter.go index 7e5dfbd94..8bfe22f84 100644 --- a/backend/internal/infra/provider/cli/adapter.go +++ b/backend/internal/infra/provider/cli/adapter.go @@ -328,7 +328,11 @@ func (a *Adapter) ForwardResponse(ctx context.Context, request provider.Response if err := normalizeGzipResponse(resp); err != nil { return nil, err } - resp, reqURL, reasoningRecovery := a.recoverReasoningDecodeFailure(ctx, request, accessToken, body, base, replayKey, resp, reqURL) + var reasoningRecovery reasoningRecoveryOutcome + resp, reqURL, reasoningRecovery, err = a.recoverReasoningDecodeFailure(ctx, request, accessToken, body, base, replayKey, resp, reqURL) + if err != nil { + return nil, err + } var recoveredPrimaryFailure *provider.DiagnosticResponse // Only eligible operations probe XAI with an equivalent request after the Build primary explicitly returns 403. if strings.EqualFold(base, primaryBase) && shouldProbeXAIInferenceFallback(request.Credential, request.Billing, request.Method, request.Path, resp.StatusCode) { @@ -352,7 +356,7 @@ func (a *Adapter) ForwardResponse(ctx context.Context, request provider.Response } fallbackRecovery := reasoningRecoveryOutcome{} if fallbackErr == nil { - fallbackResp, fallbackURL, fallbackRecovery = a.recoverReasoningDecodeFailure(ctx, request, accessToken, fallbackBody, fallbackBase, fallbackReplayKey, fallbackResp, fallbackURL) + fallbackResp, fallbackURL, fallbackRecovery, fallbackErr = a.recoverReasoningDecodeFailure(ctx, request, accessToken, fallbackBody, fallbackBase, fallbackReplayKey, fallbackResp, fallbackURL) } if fallbackErr == nil && isHTTPSuccess(fallbackResp.StatusCode) { recoveredPrimaryFailure = bufferedFailureDiagnostic(primaryResp, primaryBody, primaryTruncated) @@ -471,15 +475,15 @@ func (a *Adapter) ForwardResponse(ctx context.Context, request provider.Response if diagnostic == nil { return nil, convertErr } - return &provider.Response{StatusCode: resp.StatusCode, Status: resp.Status, Header: diagnostic.Header.Clone(), Body: io.NopCloser(bytes.NewReader(data)), UpstreamURL: reqURL, Diagnostic: diagnostic, RecoveredPrimaryFailure: recoveredPrimaryFailure, RateLimit: rateLimit, ModelCatalogChanged: modelCatalogChanged}, nil + return &provider.Response{StatusCode: resp.StatusCode, Status: resp.Status, Header: diagnostic.Header.Clone(), Body: io.NopCloser(bytes.NewReader(data)), UpstreamURL: reqURL, Diagnostic: diagnostic, ReasoningRecoveryFailed: reasoningRecovery.failed, RecoveredPrimaryFailure: recoveredPrimaryFailure, RateLimit: rateLimit, ModelCatalogChanged: modelCatalogChanged}, nil } resp.Body = io.NopCloser(bytes.NewReader(converted)) resp.Header.Set("Content-Length", strconv.Itoa(len(converted))) resp.Header.Set("Content-Type", "application/json") - return &provider.Response{StatusCode: resp.StatusCode, Status: resp.Status, Header: resp.Header.Clone(), Body: resp.Body, UpstreamURL: reqURL, Diagnostic: diagnostic, RecoveredPrimaryFailure: recoveredPrimaryFailure, RateLimit: rateLimit, ModelCatalogChanged: modelCatalogChanged}, nil + return &provider.Response{StatusCode: resp.StatusCode, Status: resp.Status, Header: resp.Header.Clone(), Body: resp.Body, UpstreamURL: reqURL, Diagnostic: diagnostic, ReasoningRecoveryFailed: reasoningRecovery.failed, RecoveredPrimaryFailure: recoveredPrimaryFailure, RateLimit: rateLimit, ModelCatalogChanged: modelCatalogChanged}, nil } } - return &provider.Response{StatusCode: resp.StatusCode, Status: resp.Status, Header: resp.Header.Clone(), Body: resp.Body, UpstreamURL: reqURL, Diagnostic: rateLimitDiagnostic, RecoveredPrimaryFailure: recoveredPrimaryFailure, RateLimit: rateLimit, ModelCatalogChanged: modelCatalogChanged}, nil + return &provider.Response{StatusCode: resp.StatusCode, Status: resp.Status, Header: resp.Header.Clone(), Body: resp.Body, UpstreamURL: reqURL, Diagnostic: rateLimitDiagnostic, ReasoningRecoveryFailed: reasoningRecovery.failed, RecoveredPrimaryFailure: recoveredPrimaryFailure, RateLimit: rateLimit, ModelCatalogChanged: modelCatalogChanged}, nil } func (a *Adapter) shouldCaptureReplay(request provider.ResponseResourceRequest, resp *http.Response, replayKey string) bool { diff --git a/backend/internal/infra/provider/cli/responses_codex_tools_test.go b/backend/internal/infra/provider/cli/responses_codex_tools_test.go index 235733020..2e68f2b60 100644 --- a/backend/internal/infra/provider/cli/responses_codex_tools_test.go +++ b/backend/internal/infra/provider/cli/responses_codex_tools_test.go @@ -404,7 +404,7 @@ func TestNativeBuildHistoryItemsArePreservedAndSanitized(t *testing.T) { } } -func TestReasoningWithoutEncryptedContentRemainsNative(t *testing.T) { +func TestReasoningWithoutEncryptedContentBecomesPortableSummary(t *testing.T) { normalized, _, err := normalizeResponsesRequest([]byte(`{ "model":"public","input":[ {"type":"reasoning","id":"rs_1","status":"completed","summary":[{"type":"summary_text","text":"who am I"}],"content":null,"encrypted_content":null,"internal_chat_message_metadata_passthrough":{"turn_id":"t1"}}, @@ -421,12 +421,12 @@ func TestReasoningWithoutEncryptedContentRemainsNative(t *testing.T) { } items := request["input"].([]any) first := items[0].(map[string]any) - if first["type"] != "reasoning" || first["status"] != nil || first["encrypted_content"] != nil || first["content"] != nil { - t.Fatalf("reasoning history = %#v", first) + if first["type"] != "message" || first["role"] != "assistant" { + t.Fatalf("unencrypted reasoning should become portable summary, got %#v", first) } - summary := first["summary"].([]any)[0].(map[string]any) - if summary["text"] != "who am I" { - t.Fatalf("reasoning summary = %#v", first["summary"]) + text := first["content"].(string) + if !strings.Contains(text, "who am I") || strings.Contains(text, "omitted") { + t.Fatalf("portable reasoning text = %q", text) } if items[1].(map[string]any)["content"] != "hi" { t.Fatalf("assistant history = %#v", items[1]) diff --git a/backend/internal/infra/provider/cli/responses_history.go b/backend/internal/infra/provider/cli/responses_history.go index c5675decb..9d470d2b4 100644 --- a/backend/internal/infra/provider/cli/responses_history.go +++ b/backend/internal/infra/provider/cli/responses_history.go @@ -259,31 +259,21 @@ func (c *responsesToolCompatibility) normalizeCustomToolCallInput(item map[strin } func sanitizeReasoningInput(item map[string]any) map[string]any { - // 官方 Grok Build 回放 reasoning 时会删除 output-only status,但会保留 - // id、summary、content 和可选 encrypted_content。密文不是回放的前置条件。 + // Keep ciphertext-backed reasoning native. Summary-only items are rewritten + // as assistant messages so the readable plan survives without sending a + // bare type=reasoning item that Grok Build may reject. converted := copyNonNullHistoryFields(item, "id", "summary", "content", "encrypted_content") converted["type"] = "reasoning" if encrypted, ok := converted["encrypted_content"].(string); ok && strings.TrimSpace(encrypted) != "" { if _, exists := converted["summary"]; !exists { converted["summary"] = []any{} } + return converted } - if !hasPortableReasoningContent(converted) { - return compatibilityBoundaryMessage("A prior model reasoning item was omitted because it has no portable content for Grok Build.") + if portable, ok := portableReasoningSummaryMessage(converted); ok { + return portable } - return converted -} - -func hasPortableReasoningContent(item map[string]any) bool { - if encrypted, ok := item["encrypted_content"].(string); ok && strings.TrimSpace(encrypted) != "" { - return true - } - for _, key := range []string{"summary", "content"} { - if values, ok := item[key].([]any); ok && len(values) > 0 { - return true - } - } - return false + return compatibilityBoundaryMessage("A prior model reasoning item was omitted because it has no portable content for Grok Build.") } // sanitizeNativeHistoryInput rebuilds history from native Grok Build InputItem fields diff --git a/backend/internal/infra/provider/cli/responses_reasoning_recovery.go b/backend/internal/infra/provider/cli/responses_reasoning_recovery.go index e0527dced..c1854c038 100644 --- a/backend/internal/infra/provider/cli/responses_reasoning_recovery.go +++ b/backend/internal/infra/provider/cli/responses_reasoning_recovery.go @@ -15,6 +15,7 @@ import ( var reasoningDecodeFailureMarkers = [][]byte{ []byte("could not decode the compaction blob"), []byte("could not decrypt the provided encrypted_content"), + []byte("invalid_encrypted_content"), } type reasoningRecoveryOutcome struct { @@ -46,12 +47,14 @@ func (o reasoningRecoveryOutcome) appendWarnings(header http.Header) { // recoverReasoningDecodeFailure handles only the upstream's explicit // pre-generation opaque-reasoning decode rejection. Recovery never changes // credential or Build/XAI plane: -// 1. remove replayed encrypted_content and retry in the same session; -// 2. when the same decode error remains (or no opaque item exists), clear the -// server-side session identity and retry once with the full portable input. +// 1. remove replayed encrypted_content, keep any readable summary as a +// portable assistant message, and retry in the same session; +// 2. when a 400 remains (or no opaque item exists), clear the server-side +// session identity and retry once with the full portable input. // -// If recovery is unsuccessful, the original 400 is returned so the Gateway -// does not rotate accounts or obscure the first failure. +// If recovery is unsuccessful, the original 400 is returned with +// reasoning_recovery_failed so the Gateway can rotate accounts. The Provider +// itself never changes credential. func (a *Adapter) recoverReasoningDecodeFailure( ctx context.Context, request provider.ResponseResourceRequest, @@ -61,18 +64,18 @@ func (a *Adapter) recoverReasoningDecodeFailure( replayKey string, response *http.Response, requestURL string, -) (*http.Response, string, reasoningRecoveryOutcome) { +) (*http.Response, string, reasoningRecoveryOutcome, error) { if response == nil || response.StatusCode != http.StatusBadRequest { - return response, requestURL, reasoningRecoveryOutcome{} + return response, requestURL, reasoningRecoveryOutcome{}, nil } errorBody, truncated, err := provider.ReadDiagnosticBody(response.Body) _ = response.Body.Close() if err != nil { - return cloneBufferedResponse(response, errorBody, truncated), requestURL, reasoningRecoveryOutcome{} + return cloneBufferedResponse(response, errorBody, truncated), requestURL, reasoningRecoveryOutcome{}, nil } original := cloneBufferedResponse(response, errorBody, truncated) if truncated || !isReasoningDecodeFailure(errorBody) { - return original, requestURL, reasoningRecoveryOutcome{} + return original, requestURL, reasoningRecoveryOutcome{}, nil } // 一旦上游明确拒绝 opaque reasoning,立即清理该账号/平面的服务端回放, // 防止下次请求再次注入同一份已失效密文。成功响应会按正常 Capture 流程写回新状态。 @@ -85,71 +88,73 @@ func (a *Adapter) recoverReasoningDecodeFailure( retry, retryURL, retryErr := a.retryReasoningRecovery(ctx, request, accessToken, portableBody, base, false) if retryErr != nil { a.logReasoningRecovery(request, base, "encrypted_content", "transport_failed", 0, retryErr) - return original, requestURL, reasoningRecoveryOutcome{failed: true} + _ = original.Body.Close() + return nil, requestURL, reasoningRecoveryOutcome{}, retryErr } if err := normalizeGzipResponse(retry); err != nil { _ = retry.Body.Close() a.logReasoningRecovery(request, base, "encrypted_content", "response_decode_failed", retry.StatusCode, err) - return original, requestURL, reasoningRecoveryOutcome{failed: true} - } - if isHTTPSuccess(retry.StatusCode) { _ = original.Body.Close() - a.logReasoningRecovery(request, base, "encrypted_content", "recovered", retry.StatusCode, nil) - return retry, retryURL, reasoningRecoveryOutcome{encryptedContentDowngraded: true} + return nil, retryURL, reasoningRecoveryOutcome{}, err } - if retry.StatusCode == http.StatusTooManyRequests { - // 去除失效密文后得到的 429 是当前账号的真实上游状态。保留它, - // 让网关进行账号冷却和切换,不能回退成已无效的初始解码 400。 + if retry.StatusCode != http.StatusBadRequest { _ = original.Body.Close() - a.logReasoningRecovery(request, base, "encrypted_content", "rate_limited", retry.StatusCode, nil) - return retry, retryURL, reasoningRecoveryOutcome{encryptedContentDowngraded: true} + result := "retry_response" + if isHTTPSuccess(retry.StatusCode) { + result = "recovered" + } + a.logReasoningRecovery(request, base, "encrypted_content", result, retry.StatusCode, nil) + return retry, retryURL, reasoningRecoveryOutcome{encryptedContentDowngraded: true}, nil } + retryStatus := retry.StatusCode sameDecodeFailure, inspectErr := responseHasReasoningDecodeFailure(retry) - if inspectErr != nil || !sameDecodeFailure { - a.logReasoningRecovery(request, base, "encrypted_content", "retry_rejected", retry.StatusCode, inspectErr) - return original, requestURL, reasoningRecoveryOutcome{failed: true} + if inspectErr != nil { + a.logReasoningRecovery(request, base, "encrypted_content", "retry_rejected", retryStatus, inspectErr) + _ = original.Body.Close() + return nil, retryURL, reasoningRecoveryOutcome{}, inspectErr + } + if sameDecodeFailure { + a.logReasoningRecovery(request, base, "encrypted_content", "decode_error_persisted", retryStatus, nil) + } else { + // Stripping ciphertext can reword the 400 (for example a bare + // reasoning item). Keep going to session reset instead of aborting. + a.logReasoningRecovery(request, base, "encrypted_content", "retry_still_400", retryStatus, nil) } - a.logReasoningRecovery(request, base, "encrypted_content", "decode_error_persisted", retry.StatusCode, nil) } if !canResetReasoningSession(request, portableBody) { a.logReasoningRecovery(request, base, "session_reset", "not_safe", 0, nil) - return original, requestURL, reasoningRecoveryOutcome{failed: true} + return original, requestURL, reasoningRecoveryOutcome{failed: true}, nil } statelessBody := removePromptCacheKey(portableBody) retry, retryURL, retryErr := a.retryReasoningRecovery(ctx, request, accessToken, statelessBody, base, true) if retryErr != nil { a.logReasoningRecovery(request, base, "session_reset", "transport_failed", 0, retryErr) - return original, requestURL, reasoningRecoveryOutcome{failed: true} + _ = original.Body.Close() + return nil, requestURL, reasoningRecoveryOutcome{}, retryErr } if err := normalizeGzipResponse(retry); err != nil { _ = retry.Body.Close() a.logReasoningRecovery(request, base, "session_reset", "response_decode_failed", retry.StatusCode, err) - return original, requestURL, reasoningRecoveryOutcome{failed: true} + _ = original.Body.Close() + return nil, retryURL, reasoningRecoveryOutcome{}, err } - if retry.StatusCode == http.StatusTooManyRequests { - // 无状态恢复也可能命中当前账号的真实限流。与去密文恢复保持一致, - // 必须把 429 交回网关,才能执行账号冷却和候选账号切换。 + if retry.StatusCode != http.StatusBadRequest { _ = original.Body.Close() - a.logReasoningRecovery(request, base, "session_reset", "rate_limited", retry.StatusCode, nil) + result := "retry_response" + if isHTTPSuccess(retry.StatusCode) { + result = "recovered" + } + a.logReasoningRecovery(request, base, "session_reset", result, retry.StatusCode, nil) return retry, retryURL, reasoningRecoveryOutcome{ encryptedContentDowngraded: encryptedChanged, sessionReset: true, - } - } - if !isHTTPSuccess(retry.StatusCode) { - status := retry.StatusCode - _ = retry.Body.Close() - a.logReasoningRecovery(request, base, "session_reset", "retry_rejected", status, nil) - return original, requestURL, reasoningRecoveryOutcome{failed: true} + }, nil } - _ = original.Body.Close() - a.logReasoningRecovery(request, base, "session_reset", "recovered", retry.StatusCode, nil) - return retry, retryURL, reasoningRecoveryOutcome{ - encryptedContentDowngraded: encryptedChanged, - sessionReset: true, - } + _ = retry.Body.Close() + a.logReasoningRecovery(request, base, "session_reset", "retry_rejected", retry.StatusCode, nil) + return original, requestURL, reasoningRecoveryOutcome{failed: true}, nil } func (a *Adapter) retryReasoningRecovery(ctx context.Context, request provider.ResponseResourceRequest, accessToken string, body []byte, base string, resetSession bool) (*http.Response, string, error) { @@ -239,9 +244,11 @@ func isReasoningDecodeFailure(body []byte) bool { return false } -// stripReasoningEncryptedContent removes opaque reasoning state while -// preserving any readable summary/content. An encrypted-only reasoning item -// becomes empty after stripping and is removed entirely. +// stripReasoningEncryptedContent removes undecodable opaque reasoning and +// compaction ciphertext so Grok Build does not fail on server-side decryption. +// Readable reasoning summaries are kept as portable assistant messages; empty +// encrypted-only reasoning items are dropped. Foreign compaction blobs become +// a boundary note because this gateway cannot decrypt them. func stripReasoningEncryptedContent(body []byte) ([]byte, bool) { var payload map[string]any if json.Unmarshal(body, &payload) != nil { @@ -255,22 +262,35 @@ func stripReasoningEncryptedContent(body []byte) ([]byte, bool) { rebuilt := make([]any, 0, len(input)) for _, raw := range input { item, ok := raw.(map[string]any) - if !ok || stringField(item, "type") != "reasoning" { + if !ok { rebuilt = append(rebuilt, raw) continue } - encrypted, ok := item["encrypted_content"].(string) - if !ok || strings.TrimSpace(encrypted) == "" { + switch stringField(item, "type") { + case "reasoning": + encrypted, hasEncrypted := item["encrypted_content"].(string) + if !hasEncrypted || strings.TrimSpace(encrypted) == "" { + if portable, ok := portableReasoningSummaryMessage(item); ok { + changed = true + rebuilt = append(rebuilt, portable) + continue + } + rebuilt = append(rebuilt, raw) + continue + } + changed = true + if portable, ok := portableReasoningSummaryMessage(item); ok { + rebuilt = append(rebuilt, portable) + } + case "compaction": + changed = true + if portable, ok := portableReasoningSummaryMessage(item); ok { + rebuilt = append(rebuilt, portable) + continue + } + rebuilt = append(rebuilt, compatibilityBoundaryMessage("A prior compacted context could not be decoded by upstream. Continue from the retained conversation messages.")) + default: rebuilt = append(rebuilt, raw) - continue - } - cleaned := cloneJSONObject(item) - delete(cleaned, "encrypted_content") - delete(cleaned, "id") - delete(cleaned, "status") - changed = true - if hasReadableReasoningContent(cleaned) { - rebuilt = append(rebuilt, cleaned) } } if !changed { @@ -284,17 +304,29 @@ func stripReasoningEncryptedContent(body []byte) ([]byte, bool) { return encoded, true } -func hasReadableReasoningContent(item map[string]any) bool { +func portableReasoningSummaryMessage(item map[string]any) (map[string]any, bool) { + text := reasoningPortableText(item) + if text == "" { + return nil, false + } + return map[string]any{ + "type": "message", "role": "assistant", + "content": "Prior model reasoning summary:\n" + text, + }, true +} + +func reasoningPortableText(item map[string]any) string { + var parts []string for _, field := range []string{"summary", "content"} { - parts, _ := item[field].([]any) - for _, raw := range parts { + values, _ := item[field].([]any) + for _, raw := range values { part, _ := raw.(map[string]any) - if strings.TrimSpace(stringField(part, "text")) != "" { - return true + if text := strings.TrimSpace(stringField(part, "text")); text != "" { + parts = append(parts, text) } } } - return false + return strings.Join(parts, "\n") } func appendCompatibilityWarning(header http.Header, warning string) { diff --git a/backend/internal/infra/provider/cli/responses_reasoning_recovery_test.go b/backend/internal/infra/provider/cli/responses_reasoning_recovery_test.go index d5580c973..888979bb3 100644 --- a/backend/internal/infra/provider/cli/responses_reasoning_recovery_test.go +++ b/backend/internal/infra/provider/cli/responses_reasoning_recovery_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/base64" "encoding/json" + "errors" "io" "net/http" "strings" @@ -22,6 +23,7 @@ func TestStripReasoningEncryptedContentPreservesOnlyPortableHistory(t *testing.T {"type":"reasoning","id":"rs_empty","status":"completed","summary":[],"encrypted_content":"opaque-empty"}, {"type":"reasoning","summary":[{"type":"summary_text","text":""}],"encrypted_content":"opaque-blank"}, {"type":"reasoning","id":"rs_summary","status":"completed","summary":[{"type":"summary_text","text":"readable"}],"encrypted_content":"opaque-summary"}, + {"type":"compaction","id":"cmp_1","encrypted_content":"opaque-compaction"}, {"type":"message","role":"assistant","content":"answer","encrypted_content":"message-value"}, {"type":"message","role":"user","content":"continue"} ] @@ -33,15 +35,26 @@ func TestStripReasoningEncryptedContentPreservesOnlyPortableHistory(t *testing.T var payload struct { Input []map[string]any `json:"input"` } - if json.Unmarshal(downgraded, &payload) != nil || len(payload.Input) != 3 { - t.Fatalf("downgraded = %s", downgraded) + if json.Unmarshal(downgraded, &payload) != nil || len(payload.Input) != 4 { + t.Fatalf("downgraded = %s, len=%d", downgraded, len(payload.Input)) } - reasoning := payload.Input[0] - if reasoning["type"] != "reasoning" || reasoning["id"] != nil || reasoning["status"] != nil || reasoning["encrypted_content"] != nil { - t.Fatalf("reasoning = %#v", reasoning) + summary := payload.Input[0] + if summary["type"] != "message" || summary["role"] != "assistant" { + t.Fatalf("readable reasoning = %#v", summary) } - if payload.Input[1]["encrypted_content"] != "message-value" { - t.Fatalf("non-reasoning encrypted content changed: %#v", payload.Input[1]) + summaryText, _ := summary["content"].(string) + if !strings.Contains(summaryText, "readable") || strings.Contains(summaryText, "omitted") { + t.Fatalf("readable reasoning text = %q", summaryText) + } + compaction := payload.Input[1] + if compaction["type"] != "message" || compaction["role"] != "developer" { + t.Fatalf("compaction boundary item = %#v", compaction) + } + if payload.Input[2]["encrypted_content"] != "message-value" { + t.Fatalf("non-reasoning encrypted content changed: %#v", payload.Input[2]) + } + if payload.Input[3]["role"] != "user" { + t.Fatalf("user message changed: %#v", payload.Input[3]) } } @@ -168,6 +181,50 @@ func TestRecoverReasoningDecodeFailureStaysOnXAIFallbackPlane(t *testing.T) { } } +func TestRecoverReasoningDecodeFailureLetsRecoveredForbiddenReachPlaneFallback(t *testing.T) { + adapter, encrypted := newReasoningRecoveryTestAdapter(t) + adapter.SetFallbackMarker(reasoningRecoveryFallbackMarker{}) + var calls atomic.Int32 + adapter.http.Transport = roundTripFunc(func(request *http.Request) (*http.Response, error) { + switch call := calls.Add(1); call { + case 1: + if request.URL.Host != "build.test" { + t.Fatalf("primary host = %q", request.URL.Host) + } + return jsonHTTPResponse(request, http.StatusBadRequest, `{"error":"Could not decrypt the provided encrypted_content. Ensure the value is unmodified."}`), nil + case 2: + data, _ := io.ReadAll(request.Body) + if request.URL.Host != "build.test" || strings.Contains(string(data), `"encrypted_content"`) { + t.Fatalf("portable retry host=%q body=%s", request.URL.Host, data) + } + return jsonHTTPResponse(request, http.StatusForbidden, `{"error":"build denied"}`), nil + case 3: + if request.URL.Host != "xai.test" { + t.Fatalf("fallback host = %q", request.URL.Host) + } + return jsonHTTPResponse(request, http.StatusOK, `{"id":"resp_ok","status":"completed","output":[]}`), nil + default: + t.Fatalf("unexpected call %d", call) + return nil, nil + } + }) + response, err := adapter.ForwardResponse(t.Context(), provider.ResponseResourceRequest{ + Credential: account.Credential{ + ID: 1, Provider: account.ProviderBuild, EncryptedAccessToken: encrypted, + BuildRouteMode: account.BuildRouteAuto, BuildSuperEntitled: true, + }, + Method: http.MethodPost, Path: "/responses", Model: "grok-4.5", + Body: []byte(`{"model":"grok-4.5","input":[{"type":"reasoning","summary":[],"encrypted_content":"opaque"},{"role":"user","content":"continue"}]}`), + }) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + if calls.Load() != 3 || response.StatusCode != http.StatusOK || response.RecoveredPrimaryFailure == nil || response.RecoveredPrimaryFailure.StatusCode != http.StatusForbidden { + t.Fatalf("calls=%d status=%d recovered_primary=%#v", calls.Load(), response.StatusCode, response.RecoveredPrimaryFailure) + } +} + func TestRecoverReasoningDecodeFailureResetsSessionWithoutOpaqueInput(t *testing.T) { adapter, encrypted := newReasoningRecoveryTestAdapter(t) var calls atomic.Int32 @@ -328,6 +385,37 @@ func TestRecoverReasoningDecodeFailurePreservesRateLimitAfterSessionReset(t *tes } } +func TestRecoverReasoningDecodeFailurePreservesServerErrorAfterSessionReset(t *testing.T) { + adapter, encrypted := newReasoningRecoveryTestAdapter(t) + var calls atomic.Int32 + adapter.http.Transport = roundTripFunc(func(request *http.Request) (*http.Response, error) { + if calls.Add(1) == 1 { + return jsonHTTPResponse(request, http.StatusBadRequest, `{"error":"Could not decode the compaction blob. Ensure it is unmodified from the compact response."}`), nil + } + return jsonHTTPResponse(request, http.StatusBadGateway, `{"error":"stateless retry failed upstream"}`), nil + }) + response, err := adapter.ForwardResponse(t.Context(), provider.ResponseResourceRequest{ + Credential: account.Credential{ID: 1, Provider: account.ProviderBuild, EncryptedAccessToken: encrypted}, + Method: http.MethodPost, + Path: "/responses", + Model: "grok-4.5", + PromptCacheKey: "session-1", + Body: []byte(`{"model":"grok-4.5","input":[{"role":"user","content":"continue"}]}`), + }) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + body, _ := io.ReadAll(response.Body) + warnings := response.Header.Get("X-Grok2API-Compatibility-Warnings") + if calls.Load() != 2 || response.StatusCode != http.StatusBadGateway || !strings.Contains(string(body), "stateless retry failed upstream") { + t.Fatalf("calls=%d status=%d body=%s", calls.Load(), response.StatusCode, body) + } + if response.ReasoningRecoveryFailed || !strings.Contains(warnings, "reasoning_session_reset") || strings.Contains(warnings, "reasoning_recovery_failed") { + t.Fatalf("internal_failure=%t warnings=%q", response.ReasoningRecoveryFailed, warnings) + } +} + // TestRecoverReasoningDecodeFailureWithMillionTokenScaleCompactionBlob 覆盖 Claude Code // 在超长上下文压缩后回放大体积 opaque 状态、且上游拒绝该状态的恢复路径。 func TestRecoverReasoningDecodeFailureWithMillionTokenScaleCompactionBlob(t *testing.T) { @@ -410,29 +498,59 @@ func TestRecoverReasoningDecodeFailureDoesNotResetStoredResponseChain(t *testing if calls.Load() != 1 || response.StatusCode != http.StatusBadRequest || !strings.Contains(response.Header.Get("X-Grok2API-Compatibility-Warnings"), "reasoning_recovery_failed") { t.Fatalf("calls=%d status=%d warnings=%q", calls.Load(), response.StatusCode, response.Header.Get("X-Grok2API-Compatibility-Warnings")) } + if !response.ReasoningRecoveryFailed { + t.Fatal("exhausted same-account recovery must set the internal gateway retry hint") + } } -func TestRecoverReasoningDecodeFailurePreservesOriginalWhenRetryFails(t *testing.T) { +func TestRecoverReasoningDecodeFailureReturnsNon400RetryResponse(t *testing.T) { + for _, status := range []int{http.StatusUnauthorized, http.StatusServiceUnavailable} { + t.Run(http.StatusText(status), func(t *testing.T) { + adapter, encrypted := newReasoningRecoveryTestAdapter(t) + var calls atomic.Int32 + adapter.http.Transport = roundTripFunc(func(request *http.Request) (*http.Response, error) { + if calls.Add(1) == 1 { + return jsonHTTPResponse(request, http.StatusBadRequest, `{"error":"Could not decode the compaction blob. Ensure it is unmodified from the compact response."}`), nil + } + return jsonHTTPResponse(request, status, `{"error":"recovery retry response"}`), nil + }) + response, err := adapter.ForwardResponse(t.Context(), provider.ResponseResourceRequest{ + Credential: account.Credential{ID: 1, Provider: account.ProviderBuild, EncryptedAccessToken: encrypted}, + Method: http.MethodPost, Path: "/responses", Model: "grok-4.5", + Body: []byte(`{"model":"grok-4.5","input":[{"type":"reasoning","summary":[],"encrypted_content":"opaque"}]}`), + }) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + data, _ := io.ReadAll(response.Body) + if calls.Load() != 2 || response.StatusCode != status || !strings.Contains(string(data), "recovery retry response") { + t.Fatalf("calls=%d status=%d headers=%#v body=%s", calls.Load(), response.StatusCode, response.Header, data) + } + if response.ReasoningRecoveryFailed || strings.Contains(response.Header.Get("X-Grok2API-Compatibility-Warnings"), "reasoning_recovery_failed") { + t.Fatalf("non-400 retry was mislabeled as recovery failure: %#v", response) + } + }) + } +} + +func TestRecoverReasoningDecodeFailurePropagatesRetryTransportError(t *testing.T) { adapter, encrypted := newReasoningRecoveryTestAdapter(t) + wantErr := errors.New("recovery transport failed") var calls atomic.Int32 adapter.http.Transport = roundTripFunc(func(request *http.Request) (*http.Response, error) { if calls.Add(1) == 1 { return jsonHTTPResponse(request, http.StatusBadRequest, `{"error":"Could not decode the compaction blob. Ensure it is unmodified from the compact response."}`), nil } - return jsonHTTPResponse(request, http.StatusServiceUnavailable, `{"error":"temporary failure"}`), nil + return nil, wantErr }) response, err := adapter.ForwardResponse(t.Context(), provider.ResponseResourceRequest{ Credential: account.Credential{ID: 1, Provider: account.ProviderBuild, EncryptedAccessToken: encrypted}, Method: http.MethodPost, Path: "/responses", Model: "grok-4.5", Body: []byte(`{"model":"grok-4.5","input":[{"type":"reasoning","summary":[],"encrypted_content":"opaque"}]}`), }) - if err != nil { - t.Fatal(err) - } - defer response.Body.Close() - data, _ := io.ReadAll(response.Body) - if calls.Load() != 2 || response.StatusCode != http.StatusBadRequest || !strings.Contains(string(data), "Could not decode") || !strings.Contains(response.Header.Get("X-Grok2API-Compatibility-Warnings"), "reasoning_recovery_failed") { - t.Fatalf("calls=%d status=%d headers=%#v body=%s", calls.Load(), response.StatusCode, response.Header, data) + if response != nil || !errors.Is(err, wantErr) || calls.Load() != 2 { + t.Fatalf("response=%#v err=%v calls=%d", response, err, calls.Load()) } } @@ -466,3 +584,102 @@ type reasoningRecoveryFallbackMarker struct{} func (reasoningRecoveryFallbackMarker) MarkBuildAPIFallback(context.Context, uint64, bool) error { return nil } + +func TestRecoverReasoningDecodeFailureKeepsReadableSummary(t *testing.T) { + adapter, encrypted := newReasoningRecoveryTestAdapter(t) + var calls atomic.Int32 + adapter.http.Transport = roundTripFunc(func(request *http.Request) (*http.Response, error) { + call := calls.Add(1) + data, _ := io.ReadAll(request.Body) + if call == 1 { + if !strings.Contains(string(data), `"encrypted_content":"opaque"`) { + t.Fatalf("first body = %s", data) + } + return jsonHTTPResponse(request, http.StatusBadRequest, `{"error":"Could not decrypt the provided encrypted_content. Ensure the value is unmodified."}`), nil + } + if strings.Contains(string(data), `"encrypted_content"`) || strings.Contains(string(data), `"type":"reasoning"`) { + t.Fatalf("retry still has opaque reasoning: %s", data) + } + if !strings.Contains(string(data), "do not touch Y") { + t.Fatalf("retry dropped readable summary: %s", data) + } + return jsonHTTPResponse(request, http.StatusOK, `{"id":"resp_ok","status":"completed","output":[]}`), nil + }) + response, err := adapter.ForwardResponse(t.Context(), provider.ResponseResourceRequest{ + Credential: account.Credential{ID: 1, Provider: account.ProviderBuild, EncryptedAccessToken: encrypted}, + Method: http.MethodPost, Path: "/responses", Model: "grok-4.5", PromptCacheKey: "session-1", + Body: []byte(`{"model":"grok-4.5","input":[{"type":"reasoning","summary":[{"type":"summary_text","text":"do not touch Y"}],"encrypted_content":"opaque"},{"role":"user","content":"continue"}]}`), + }) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + if calls.Load() != 2 || response.StatusCode != http.StatusOK || !strings.Contains(response.Header.Get("X-Grok2API-Compatibility-Warnings"), "reasoning_encrypted_content_downgraded") { + t.Fatalf("calls=%d status=%d warnings=%q", calls.Load(), response.StatusCode, response.Header.Get("X-Grok2API-Compatibility-Warnings")) + } +} + +func TestRecoverReasoningDecodeFailureContinuesToSessionResetWhenStripRewords400(t *testing.T) { + adapter, encrypted := newReasoningRecoveryTestAdapter(t) + var calls atomic.Int32 + adapter.http.Transport = roundTripFunc(func(request *http.Request) (*http.Response, error) { + call := calls.Add(1) + data, _ := io.ReadAll(request.Body) + switch call { + case 1: + return jsonHTTPResponse(request, http.StatusBadRequest, `{"error":"Could not decrypt the provided encrypted_content. Ensure the value is unmodified."}`), nil + case 2: + if strings.Contains(string(data), `"encrypted_content"`) || request.Header.Get("x-grok-session-id") == "" { + t.Fatalf("opaque downgrade body=%s headers=%#v", data, request.Header) + } + return jsonHTTPResponse(request, http.StatusBadRequest, `{"error":"invalid request history"}`), nil + case 3: + if strings.Contains(string(data), `"encrypted_content"`) || strings.Contains(string(data), `"prompt_cache_key"`) || request.Header.Get("x-grok-session-id") != "" { + t.Fatalf("session reset body=%s headers=%#v", data, request.Header) + } + return jsonHTTPResponse(request, http.StatusOK, `{"id":"resp_ok","status":"completed","output":[]}`), nil + default: + t.Fatalf("unexpected call %d", call) + return nil, nil + } + }) + response, err := adapter.ForwardResponse(t.Context(), provider.ResponseResourceRequest{ + Credential: account.Credential{ID: 1, Provider: account.ProviderBuild, EncryptedAccessToken: encrypted}, + Method: http.MethodPost, Path: "/responses", Model: "grok-4.5", PromptCacheKey: "session-1", + Body: []byte(`{"model":"grok-4.5","input":[{"type":"reasoning","summary":[],"encrypted_content":"opaque"},{"role":"user","content":"continue"}]}`), + }) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + warnings := response.Header.Get("X-Grok2API-Compatibility-Warnings") + if calls.Load() != 3 || response.StatusCode != http.StatusOK || !strings.Contains(warnings, "reasoning_encrypted_content_downgraded") || !strings.Contains(warnings, "reasoning_session_reset") { + t.Fatalf("calls=%d status=%d warnings=%q", calls.Load(), response.StatusCode, warnings) + } +} + +func TestRecoverReasoningDecodeFailureCompactionBlob(t *testing.T) { + adapter, encrypted := newReasoningRecoveryTestAdapter(t) + var calls atomic.Int32 + adapter.http.Transport = roundTripFunc(func(request *http.Request) (*http.Response, error) { + call := calls.Add(1) + if call == 1 { + return jsonHTTPResponse(request, http.StatusBadRequest, `{"code":"invalid-argument","error":"Could not decode the compaction blob. Ensure it is unmodified from the compact response."}`), nil + } + return jsonHTTPResponse(request, http.StatusOK, `{"id":"resp_ok","status":"completed","output":[]}`), nil + }) + + body := []byte(`{"model":"grok-4.6","messages":[{"role":"user","content":"hello"}]}`) + resp, err := adapter.ForwardResponse(context.Background(), provider.ResponseResourceRequest{ + Credential: account.Credential{ID: 131, Provider: account.ProviderBuild, EncryptedAccessToken: encrypted}, + Method: http.MethodPost, Path: "/responses", Model: "grok-4.6", PromptCacheKey: "session-compaction-test", + NormalizeBody: true, Operation: conversation.OperationChat, Body: body, + }) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("expected status 200, got %d", resp.StatusCode) + } +} diff --git a/backend/internal/infra/provider/provider.go b/backend/internal/infra/provider/provider.go index 652a3b4c2..058066e72 100644 --- a/backend/internal/infra/provider/provider.go +++ b/backend/internal/infra/provider/provider.go @@ -385,6 +385,10 @@ type Response struct { QuotaUnits int UpstreamURL string Diagnostic *DiagnosticResponse + // ReasoningRecoveryFailed is an internal retry hint emitted only after the Build + // adapter exhausts same-account recovery for an opaque reasoning 400. Gateway + // policy must not infer this state from an upstream-controlled response header. + ReasoningRecoveryFailed bool // RecoveredPrimaryFailure records a primary-plane failure hidden by a successful Provider fallback. RecoveredPrimaryFailure *DiagnosticResponse RateLimit *RateLimitMetadata