Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 46 additions & 0 deletions backend/internal/application/gateway/failure_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -243,6 +243,52 @@ func TestHTTPUpstreamFailureLeavesPaymentRecoveryKindToBilling(t *testing.T) {
}
}

func TestRetryableResponseRotatesOnlyOnInternalBuildReasoningRecoveryFailure(t *testing.T) {
failedHeader := make(http.Header)
failedHeader.Set("X-Grok2API-Compatibility-Warnings", "reasoning_encrypted_content_downgraded,reasoning_recovery_failed")
failed := &provider.Response{
StatusCode: http.StatusBadRequest,
Header: failedHeader,
Body: io.NopCloser(strings.NewReader(`{"error":"Could not decode the compaction blob"}`)),
ReasoningRecoveryFailed: true,
}
if !isRetryableResponse(failed, accountdomain.ProviderBuild) {
t.Fatal("reasoning_recovery_failed 400 must rotate accounts")
}
if isRetryableResponse(failed, accountdomain.ProviderWeb) {
t.Fatal("reasoning recovery failover must remain Build-specific")
}

spoofed := &provider.Response{
StatusCode: http.StatusBadRequest,
Header: failedHeader,
Body: io.NopCloser(strings.NewReader(`{"error":"unrelated bad request"}`)),
}
if isRetryableResponse(spoofed, accountdomain.ProviderBuild) {
t.Fatal("an upstream-controlled compatibility warning must not trigger account rotation")
}

plain := &provider.Response{
StatusCode: http.StatusBadRequest,
Header: make(http.Header),
Body: io.NopCloser(strings.NewReader(`{"error":"Could not decode the compaction blob"}`)),
Diagnostic: &provider.DiagnosticResponse{Body: []byte(`{"error":"Could not decode the compaction blob. Ensure it is unmodified from the compact response."}`)},
}
if isRetryableResponse(plain, accountdomain.ProviderBuild) {
t.Fatal("plain compaction 400 must not rotate accounts without reasoning_recovery_failed")
}

unrelated := &provider.Response{
StatusCode: http.StatusBadRequest,
Header: make(http.Header),
Body: io.NopCloser(strings.NewReader(`{"error":"invalid request history"}`)),
Diagnostic: &provider.DiagnosticResponse{Body: []byte(`{"error":"could not decode request json"}`)},
}
if isRetryableResponse(unrelated, accountdomain.ProviderBuild) {
t.Fatal("unrelated 400 must not match encrypted_content/decode substrings")
}
}

func TestRetryableResponseHonorsUpstreamRetryVeto(t *testing.T) {
response := &provider.Response{
StatusCode: http.StatusInternalServerError,
Expand Down
12 changes: 11 additions & 1 deletion backend/internal/application/gateway/service.go
Original file line number Diff line number Diff line change
Expand Up @@ -2092,8 +2092,18 @@ func isRetryable(status int) bool {
return status == 402 || status == 403 || status == 429 || status >= 500
}

func isReasoningRecoveryFailedResponse(response *provider.Response, upstreamProvider accountdomain.Provider) bool {
return upstreamProvider == accountdomain.ProviderBuild && response != nil && response.ReasoningRecoveryFailed
}

func isRetryableResponse(response *provider.Response, upstreamProvider accountdomain.Provider) bool {
if response == nil || !isRetryable(response.StatusCode) {
if response == nil {
return false
}
if response.StatusCode == http.StatusBadRequest && isReasoningRecoveryFailedResponse(response, upstreamProvider) {
return true
}
if !isRetryable(response.StatusCode) {
return false
}
// Account-scoped payment failures must always rotate accounts.
Expand Down
14 changes: 9 additions & 5 deletions backend/internal/infra/provider/cli/adapter.go
Original file line number Diff line number Diff line change
Expand Up @@ -328,7 +328,11 @@ func (a *Adapter) ForwardResponse(ctx context.Context, request provider.Response
if err := normalizeGzipResponse(resp); err != nil {
return nil, err
}
resp, reqURL, reasoningRecovery := a.recoverReasoningDecodeFailure(ctx, request, accessToken, body, base, replayKey, resp, reqURL)
var reasoningRecovery reasoningRecoveryOutcome
resp, reqURL, reasoningRecovery, err = a.recoverReasoningDecodeFailure(ctx, request, accessToken, body, base, replayKey, resp, reqURL)
if err != nil {
return nil, err
}
var recoveredPrimaryFailure *provider.DiagnosticResponse
// Only eligible operations probe XAI with an equivalent request after the Build primary explicitly returns 403.
if strings.EqualFold(base, primaryBase) && shouldProbeXAIInferenceFallback(request.Credential, request.Billing, request.Method, request.Path, resp.StatusCode) {
Expand All @@ -352,7 +356,7 @@ func (a *Adapter) ForwardResponse(ctx context.Context, request provider.Response
}
fallbackRecovery := reasoningRecoveryOutcome{}
if fallbackErr == nil {
fallbackResp, fallbackURL, fallbackRecovery = a.recoverReasoningDecodeFailure(ctx, request, accessToken, fallbackBody, fallbackBase, fallbackReplayKey, fallbackResp, fallbackURL)
fallbackResp, fallbackURL, fallbackRecovery, fallbackErr = a.recoverReasoningDecodeFailure(ctx, request, accessToken, fallbackBody, fallbackBase, fallbackReplayKey, fallbackResp, fallbackURL)
}
if fallbackErr == nil && isHTTPSuccess(fallbackResp.StatusCode) {
recoveredPrimaryFailure = bufferedFailureDiagnostic(primaryResp, primaryBody, primaryTruncated)
Expand Down Expand Up @@ -471,15 +475,15 @@ func (a *Adapter) ForwardResponse(ctx context.Context, request provider.Response
if diagnostic == nil {
return nil, convertErr
}
return &provider.Response{StatusCode: resp.StatusCode, Status: resp.Status, Header: diagnostic.Header.Clone(), Body: io.NopCloser(bytes.NewReader(data)), UpstreamURL: reqURL, Diagnostic: diagnostic, RecoveredPrimaryFailure: recoveredPrimaryFailure, RateLimit: rateLimit, ModelCatalogChanged: modelCatalogChanged}, nil
return &provider.Response{StatusCode: resp.StatusCode, Status: resp.Status, Header: diagnostic.Header.Clone(), Body: io.NopCloser(bytes.NewReader(data)), UpstreamURL: reqURL, Diagnostic: diagnostic, ReasoningRecoveryFailed: reasoningRecovery.failed, RecoveredPrimaryFailure: recoveredPrimaryFailure, RateLimit: rateLimit, ModelCatalogChanged: modelCatalogChanged}, nil
}
resp.Body = io.NopCloser(bytes.NewReader(converted))
resp.Header.Set("Content-Length", strconv.Itoa(len(converted)))
resp.Header.Set("Content-Type", "application/json")
return &provider.Response{StatusCode: resp.StatusCode, Status: resp.Status, Header: resp.Header.Clone(), Body: resp.Body, UpstreamURL: reqURL, Diagnostic: diagnostic, RecoveredPrimaryFailure: recoveredPrimaryFailure, RateLimit: rateLimit, ModelCatalogChanged: modelCatalogChanged}, nil
return &provider.Response{StatusCode: resp.StatusCode, Status: resp.Status, Header: resp.Header.Clone(), Body: resp.Body, UpstreamURL: reqURL, Diagnostic: diagnostic, ReasoningRecoveryFailed: reasoningRecovery.failed, RecoveredPrimaryFailure: recoveredPrimaryFailure, RateLimit: rateLimit, ModelCatalogChanged: modelCatalogChanged}, nil
}
}
return &provider.Response{StatusCode: resp.StatusCode, Status: resp.Status, Header: resp.Header.Clone(), Body: resp.Body, UpstreamURL: reqURL, Diagnostic: rateLimitDiagnostic, RecoveredPrimaryFailure: recoveredPrimaryFailure, RateLimit: rateLimit, ModelCatalogChanged: modelCatalogChanged}, nil
return &provider.Response{StatusCode: resp.StatusCode, Status: resp.Status, Header: resp.Header.Clone(), Body: resp.Body, UpstreamURL: reqURL, Diagnostic: rateLimitDiagnostic, ReasoningRecoveryFailed: reasoningRecovery.failed, RecoveredPrimaryFailure: recoveredPrimaryFailure, RateLimit: rateLimit, ModelCatalogChanged: modelCatalogChanged}, nil
}

func (a *Adapter) shouldCaptureReplay(request provider.ResponseResourceRequest, resp *http.Response, replayKey string) bool {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -404,7 +404,7 @@ func TestNativeBuildHistoryItemsArePreservedAndSanitized(t *testing.T) {
}
}

func TestReasoningWithoutEncryptedContentRemainsNative(t *testing.T) {
func TestReasoningWithoutEncryptedContentBecomesPortableSummary(t *testing.T) {
normalized, _, err := normalizeResponsesRequest([]byte(`{
"model":"public","input":[
{"type":"reasoning","id":"rs_1","status":"completed","summary":[{"type":"summary_text","text":"who am I"}],"content":null,"encrypted_content":null,"internal_chat_message_metadata_passthrough":{"turn_id":"t1"}},
Expand All @@ -421,12 +421,12 @@ func TestReasoningWithoutEncryptedContentRemainsNative(t *testing.T) {
}
items := request["input"].([]any)
first := items[0].(map[string]any)
if first["type"] != "reasoning" || first["status"] != nil || first["encrypted_content"] != nil || first["content"] != nil {
t.Fatalf("reasoning history = %#v", first)
if first["type"] != "message" || first["role"] != "assistant" {
t.Fatalf("unencrypted reasoning should become portable summary, got %#v", first)
}
summary := first["summary"].([]any)[0].(map[string]any)
if summary["text"] != "who am I" {
t.Fatalf("reasoning summary = %#v", first["summary"])
text := first["content"].(string)
if !strings.Contains(text, "who am I") || strings.Contains(text, "omitted") {
t.Fatalf("portable reasoning text = %q", text)
}
if items[1].(map[string]any)["content"] != "hi" {
t.Fatalf("assistant history = %#v", items[1])
Expand Down
24 changes: 7 additions & 17 deletions backend/internal/infra/provider/cli/responses_history.go
Original file line number Diff line number Diff line change
Expand Up @@ -259,31 +259,21 @@ func (c *responsesToolCompatibility) normalizeCustomToolCallInput(item map[strin
}

func sanitizeReasoningInput(item map[string]any) map[string]any {
// 官方 Grok Build 回放 reasoning 时会删除 output-only status,但会保留
// id、summary、content 和可选 encrypted_content。密文不是回放的前置条件。
// Keep ciphertext-backed reasoning native. Summary-only items are rewritten
// as assistant messages so the readable plan survives without sending a
// bare type=reasoning item that Grok Build may reject.
converted := copyNonNullHistoryFields(item, "id", "summary", "content", "encrypted_content")
converted["type"] = "reasoning"
if encrypted, ok := converted["encrypted_content"].(string); ok && strings.TrimSpace(encrypted) != "" {
if _, exists := converted["summary"]; !exists {
converted["summary"] = []any{}
}
return converted
}
if !hasPortableReasoningContent(converted) {
return compatibilityBoundaryMessage("A prior model reasoning item was omitted because it has no portable content for Grok Build.")
if portable, ok := portableReasoningSummaryMessage(converted); ok {
return portable
}
return converted
}

func hasPortableReasoningContent(item map[string]any) bool {
if encrypted, ok := item["encrypted_content"].(string); ok && strings.TrimSpace(encrypted) != "" {
return true
}
for _, key := range []string{"summary", "content"} {
if values, ok := item[key].([]any); ok && len(values) > 0 {
return true
}
}
return false
return compatibilityBoundaryMessage("A prior model reasoning item was omitted because it has no portable content for Grok Build.")
}

// sanitizeNativeHistoryInput rebuilds history from native Grok Build InputItem fields
Expand Down
Loading
Loading