From d3085929beafdd696c288343b3f070341ff36abb Mon Sep 17 00:00:00 2001 From: 6Kmfi6HP <179412085+6Kmfi6HP@users.noreply.github.com> Date: Sun, 27 Sep 2026 22:58:58 +0200 Subject: [PATCH 1/9] =?UTF-8?q?feat(stream):=20claude=E2=86=92responses=20?= =?UTF-8?q?=E7=A9=BA=E6=B5=81=E5=85=9C=E5=BA=95=E4=B8=8E=E9=A6=96=20token?= =?UTF-8?q?=20=E5=89=8D=E9=87=8D=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 修法对照 ParalonCloud「prefill 阶段 tunnel 被宰」三规则: - R1 首字节前可重试:peekFirstOutput 窥视首个完整 SSE 帧,EOF/超时/上游 error 帧一律按未 commit 处理,由 claudeResponsesStreamWithRetry 换 key 重发 - R2 已 commit 不重试:已有 text/tool 产出时 EOF 仍合成 stop(原行为) - 兜底:仅 thinking 没有 text 时 reasoningFallback 提升为 text,agent 至少 拿到思考内容而不是空手中断 config 新增 stream_empty_retry_max(默认 1)、stream_first_byte_timeout_ms (默认 30000),仅 config.json 控制,不进 admin UI。重试经 callOpenCodeEndpoint 自动落到 key_pool 下一个 key,不会重复同一根死 pipe;input tokens 在缓存 命中场景下重试近零成本。 测试:claude_responses_empty_retry_test.go 8 用例覆盖空流/挂死/错误帧/ 部分流兜底;既有 TestClaudeResponsesStream_Mapping 不回归。`make build / vet / fmt / test` 全绿。 参考:ParalonCloud《Stream ended without finish_reason》规则、new-api #3275。 Co-Authored-By: Claude Code --- config.example.json | 6 +- docs/CONFIGURATION.md | 26 ++ internal/app/claude.go | 12 + internal/app/claude_responses.go | 296 +++++++++++++++- .../app/claude_responses_empty_retry_test.go | 316 ++++++++++++++++++ internal/app/config.go | 6 + internal/app/main_test.go | 27 +- internal/config/config.go | 29 ++ internal/domain/types.go | 8 + 9 files changed, 703 insertions(+), 23 deletions(-) create mode 100644 internal/app/claude_responses_empty_retry_test.go diff --git a/config.example.json b/config.example.json index 1396aea..5c2fb56 100644 --- a/config.example.json +++ b/config.example.json @@ -29,5 +29,9 @@ {"id": "k1", "key": "sk-placeholder-1", "group": "", "weight": 1, "enabled": true, "note": "account A"}, "sk-placeholder-2" ] - } + }, + "_comment_stream_empty_retry": "claude→responses 流式链路的空流兜底:上游 200 后首个有效产出前若遇到空流 EOF / 无数据超时,最多重试 N 次(0=关闭,默认 1)。重试会经 key_pool 自动切到下一个可用 key。", + "stream_empty_retry_max": 1, + "_comment_stream_first_byte_timeout_ms": "空流检测的首字节看门狗(毫秒,默认 30000,<=0 关闭)。上游 200 后一直没发任何 SSE 数据,超过该阈值视为空流并触发重试。", + "stream_first_byte_timeout_ms": 30000 } diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index b9cff9f..39b3df6 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -277,6 +277,32 @@ opencode zen 上游的 base URL 列表。默认（未设置或为空数组）为 > 真实运行验证：`opencode2api launch claude --model mimo-v2.6-flash` 第二轮 `prompt_cached_tokens` 从 ~28.7k 提升到 ~32.4k（≈99.9% 的 prompt 命中），`big-pickle` 31.5k/31.6k；`codex --model mimo-v2.6-flash` `prompt_cached_tokens=9.92k`（≈98%），都已通过 `OPENCODE2API_CACHE_DEBUG=1` 中的 `cache_debug_usage` 观察。先前行为是只在 `buildUpstreamBody` 时注入顶层 `prompt_cache_retention`；现在 chat/claude/responses 直通（remembered）与 chat→responses 桥都统一补齐，并保持 IDEMPOTENT（上游已有字段时不覆盖）。 +### `stream_empty_retry_max` / `stream_first_byte_timeout_ms` + +claude→responses 流式链路的「空流兜底 + 首 token 前重试」。覆盖两类常见上游故障: + +- **prefill 阶段被宰**:上游代理(CF / nginx)在首个 token 前杀 tunnel,网关只收到一个干净的 EOF——按旧实现客户端会看到 `stream ended without completion`,agent 中断。 +- **挂死**:上游接受了连接但既不发数据也不关,客户端永久等待。 + +开启后(默认开启):上游 200 收到、但还没向客户端 WriteHeader 之前的窗口里,遇到 **空流 EOF / 超时未发数据 / 上游只发 `response.failed` 错误帧**,静默重发同一份请求最多 `stream_empty_retry_max` 次,客户端完全无感;重试经 `key_pool` 自动落到下一个可用 key,不会重复同一根 pipe。 + +```json +{ + "stream_empty_retry_max": 1, + "stream_first_byte_timeout_ms": 30000 +} +``` + +- `stream_empty_retry_max`:重试次数,默认 `1`,`0` 关闭。每个 attempt 都用同一份请求体重发(prompt_cache_key 稳定,input tokens 在缓存命中时接近零成本)。 +- `stream_first_byte_timeout_ms`:peek 窗口毫秒数,默认 `30000`(30s)。覆盖大多数上游 prefill 时间;`<=0` 关闭看门狗,仅 EOF/error 触发。 + +**不重试的情况**(不改的承诺): +- 已向客户端写过任何字节后 EOF/杀流——按 ParalonCloud Rule 2 合成正常 stop 收尾(已有产出交付给 agent)。 +- 仅有 thinking 没有 text 的 EOF——`reasoningFallback` 兜底把思考内容提升为 text,agent 拿到思考、不发 error。 +- 非流式请求(`stream: false`)走的是另一条路径,与本机制无关。 + +> 真实运行验证:故意用反代在 5s 时杀 upstream tunnel,agent 端原本会 `stream ended without completion` 中断;开启本机制后第一次空流透明重试到下一个 key,整轮圆满完成。 + ## 管理面板 打开 `http://127.0.0.1:8000/` 可进入管理面板。面板可以修改配置、刷新模型和查看 token 统计。管理面板现已可设置 `prompt_cache_retention`、`cache_control_breakpoints`、`socks5_sticky`、`text_only_models`（「模型与路由」/「网络与代理」Tab），保存时这些字段随其余配置一并持久化到 `config.json`，不再被静默回擦。 diff --git a/internal/app/claude.go b/internal/app/claude.go index c7817c2..d19c145 100644 --- a/internal/app/claude.go +++ b/internal/app/claude.go @@ -96,6 +96,18 @@ func (r *streamReader) Read() <-chan streamReadResult { return r.readCh } // disabled. func (r *streamReader) Keepalive() <-chan time.Time { return r.keepCh } +// enableKeepalive 在 reader 已启动后补一个 keepalive ticker。用于「先 +// 零 keepalive 完成 peek 窗口(避免把 ticker 当 watch dog),commit 后 +// 再切到常规心跳」。幂等:重复调用只保留首个 ticker;Close 会停掉它。 +func (r *streamReader) enableKeepalive(interval time.Duration) { + if interval <= 0 || r.keepCh != nil { + return + } + ticker := time.NewTicker(interval) + r.keepCh = ticker.C + r.stopKeepalive = ticker.Stop +} + // Close stops the reader: it signals the goroutine, unblocks any pending // read by closing the upstream body, and waits for the goroutine to exit. func (r *streamReader) Close() { diff --git a/internal/app/claude_responses.go b/internal/app/claude_responses.go index 1421308..d16964b 100644 --- a/internal/app/claude_responses.go +++ b/internal/app/claude_responses.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "github.com/6Kmfi6HP/opencode2api/internal/config" "github.com/6Kmfi6HP/opencode2api/internal/logging" @@ -970,8 +971,14 @@ func probeClaudeViaResponses(ctx context.Context, w http.ResponseWriter, auth Up logging.FromContext(ctx).Info("claude_responses_probe_succeeded", "model", modelID, "stream", stream) if stream { - claudeResponsesStreamHandler(ctx, w, rc, modelID, wantReasoning) - return true + committed, streamErr := claudeResponsesStreamWithRetry(ctx, w, auth, modelID, claudeReq, wantReasoning, rc) + if committed { + return true + } + // 全部 attempt 都未 commit(空流/timeout),让调用方走 fallback。 + logging.FromContext(ctx).Warn("claude-responses probe stream empty after retries", + "model", modelID, "err", streamErr) + return false } respBody, readErr := io.ReadAll(io.LimitReader(rc, 32*1024*1024)) if readErr != nil { @@ -1013,8 +1020,15 @@ func forwardClaudeViaResponses(ctx context.Context, w http.ResponseWriter, auth } if stream && status >= 200 && status < 300 { - claudeResponsesStreamHandler(ctx, w, rc, modelID, wantReasoning) - return true + committed, streamErr := claudeResponsesStreamWithRetry(ctx, w, auth, modelID, claudeReq, wantReasoning, rc) + if committed { + return true + } + // 全部 attempt 都未 commit(空流/timeout/rc 异常),让调用方走 + // 更上层的 fallback(probe 的调用方会落到常规 chat 翻译路径)。 + logging.FromContext(ctx).Warn("claude-responses forward stream empty after retries", + "model", modelID, "status", status, "err", streamErr) + return false } respBody, readErr := io.ReadAll(io.LimitReader(rc, 32*1024*1024)) @@ -1048,8 +1062,6 @@ func forwardClaudeViaResponses(ctx context.Context, w http.ResponseWriter, auth return true } -// ======================== Claude 经原生 responses 的流式转换 ======================== - type claudeResponsesBlock struct { claudeIndex int kind string // text | thinking | tool @@ -1061,14 +1073,221 @@ type claudeResponsesBlock struct { signature string } -func claudeResponsesStreamHandler(ctx context.Context, w http.ResponseWriter, rc io.Reader, model string, wantReasoning bool) { +// ======================== Claude 经原生 responses 的流式转换 ======================== + +// claudeResponsesStreamWithRetry 负责把「peek 失败的可重试错误」翻译为最多 +// StreamEmptyRetryMax 次同请求重试。首轮复用调用方已打开的 firstRC；后续 +// attempt 重新构请求（callOpenCodeEndpoint 会自动切到 key pool 的下一个 +// 可用 key——空流不再重试同一把死 key）。 +// +// 返回与 claudeResponsesStreamHandler 一致：(true, nil) 已 commit / +// (false, err) 全部 attempt 都未 commit。ctx.Done 与所有错误原样透传。 +func claudeResponsesStreamWithRetry(ctx context.Context, w http.ResponseWriter, auth UpstreamAuth, modelID string, claudeReq ClaudeRequest, wantReasoning bool, firstRC io.ReadCloser) (bool, error) { + maxRetry := config.StreamEmptyRetryMax() + rc := firstRC + for attempt := 0; attempt <= maxRetry; attempt++ { + committed, err := claudeResponsesStreamHandler(ctx, w, rc, modelID, wantReasoning, nil) + if committed { + return true, nil + } + rc.Close() + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return false, err + } + if attempt < maxRetry { + logging.FromContext(ctx).Warn("claude-responses empty stream, retrying with next key", + "model", modelID, "attempt", attempt+1, "max_retry", maxRetry, "cause", err) + nextRC, status, _, callErr := callOpenCodeEndpoint(ctx, "responses", claudeToResponsesBody(claudeReq, modelID), modelID, auth) + if callErr != nil || status < 200 || status >= 300 { + if nextRC != nil { + nextRC.Close() + } + if callErr != nil { + return false, callErr + } + return false, fmt.Errorf("upstream status %d on retry", status) + } + rc = nextRC + continue + } + return false, err + } + return false, errStreamIncompleteNoCommit +} + +// errStreamIncompleteNoCommit：peek 窗口内上游空流 EOF / 首字节超时 / 读 +// 错误 / 上游错误事件——尚未向客户端写过任何字节，调用方可安全重试。 +var errStreamIncompleteNoCommit = errors.New("stream incomplete before first client byte") + +// peekOutcome 是 peekFirstOutput 的结果。consumed 不为空时表示「已有完整 +// SSE 帧被消费」，调用方应把它原样喂回 handler 主循环。 +type peekOutcome struct { + consumed []streamReadResult + err error + // reader 是 peek 内部使用的流式 reader（已经包了一层自己的 bufio 缓 + // 冲）。调用方应把它原样续用做主循环 reader ——否则其内部 bufio 里 + // 已经预读的行会被两个独立 bufio 撕成两半。 + reader *streamReader +} + +// peekFirstOutput 在向上游拿到 200、但还未向客户端 WriteHeader 之前「窥 +// 视」首个**完整 SSE 帧**(空行收尾；流末尾则接受 EOF 收尾)。 +// +// 返回约定： +// - hasFrame=true 且 errorEvent=false:上游已产出有效事件(壳或 +// delta)——commit,handler 进入主循环继续。 +// - hasFrame=true 且 errorEvent=true:上游发了 error / response.failed +// ——按 errStreamIncompleteNoCommit 返回,由调用方走未 commit 重试。 +// - hasFrame=false:窗口内 EOF、读错或超时——同样返回 +// errStreamIncompleteNoCommit(空流,可安全重试)。 +// +// 关键不变量:返回时 consumed 一定落在帧边界(空行或 EOF 行)之后, +// 绝不截断在帧中间——否则 handler 侧的 bufio 续读会把当前帧的剩余行 +// 与后续帧拼错。 +func peekFirstOutput(ctx context.Context, rc io.Reader, timeout time.Duration) peekOutcome { + reader := newStreamReader(ctx, rc, 0) + // 不要 defer Close:成功路径里调用方会续用这个 reader(它的 bufio + // 里可能已经预读了后续行);只有失败路径在这里显式关闭。 + + var timeoutCh <-chan time.Time + var timer *time.Timer + if timeout > 0 { + timer = time.NewTimer(timeout) + timeoutCh = timer.C + defer timer.Stop() + } + + var consumed []streamReadResult + var frameBuf []string + // hasFrame：已经见过至少一个完整 data 帧（含 ERROR 帧——错误帧也是「有产出」）。 + hasFrame := false + // errorEvent：已见帧里存在 error / response.failed。 + errorEvent := false + + // flushFrame 在当前帧边界（空行或 EOF）结算 frameBuf：判定 hasFrame / + // errorEvent，并原样把整帧追加进 consumed。clearPending 表示丢弃而不是 + // 结算（用于 input：上游事件 payload 里夹的裸 error JSON，不算帧）。 + flushFrame := func() { + for _, dl := range frameBuf { + t := strings.TrimSpace(dl) + if t == "" || t == "[DONE]" { + continue + } + hasFrame = true + if !errorEvent { + var evt map[string]any + if json.Unmarshal([]byte(t), &evt) == nil { + typ, _ := evt["type"].(string) + if typ == "error" || typ == "response.failed" { + errorEvent = true + } else if _, hasErr := evt["error"]; hasErr { + errorEvent = true + } + } + } + } + frameBuf = nil + } + + for { + select { + case <-ctx.Done(): + return peekOutcome{err: ctx.Err()} + case <-ctx.Done(): + reader.Close() + return peekOutcome{err: ctx.Err()} + case <-timeoutCh: + reader.Close() + return peekOutcome{err: errStreamIncompleteNoCommit} + case res := <-reader.Read(): + consumed = append(consumed, res) + trimmed := strings.TrimSpace(res.line) + switch { + case trimmed == "": + flushFrame() + case strings.HasPrefix(trimmed, ":"): + // SSE 心跳注释：不算产出。 + case strings.HasPrefix(trimmed, "data:"): + frameBuf = append(frameBuf, strings.TrimSpace(strings.TrimPrefix(trimmed, "data:"))) + case strings.HasPrefix(trimmed, "event:"): + // 仅记录属于哪个事件；data 到帧尾才结算。 + case strings.HasPrefix(trimmed, "{"): + // 非标准裸 JSON 行：直接当一帧。 + frameBuf = append(frameBuf, trimmed) + flushFrame() + } + if res.err != nil { + // EOF：流自然终止。当前若有残帧,先按帧结算;随后无论 + // 是否有产出都关闭——截在 EOF 处 consumed 已对齐帧边界。 + flushFrame() + if !hasFrame || errorEvent { + // 失败路径:把 reader 也关掉,免得泄漏 goroutine。 + reader.Close() + return peekOutcome{consumed: consumed, err: errStreamIncompleteNoCommit} + } + return peekOutcome{consumed: consumed, reader: reader} + } + if hasFrame { + if errorEvent { + reader.Close() + return peekOutcome{consumed: consumed, err: errStreamIncompleteNoCommit} + } + return peekOutcome{consumed: consumed, reader: reader} + } + } + } +} + +// claudeResponsesStreamHandler 把上游 Responses SSE 翻译为 Claude SSE。 +// +// 返回值的约定（与 peekFirstOutput 配合，实现「首 token 前可重试」）： +// - (true, nil)：已向客户端写过至少一个字节（含 ping），上游流正常翻 +// 译完毕或按已有规则合成收尾。这是唯一「已 commit」的返回。 +// - (false, err)：未向客户端写过任何字节——peek 窗口里上游给了空流/ +// 错误事件/EOF/超时。调用方可以安全地关闭当前 rc、换 key 重发请求。 +// +// peeked 非空时直接进入主循环（此调用已是某次 peek-commit 之后的干跑）， +// 不再二次 peek、不再做首字节看门狗（窗口已在第一次调用里耗尽）。 +func claudeResponsesStreamHandler(ctx context.Context, w http.ResponseWriter, rc io.Reader, model string, wantReasoning bool, peeked []streamReadResult) (bool, error) { + var reader *streamReader + if len(peeked) == 0 { + peek := peekFirstOutput(ctx, rc, time.Duration(config.StreamFirstByteTimeoutMs())*time.Millisecond) + if peek.err != nil { + return false, peek.err + } + peeked = peek.consumed + reader = peek.reader + if reader == nil { + // 上游 EOF 但已有完整帧（极少见：单帧流）。续读的 reader 直接 + // 落在已 EOF 的 rc 上,主循环立即收 EOF 并走 finalize。 + reader = newStreamReader(ctx, rc, 15*time.Second) + } else { + // 复用 peek 的 reader(它的 bufio 可能已预读后续行);顺手开 + // keepalive(此前为 0,看门狗由 timeout 承担)。 + reader.enableKeepalive(15 * time.Second) + } + } else { + reader = newStreamReader(ctx, rc, 15*time.Second) + } + defer reader.Close() + w.Header().Set("Content-Type", "text/event-stream") w.Header().Set("Cache-Control", "no-cache") w.Header().Set("Connection", "keep-alive") - w.WriteHeader(http.StatusOK) flusher, _ := w.(http.Flusher) stats := &logging.StreamStats{Start: time.Now()} + // flushPending 按序回放 peek 阶段攒下的行（peek 阶段已按 SSE 行结构 + // 验证过帧完整性），随后转为 nil 直读上游。 + var pending []streamReadResult + if len(peeked) > 0 { + pending = append([]streamReadResult(nil), peeked...) + } + flushPending := func() []streamReadResult { + out := pending + pending = nil + return out + } msgID := fmt.Sprintf("msg_%s", randomString(24)) blockIndex := 0 @@ -1094,7 +1313,12 @@ func claudeResponsesStreamHandler(ctx context.Context, w http.ResponseWriter, rc fullTextLen := 0 fullReasoningLen := 0 + wroteHeader := false emitEvent := func(event string, data any) { + if !wroteHeader { + w.WriteHeader(http.StatusOK) + wroteHeader = true + } writeSSEEvent(w, flusher, event, data) } // adoptResponseID: response.created/in_progress 的 response.id 非空时， @@ -1209,8 +1433,10 @@ func claudeResponsesStreamHandler(ctx context.Context, w http.ResponseWriter, rc } stats.ReasoningChars += len(text) fullReasoningLen += len(text) + // 无论 wantReasoning 与否都累积 reasoningFallback：空流 EOF 兜底 + // （finalizeClaudeResponsesStream 的「仅有思考无文本」分支）依赖它。 + reasoningFallback.WriteString(text) if wantReasoning { - reasoningFallback.WriteString(text) startThinkingBlock(b) emitEvent("content_block_delta", map[string]any{ "type": "content_block_delta", "index": b.claudeIndex, @@ -1307,15 +1533,36 @@ func claudeResponsesStreamHandler(ctx context.Context, w http.ResponseWriter, rc } } - // keepalive：首 token 前客户端仅能收到 ping。 - reader := newStreamReader(ctx, rc, 15*time.Second) - defer reader.Close() + // 先回放 peek 阶段攒下的行（帧边界已验证），再直读上游。peeked 为 + // nil 时表示正常路径（peek 在函数开头已做过），此处 pending 必为空、 + // 循环等同于旧行为。 + for _, res := range flushPending() { + line := res.line + trimmedRight := strings.TrimRight(line, "\r\n") + trimmed := strings.TrimSpace(line) + if trimmed == "" { + flushFrame() + if finished { + break + } + } else if strings.HasPrefix(trimmed, ":") { + } else if strings.HasPrefix(trimmed, "event:") { + frameEvent = strings.TrimSpace(strings.TrimPrefix(trimmed, "event:")) + } else if strings.HasPrefix(trimmed, "data:") { + frameData = append(frameData, strings.TrimSpace(strings.TrimPrefix(trimmedRight, "data:"))) + } else if strings.HasPrefix(trimmed, "{") { + frameData = append(frameData, trimmed) + } + if finished && len(frameData) == 0 { + continue + } + } loop: for { select { case <-ctx.Done(): - return + return true, nil case <-reader.Keepalive(): emitEvent("ping", map[string]any{"type": "ping"}) case res := <-reader.Read(): @@ -1348,15 +1595,31 @@ loop: flushFrame() } if !finalized { - if producedText || len(toolOrder) > 0 { + switch { + case producedText || len(toolOrder) > 0: // 上游干净 EOF 但缺 completed（如 muse-spark 系只发事件不发 DONE）： // 合成正常结束，不报错。 stats.SawFinish = true stats.FinishReason = "stop" finished = true doFinalize() - } else { - emitError("stream ended without completion") + case reasoningFallback.Len() > 0: + // 仅 thinking 无 text/tool：上游在思考阶段被杀。 + // reasoningFallback 无条件累积（见 emitThinkingDelta）， + // finalize 里「空回复保护」会把它提升为文本块——agent + // 至少拿到思考内容，不会空手中断。 + stats.SawFinish = true + stats.FinishReason = "stop" + stats.PromotedReasoning = true + finished = true + doFinalize() + default: + // 完全空流且已过了 peek 阶段（说明 peek 时见过壳 + // 事件，之后才断）：保持显式 error，不默默吞掉。 + stats.SawFinish = false + logging.FromContext(ctx).Warn("claude-responses stream ended without content", + "model", model, "text_chars", stats.TextChars, "reasoning_chars", stats.ReasoningChars) + emitError("upstream ended stream before any content (empty completion)") } } break loop @@ -1370,6 +1633,7 @@ loop: _ = fullReasoningLen _ = fullTextLen _ = bytes.MinRead + return true, nil } func indexOfToolOrder(order []int, v int) (int, bool) { diff --git a/internal/app/claude_responses_empty_retry_test.go b/internal/app/claude_responses_empty_retry_test.go new file mode 100644 index 0000000..e4c3989 --- /dev/null +++ b/internal/app/claude_responses_empty_retry_test.go @@ -0,0 +1,316 @@ +package app + +// claude→responses 流式链路的「空流兜底 + 首 token 前重试」端到端回归。 +// +// 覆盖三档故障形态(对照 docs/CONFIGURATION.md 的 stream_empty_retry_max): +// 1. 空流 EOF(整流 EOF、零产出)→ 双层兜底:reasoningFallback 提升 / +// claudeResponsesStreamWithRetry 重试。 +// 2. 首字节超时(StreamFirstByteTimeoutMs 内无任何 SSE 行)。 +// 3. 上游返回 200 后直接发 response.failed / error 帧(本应回 502 的活 +// 失败的 2xx 形态)。 +// +// 4. 对比基线:有部分产出后 EOF——按「已 commit,合成 stop」(ParalonCloud +// Rule 2),不重试、不发 error。 + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/6Kmfi6HP/opencode2api/internal/config" +) + +// stubRetryConfig 把 StreamEmptyRetryMax / StreamFirstByteTimeoutMs 覆盖为 +// 测试所需值,t.Cleanup 里恢复。 +func stubRetryConfig(t *testing.T, retryMax, firstByteTimeoutMs int) { + t.Helper() + old := config.Get() + config.Update(func(s *config.Snapshot) { + s.StreamEmptyRetryMax = retryMax + s.StreamFirstByteTimeoutMs = firstByteTimeoutMs + }) + t.Cleanup(func() { + config.Update(func(s *config.Snapshot) { + s.StreamEmptyRetryMax = old.StreamEmptyRetryMax + s.StreamFirstByteTimeoutMs = old.StreamFirstByteTimeoutMs + }) + }) +} + +// stubNativeModel 把 modelID 标记为「已知原生 responses 模型」,让 +// claudeMessagesHandler 直接走 forwardClaudeViaResponses,而不是先跌进 +// chat 翻译再 probe。 +func stubNativeModel(t *testing.T, modelID string) { + t.Helper() + oldModelAlias := getModelKeywordRules() + applyConfig(AppConfig{}) + nativeResponsesModels.Lock() + nativeResponsesModels.ids[modelID] = true + nativeResponsesModels.Unlock() + t.Cleanup(func() { + applyConfig(AppConfig{ModelAlias: oldModelAlias}) + nativeResponsesModels.Lock() + delete(nativeResponsesModels.ids, modelID) + nativeResponsesModels.Unlock() + }) +} + +// 一份「正常完结」的最小 responses SSE 流(2 行 text delta + completed)。 +const claudeResponsesHealthySSE = "event: response.created\n" + + `data: {"type":"response.created","response":{"id":"resp_ok"}}` + "\n\n" + + "event: response.output_item.added\n" + + `data: {"type":"response.output_item.added","output_index":0,"item":{"id":"msg_1","type":"message","role":"assistant"}}` + "\n\n" + + "event: response.output_text.delta\n" + + `data: {"type":"response.output_text.delta","output_index":0,"item_id":"msg_1","delta":"hello"}` + "\n\n" + + "event: response.completed\n" + + `data: {"type":"response.completed","response":{"id":"resp_ok","status":"completed","output":[{"id":"msg_1","type":"message","content":[{"type":"output_text","text":"hello"}]}],"usage":{"input_tokens":1,"output_tokens":1,"total_tokens":2}}}` + "\n\n" + +// 1. 首轮空流 → 重试拿到正常流:客户端只看到一条干净的成功流,不应出现 +// error 事件、不应看到前半截残流。这是真正的痛点场景(prefill 阶段 +// tunnel 被宰,直接干净 EOF)。 +func TestClaudeResponsesStream_EmptyEOF_RetriesOnce(t *testing.T) { + stubRetryConfig(t, 1, 5000) + stubNativeModel(t, "claude-stream-model") + + installFakeOpenCodeClient(t, []fakeUpstreamResponse{ + {status: http.StatusOK, body: ``, header: http.Header{"Content-Type": []string{"text/event-stream"}}}, + {status: http.StatusOK, body: claudeResponsesHealthySSE, header: http.Header{"Content-Type": []string{"text/event-stream"}}}, + }) + + req := httptest.NewRequest(http.MethodPost, "/v1/messages", + strings.NewReader(`{"model":"claude-stream-model","max_tokens":32,"stream":true,"messages":[{"role":"user","content":"hi"}]}`)) + rec := httptest.NewRecorder() + claudeMessagesHandler(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200, body=%s", rec.Code, rec.Body.String()) + } + body := rec.Body.String() + if !strings.Contains(body, `"content_block_delta"`) || !strings.Contains(body, "hello") { + t.Fatalf("expected text delta from retry, got:\n%s", body) + } + if !strings.Contains(body, `"message_stop"`) { + t.Fatalf("expected clean message_stop, got:\n%s", body) + } + if strings.Contains(body, `"error"`) || strings.Contains(body, "upstream ended stream") { + t.Fatalf("client must not see retry-induced error:\n%s", body) + } +} + +// 2. 两次都空流:重试额度用尽,客户端收到一条明确的 error 事件(不能 +// 静默吞掉、也不能伪造正常收尾)。 +func TestClaudeResponsesStream_EmptyEOF_ExhaustsRetryThenErrors(t *testing.T) { + stubRetryConfig(t, 1, 5000) + stubNativeModel(t, "claude-stream-model") + + installFakeOpenCodeClient(t, []fakeUpstreamResponse{ + {status: http.StatusOK, body: ``, header: http.Header{"Content-Type": []string{"text/event-stream"}}}, + {status: http.StatusOK, body: ``, header: http.Header{"Content-Type": []string{"text/event-stream"}}}, + }) + + req := httptest.NewRequest(http.MethodPost, "/v1/messages", + strings.NewReader(`{"model":"claude-stream-model","max_tokens":32,"stream":true,"messages":[{"role":"user","content":"hi"}]}`)) + rec := httptest.NewRecorder() + claudeMessagesHandler(rec, req) + + // 一直未 commit,所以走 forwardClaudeViaResponses 的 fallback,落 502。 + if rec.Code != http.StatusBadGateway { + t.Fatalf("status = %d, want 502 (fallback after retries exhausted), body=%s", rec.Code, rec.Body.String()) + } +} + +// 3) 重试关闭(retryMax=0):首轮空流直接落 502,不重试。 +func TestClaudeResponsesStream_EmptyEOF_RetryDisabled(t *testing.T) { + stubRetryConfig(t, 0, 5000) + stubNativeModel(t, "claude-stream-model") + + installFakeOpenCodeClient(t, []fakeUpstreamResponse{ + {status: http.StatusOK, body: ``, header: http.Header{"Content-Type": []string{"text/event-stream"}}}, + }) + + req := httptest.NewRequest(http.MethodPost, "/v1/messages", + strings.NewReader(`{"model":"claude-stream-model","max_tokens":32,"stream":true,"messages":[{"role":"user","content":"hi"}]}`)) + rec := httptest.NewRecorder() + claudeMessagesHandler(rec, req) + + if rec.Code != http.StatusBadGateway { + t.Fatalf("status = %d, want 502 (no retry), body=%s", rec.Code, rec.Body.String()) + } +} + +// 4. 已有部分产出后 EOF(话题做了一半被宰):按 ParalonCloud Rule 2 不 +// 重试,按现有「合成 stop」逻辑正常关流——既不让 agent 拿半截,也不 +// 多达一次重算 input。 +func TestClaudeResponsesStream_PartialEOF_SynthesizesStopNoRetry(t *testing.T) { + stubRetryConfig(t, 1, 5000) + stubNativeModel(t, "claude-stream-model") + + partialSSE := "event: response.created\n" + + `data: {"type":"response.created","response":{"id":"resp_partial"}}` + "\n\n" + + "event: response.output_item.added\n" + + `data: {"type":"response.output_item.added","output_index":0,"item":{"id":"msg_1","type":"message","role":"assistant"}}` + "\n\n" + + "event: response.output_text.delta\n" + + `data: {"type":"response.output_text.delta","output_index":0,"item_id":"msg_1","delta":"partial answer"}` + "\n\n" + // 故意没有 response.completed ——上游在 text delta 后死掉。 + + installFakeOpenCodeClient(t, []fakeUpstreamResponse{ + {status: http.StatusOK, body: partialSSE, header: http.Header{"Content-Type": []string{"text/event-stream"}}}, + // 即便是当 retry 候选位准备的一条正常流,也绝不应被发出。 + {status: http.StatusOK, body: claudeResponsesHealthySSE, header: http.Header{"Content-Type": []string{"text/event-stream"}}}, + }) + + req := httptest.NewRequest(http.MethodPost, "/v1/messages", + strings.NewReader(`{"model":"claude-stream-model","max_tokens":32,"stream":true,"messages":[{"role":"user","content":"hi"}]}`)) + rec := httptest.NewRecorder() + claudeMessagesHandler(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200, body=%s", rec.Code, rec.Body.String()) + } + body := rec.Body.String() + // 半截正文必须送达。 + if !strings.Contains(body, "partial answer") { + t.Fatalf("expected partial text to reach client:\n%s", body) + } + // 合成正常收尾,不发 error。 + if !strings.Contains(body, `"message_stop"`) { + t.Fatalf("expected synthesized message_stop:\n%s", body) + } + if strings.Contains(body, `"error"`) { + t.Fatalf("must not emit error on partial EOF (already committed):\n%s", body) + } + // 重试不应被触发:fallback 槽位里的正常流不应出现在响应里。 + if strings.Contains(body, "hello") { + t.Fatalf("retry slot must not have been consumed (already-committed stream):\n%s", body) + } +} + +// 5. 仅有 thinking 死流:agent 至少拿到思考内容(reasoningFallback 提升 +// 为 text),不发 error。这是「disconnect during thinking」的兜底。 +func TestClaudeResponsesStream_ThinkingOnlyEOF_PromotesReasoning(t *testing.T) { + stubRetryConfig(t, 0, 5000) // 关重试,才能直接观察兜底行为(开着重试会先重试一次) + stubNativeModel(t, "claude-stream-model") + + thinkingOnlySSE := "event: response.created\n" + + `data: {"type":"response.created","response":{"id":"resp_thinking"}}` + "\n\n" + + "event: response.output_item.added\n" + + `data: {"type":"response.output_item.added","output_index":0,"item":{"id":"r1","type":"reasoning"}}` + "\n\n" + + "event: response.reasoning_text.delta\n" + + `data: {"type":"response.reasoning_text.delta","output_index":0,"item_id":"r1","delta":"let me think"}` + "\n\n" + // EOF：上游在思考阶段被杀。 + + installFakeOpenCodeClient(t, []fakeUpstreamResponse{ + {status: http.StatusOK, body: thinkingOnlySSE, header: http.Header{"Content-Type": []string{"text/event-stream"}}}, + }) + + req := httptest.NewRequest(http.MethodPost, "/v1/messages", + strings.NewReader(`{"model":"claude-stream-model","max_tokens":32,"stream":true,"messages":[{"role":"user","content":"hi"}]}`)) + rec := httptest.NewRecorder() + claudeMessagesHandler(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200, body=%s", rec.Code, rec.Body.String()) + } + body := rec.Body.String() + // 思考内容必须以 text_delta 形式送达(空回复保护)。 + if !strings.Contains(body, "let me think") { + t.Fatalf("expected reasoning promoted to text, got:\n%s", body) + } + if !strings.Contains(body, `"message_stop"`) { + t.Fatalf("expected synthesized message_stop:\n%s", body) + } + if strings.Contains(body, `"error"`) { + t.Fatalf("must not emit error when reasoning was promoted:\n%s", body) + } +} + +// 6. 上游返回 200 但首帧就是 response.failed:按未 commit 处理,走重试, +// 而不是把半截 SSE 流甩给客户端。 +func TestClaudeResponsesStream_ErrorFrame_Retries(t *testing.T) { + stubRetryConfig(t, 1, 5000) + stubNativeModel(t, "claude-stream-model") + + errorOnlySSE := "event: response.failed\n" + + `data: {"type":"response.failed","response":{"id":"resp_bad","status":"failed","error":{"message":"upstream internal"}}}` + "\n\n" + + installFakeOpenCodeClient(t, []fakeUpstreamResponse{ + {status: http.StatusOK, body: errorOnlySSE, header: http.Header{"Content-Type": []string{"text/event-stream"}}}, + {status: http.StatusOK, body: claudeResponsesHealthySSE, header: http.Header{"Content-Type": []string{"text/event-stream"}}}, + }) + + req := httptest.NewRequest(http.MethodPost, "/v1/messages", + strings.NewReader(`{"model":"claude-stream-model","max_tokens":32,"stream":true,"messages":[{"role":"user","content":"hi"}]}`)) + rec := httptest.NewRecorder() + claudeMessagesHandler(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200, body=%s", rec.Code, rec.Body.String()) + } + body := rec.Body.String() + if !strings.Contains(body, "hello") { + t.Fatalf("expected text from retry, got:\n%s", body) + } + // 上游的 response.failed 不应直接翻译成 client-visible 的 error 帧 + // (它已被 peek 吞掉并触发重试)。 + if strings.Contains(body, "upstream internal") { + t.Fatalf("upstream error must not leak to client when retry succeeded:\n%s", body) + } +} + +// 7. 首字节看门狗:上游接受连接后只在窗口内发心跳注释、不发任何 data +// 帧——超过 stream_first_byte_timeout_ms 应触发空流重试。 +func TestClaudeResponsesStream_SilentUpstream_TimesOut(t *testing.T) { + stubRetryConfig(t, 1, 200) // 200ms 首字节超时 + stubNativeModel(t, "claude-stream-model") + + // 上游 body 里只有 SSE 注释行(心跳),没有任何 data: 帧,然后 EOF。 + // peek 会消费完这些注释,在 EOF 处判定「无产出」并触发重试。 + silentBody := ": keepalive\n: keepalive\n: keepalive\n" + + installFakeOpenCodeClient(t, []fakeUpstreamResponse{ + {status: http.StatusOK, body: silentBody, header: http.Header{"Content-Type": []string{"text/event-stream"}}}, + {status: http.StatusOK, body: claudeResponsesHealthySSE, header: http.Header{"Content-Type": []string{"text/event-stream"}}}, + }) + + req := httptest.NewRequest(http.MethodPost, "/v1/messages", + strings.NewReader(`{"model":"claude-stream-model","max_tokens":32,"stream":true,"messages":[{"role":"user","content":"hi"}]}`)) + rec := httptest.NewRecorder() + claudeMessagesHandler(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200, body=%s", rec.Code, rec.Body.String()) + } + if !strings.Contains(rec.Body.String(), "hello") { + t.Fatalf("expected retry to recover stream after silent-upstream:\n%s", rec.Body.String()) + } +} + +// 8. 完全死锁(连接接受后一字节都不发,纯挂在 socket 上)——由 +// stream_first_byte_timeout_ms 兜出,不应傻等到外层 ctx 超时。 +func TestClaudeResponsesStream_HungUpstream_FirstByteWatchdogFires(t *testing.T) { + if testing.Short() { + t.Skip("skipping hung-upstream test in -short mode") + } + stubRetryConfig(t, 1, 200) // 200ms 首字节超时 + stubNativeModel(t, "claude-stream-model") + + installFakeOpenCodeClient(t, []fakeUpstreamResponse{ + // 用 slowReader 的 Close-only 行为模拟挂死:Read 永远阻塞,直 + // 到 Close 才返回 EOF。peek 的 timeout 必须先于它触发。 + {status: http.StatusOK, body: "", header: http.Header{"Content-Type": []string{"text/event-stream"}}, customBody: newSlowReader()}, + {status: http.StatusOK, body: claudeResponsesHealthySSE, header: http.Header{"Content-Type": []string{"text/event-stream"}}}, + }) + + req := httptest.NewRequest(http.MethodPost, "/v1/messages", + strings.NewReader(`{"model":"claude-stream-model","max_tokens":32,"stream":true,"messages":[{"role":"user","content":"hi"}]}`)) + rec := httptest.NewRecorder() + claudeMessagesHandler(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200, body=%s", rec.Code, rec.Body.String()) + } + if !strings.Contains(rec.Body.String(), "hello") { + t.Fatalf("expected retry to recover from hung upstream:\n%s", rec.Body.String()) + } +} diff --git a/internal/app/config.go b/internal/app/config.go index 258dfd0..513aff9 100644 --- a/internal/app/config.go +++ b/internal/app/config.go @@ -188,6 +188,12 @@ func applyConfig(cfg AppConfig) { if cfg.TextOnlyModels != nil { s.TextOnlyModels = append([]string(nil), cfg.TextOnlyModels...) } + if cfg.StreamEmptyRetryMax != nil { + s.StreamEmptyRetryMax = *cfg.StreamEmptyRetryMax + } + if cfg.StreamFirstByteTimeoutMs != nil { + s.StreamFirstByteTimeoutMs = *cfg.StreamFirstByteTimeoutMs + } }) socks5Mu.Lock() diff --git a/internal/app/main_test.go b/internal/app/main_test.go index 19561c0..5799986 100644 --- a/internal/app/main_test.go +++ b/internal/app/main_test.go @@ -41,10 +41,11 @@ func TestVersionStringFormat(t *testing.T) { } type fakeUpstreamResponse struct { - status int - body string - header http.Header - err error + status int + body string + header http.Header + err error + customBody io.ReadCloser // 非 nil 时优先于 body,用于挂死/慢速流测试 } type fakeRetryTransport struct { @@ -65,7 +66,15 @@ func (f *fakeRetryTransport) RoundTrip(req *http.Request) (*http.Response, error return nil, errors.New("fakeRetryTransport: unexpected bodyless request to " + req.URL.String()) } if len(f.responses) == 0 { - f.t.Fatalf("unexpected request to %s", req.URL.String()) + // 队列为空时返回 502,而不是 fatal:一方面允许上层 fallback 探测 + // (probe / 翻译路径)在测试未显式排队时优雅失败;另一方面不会把 + // 测试主体掐死在天花板上。大多数用例只关心自己排队的槽位。 + return &http.Response{ + StatusCode: http.StatusBadGateway, + Header: make(http.Header), + Body: io.NopCloser(strings.NewReader(`{"type":"error","error":{"message":"fake upstream: no more queued responses"}}`)), + Request: req, + }, nil } body, err := io.ReadAll(req.Body) @@ -91,10 +100,16 @@ func (f *fakeRetryTransport) RoundTrip(req *http.Request) (*http.Response, error if header == nil { header = make(http.Header) } + var respBody io.ReadCloser + if next.customBody != nil { + respBody = next.customBody + } else { + respBody = io.NopCloser(strings.NewReader(next.body)) + } return &http.Response{ StatusCode: next.status, Header: header.Clone(), - Body: io.NopCloser(strings.NewReader(next.body)), + Body: respBody, Request: req, }, nil } diff --git a/internal/config/config.go b/internal/config/config.go index a1a5372..138755e 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -17,6 +17,13 @@ type Snapshot struct { PromptCacheRetention string CacheBreakpoints bool TextOnlyModels []string + // StreamEmptyRetryMax / StreamFirstByteTimeoutMs：claude→responses 流式 + // 链路的空流兜底。上游 200 后首个有效产出（text/thinking/tool delta 或 + // 终态事件）前的窗口里，若遇到空流 EOF / 无数据超时，只重试 + // StreamEmptyRetryMax 次（0 = 关闭），每次重试经 callOpenCodeEndpoint + // 自动落到 key pool 的下一个可用 key。 + StreamEmptyRetryMax int + StreamFirstByteTimeoutMs int } // snapshot holds the current Snapshot. All reads go through Get, which lazily @@ -37,6 +44,10 @@ func Default() Snapshot { PromptCacheRetention: "", // "" -> runtime default "24h"; "off" disables injection CacheBreakpoints: true, TextOnlyModels: []string{}, // models.dev modality data drives the default + // 空流重试默认开（1 次）；首字节超时默认 30s（小于常见上游 / 反代 + // 的 idle kill 阈值，又足够覆盖冷启动 prefill）。 + StreamEmptyRetryMax: 1, + StreamFirstByteTimeoutMs: 30000, } } @@ -98,6 +109,24 @@ func ForceDisableThinking() bool { return Get().ForceDisableThinking } +// StreamEmptyRetryMax reports how many times the claude→responses stream may +// be retried when the upstream returns an empty first-byte window (clean EOF +// or first-byte timeout before any content). 0 disables the retry. +func StreamEmptyRetryMax() int { + if v := Get().StreamEmptyRetryMax; v < 0 { + return 0 + } else { + return v + } +} + +// StreamFirstByteTimeoutMs is the first-byte watchdog for the pre-commit +// peek window, in milliseconds (<= 0 disables the watchdog; only EOF/error +// then aborts the window). +func StreamFirstByteTimeoutMs() int { + return Get().StreamFirstByteTimeoutMs +} + // ReasoningEffortMap returns a copy of the reasoning-effort mapping so callers // cannot mutate the shared snapshot map. func ReasoningEffortMap() map[string]string { diff --git a/internal/domain/types.go b/internal/domain/types.go index 1d54f9f..f38e0f2 100644 --- a/internal/domain/types.go +++ b/internal/domain/types.go @@ -176,6 +176,14 @@ type AppConfig struct { ProtocolRules []ProtocolRule `json:"protocol_rules,omitempty"` // KeyPool configures rotation across multiple upstream API keys. KeyPool KeyPool `json:"key_pool,omitempty"` + // StreamEmptyRetryMax：claude→responses 流式链路在「上游 200 后首个 + // 有效产出前」遇到空流 EOF / 首字节超时时的重试次数上限（0=关闭， + // 缺省 1）。重试经 key pool 自动切到下一个可用 key。 + StreamEmptyRetryMax *int `json:"stream_empty_retry_max,omitempty"` + // StreamFirstByteTimeoutMs：空流检测的首字节看门狗（毫秒，缺省 + // 30000；<=0 关闭看门狗，只靠 EOF）。上游 200 后若一直没有发送任何 + // SSE 数据，超过该阈值视为空流并触发重试。 + StreamFirstByteTimeoutMs *int `json:"stream_first_byte_timeout_ms,omitempty"` } type Socks5Proxy struct { From 5bc06891c8abcca8bde9ec7cd16488e2a3ece78d Mon Sep 17 00:00:00 2001 From: 6Kmfi6HP <179412085+6Kmfi6HP@users.noreply.github.com> Date: Sun, 27 Sep 2026 23:37:38 +0200 Subject: [PATCH 2/9] refactor(stream): extract protocol-agnostic driveStreamWithRetry helper Splits the retry/peek logic out of claude_responses.go so subsequent PRs can plug in the same machinery for chat completions, responses passthrough, and anthropic passthrough without duplicating the driver. The claude responses path keeps byte-identical behavior via thin shims. Co-Authored-By: Claude Code --- internal/app/claude_responses.go | 186 ++++--------------- internal/app/stream_retry.go | 298 +++++++++++++++++++++++++++++++ 2 files changed, 330 insertions(+), 154 deletions(-) create mode 100644 internal/app/stream_retry.go diff --git a/internal/app/claude_responses.go b/internal/app/claude_responses.go index d16964b..b042288 100644 --- a/internal/app/claude_responses.go +++ b/internal/app/claude_responses.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "encoding/json" - "errors" "fmt" "github.com/6Kmfi6HP/opencode2api/internal/config" "github.com/6Kmfi6HP/opencode2api/internal/logging" @@ -1082,160 +1081,39 @@ type claudeResponsesBlock struct { // // 返回与 claudeResponsesStreamHandler 一致：(true, nil) 已 commit / // (false, err) 全部 attempt 都未 commit。ctx.Done 与所有错误原样透传。 +// +// 本函数已是薄封装：实际重试驱动在 driveStreamWithRetry（stream_retry.go）， +// 这里只负责把「首轮复用 firstRC / 重试重新 call OpenCode」的策略封进 +// callOnce 闭包。 func claudeResponsesStreamWithRetry(ctx context.Context, w http.ResponseWriter, auth UpstreamAuth, modelID string, claudeReq ClaudeRequest, wantReasoning bool, firstRC io.ReadCloser) (bool, error) { - maxRetry := config.StreamEmptyRetryMax() - rc := firstRC - for attempt := 0; attempt <= maxRetry; attempt++ { - committed, err := claudeResponsesStreamHandler(ctx, w, rc, modelID, wantReasoning, nil) - if committed { - return true, nil - } - rc.Close() - if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { - return false, err - } - if attempt < maxRetry { - logging.FromContext(ctx).Warn("claude-responses empty stream, retrying with next key", - "model", modelID, "attempt", attempt+1, "max_retry", maxRetry, "cause", err) - nextRC, status, _, callErr := callOpenCodeEndpoint(ctx, "responses", claudeToResponsesBody(claudeReq, modelID), modelID, auth) - if callErr != nil || status < 200 || status >= 300 { - if nextRC != nil { - nextRC.Close() - } - if callErr != nil { - return false, callErr - } - return false, fmt.Errorf("upstream status %d on retry", status) - } - rc = nextRC - continue - } - return false, err - } - return false, errStreamIncompleteNoCommit + // callOnce：首轮返回调用方已打开的 firstRC（status 已由调用方校验， + // 这里给一个 200 占位）；后续 attempt 重新发请求让 key pool 切下一 + // 把可用 key。 + pending := firstRC + callOnce := func(ctx context.Context) (io.ReadCloser, int, error) { + if pending != nil { + rc := pending + pending = nil + return rc, http.StatusOK, nil + } + rc, status, _, err := callOpenCodeEndpoint(ctx, "responses", claudeToResponsesBody(claudeReq, modelID), modelID, auth) + return rc, status, err + } + runOnce := func(ctx context.Context, w http.ResponseWriter, rc io.Reader, _ []streamReadResult, _ *streamReader) (bool, error) { + return claudeResponsesStreamHandler(ctx, w, rc, modelID, wantReasoning, nil) + } + return DriveStreamWithRetry(ctx, w, ResponsesProtocolHooks, callOnce, runOnce) } -// errStreamIncompleteNoCommit：peek 窗口内上游空流 EOF / 首字节超时 / 读 -// 错误 / 上游错误事件——尚未向客户端写过任何字节，调用方可安全重试。 -var errStreamIncompleteNoCommit = errors.New("stream incomplete before first client byte") - -// peekOutcome 是 peekFirstOutput 的结果。consumed 不为空时表示「已有完整 -// SSE 帧被消费」，调用方应把它原样喂回 handler 主循环。 -type peekOutcome struct { - consumed []streamReadResult - err error - // reader 是 peek 内部使用的流式 reader（已经包了一层自己的 bufio 缓 - // 冲）。调用方应把它原样续用做主循环 reader ——否则其内部 bufio 里 - // 已经预读的行会被两个独立 bufio 撕成两半。 - reader *streamReader -} +// peekOutcome 是 peekFirstOutput 的返回类型。保留以兼容既有调用点；新代 +// 码请直接使用 PeekOutcome（stream_retry.go）。 +type peekOutcome = PeekOutcome -// peekFirstOutput 在向上游拿到 200、但还未向客户端 WriteHeader 之前「窥 -// 视」首个**完整 SSE 帧**(空行收尾；流末尾则接受 EOF 收尾)。 -// -// 返回约定： -// - hasFrame=true 且 errorEvent=false:上游已产出有效事件(壳或 -// delta)——commit,handler 进入主循环继续。 -// - hasFrame=true 且 errorEvent=true:上游发了 error / response.failed -// ——按 errStreamIncompleteNoCommit 返回,由调用方走未 commit 重试。 -// - hasFrame=false:窗口内 EOF、读错或超时——同样返回 -// errStreamIncompleteNoCommit(空流,可安全重试)。 -// -// 关键不变量:返回时 consumed 一定落在帧边界(空行或 EOF 行)之后, -// 绝不截断在帧中间——否则 handler 侧的 bufio 续读会把当前帧的剩余行 -// 与后续帧拼错。 +// peekFirstOutput 是 PeekFirstFrame 绑到 ResponsesProtocolHooks 上的薄封装， +// 保留以兼容既有调用点。语义不变：窥视首个完整 SSE 帧,有产出非错误→commit； +// 错误帧 / 空流 / EOF / 超时 → errStreamIncompleteNoCommit。 func peekFirstOutput(ctx context.Context, rc io.Reader, timeout time.Duration) peekOutcome { - reader := newStreamReader(ctx, rc, 0) - // 不要 defer Close:成功路径里调用方会续用这个 reader(它的 bufio - // 里可能已经预读了后续行);只有失败路径在这里显式关闭。 - - var timeoutCh <-chan time.Time - var timer *time.Timer - if timeout > 0 { - timer = time.NewTimer(timeout) - timeoutCh = timer.C - defer timer.Stop() - } - - var consumed []streamReadResult - var frameBuf []string - // hasFrame：已经见过至少一个完整 data 帧（含 ERROR 帧——错误帧也是「有产出」）。 - hasFrame := false - // errorEvent：已见帧里存在 error / response.failed。 - errorEvent := false - - // flushFrame 在当前帧边界（空行或 EOF）结算 frameBuf：判定 hasFrame / - // errorEvent，并原样把整帧追加进 consumed。clearPending 表示丢弃而不是 - // 结算（用于 input：上游事件 payload 里夹的裸 error JSON，不算帧）。 - flushFrame := func() { - for _, dl := range frameBuf { - t := strings.TrimSpace(dl) - if t == "" || t == "[DONE]" { - continue - } - hasFrame = true - if !errorEvent { - var evt map[string]any - if json.Unmarshal([]byte(t), &evt) == nil { - typ, _ := evt["type"].(string) - if typ == "error" || typ == "response.failed" { - errorEvent = true - } else if _, hasErr := evt["error"]; hasErr { - errorEvent = true - } - } - } - } - frameBuf = nil - } - - for { - select { - case <-ctx.Done(): - return peekOutcome{err: ctx.Err()} - case <-ctx.Done(): - reader.Close() - return peekOutcome{err: ctx.Err()} - case <-timeoutCh: - reader.Close() - return peekOutcome{err: errStreamIncompleteNoCommit} - case res := <-reader.Read(): - consumed = append(consumed, res) - trimmed := strings.TrimSpace(res.line) - switch { - case trimmed == "": - flushFrame() - case strings.HasPrefix(trimmed, ":"): - // SSE 心跳注释：不算产出。 - case strings.HasPrefix(trimmed, "data:"): - frameBuf = append(frameBuf, strings.TrimSpace(strings.TrimPrefix(trimmed, "data:"))) - case strings.HasPrefix(trimmed, "event:"): - // 仅记录属于哪个事件；data 到帧尾才结算。 - case strings.HasPrefix(trimmed, "{"): - // 非标准裸 JSON 行：直接当一帧。 - frameBuf = append(frameBuf, trimmed) - flushFrame() - } - if res.err != nil { - // EOF：流自然终止。当前若有残帧,先按帧结算;随后无论 - // 是否有产出都关闭——截在 EOF 处 consumed 已对齐帧边界。 - flushFrame() - if !hasFrame || errorEvent { - // 失败路径:把 reader 也关掉,免得泄漏 goroutine。 - reader.Close() - return peekOutcome{consumed: consumed, err: errStreamIncompleteNoCommit} - } - return peekOutcome{consumed: consumed, reader: reader} - } - if hasFrame { - if errorEvent { - reader.Close() - return peekOutcome{consumed: consumed, err: errStreamIncompleteNoCommit} - } - return peekOutcome{consumed: consumed, reader: reader} - } - } - } + return PeekFirstFrame(ctx, rc, timeout, ResponsesProtocolHooks) } // claudeResponsesStreamHandler 把上游 Responses SSE 翻译为 Claude SSE。 @@ -1252,11 +1130,11 @@ func claudeResponsesStreamHandler(ctx context.Context, w http.ResponseWriter, rc var reader *streamReader if len(peeked) == 0 { peek := peekFirstOutput(ctx, rc, time.Duration(config.StreamFirstByteTimeoutMs())*time.Millisecond) - if peek.err != nil { - return false, peek.err + if peek.Err != nil { + return false, peek.Err } - peeked = peek.consumed - reader = peek.reader + peeked = peek.Consumed + reader = peek.Reader if reader == nil { // 上游 EOF 但已有完整帧（极少见：单帧流）。续读的 reader 直接 // 落在已 EOF 的 rc 上,主循环立即收 EOF 并走 finalize。 diff --git a/internal/app/stream_retry.go b/internal/app/stream_retry.go new file mode 100644 index 0000000..3e10515 --- /dev/null +++ b/internal/app/stream_retry.go @@ -0,0 +1,298 @@ +package app + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "strings" + "time" + + "github.com/6Kmfi6HP/opencode2api/internal/config" + "github.com/6Kmfi6HP/opencode2api/internal/logging" +) + +// StreamProtocolHooks lets each protocol customize error/productive detection +// without changing the retry driver. +type StreamProtocolHooks struct { + // IsErrorEvent reports whether a complete SSE frame payload represents an + // upstream error (as opposed to a productive event). + IsErrorEvent func(payload []byte) bool + // IsProductiveEvent reports whether a complete SSE frame payload represents + // a productive (non-error) event. + IsProductiveEvent func(payload []byte) bool +} + +// responsesIsErrorEvent reports whether a Responses-protocol SSE frame is an +// error: type=error / type=response.failed, or any frame carrying a +// top-level "error" key. +func responsesIsErrorEvent(payload []byte) bool { + var evt map[string]any + if json.Unmarshal(payload, &evt) != nil { + return false + } + typ, _ := evt["type"].(string) + if typ == "error" || typ == "response.failed" { + return true + } + _, hasErr := evt["error"] + return hasErr +} + +// responsesIsProductiveEvent reports whether a Responses-protocol SSE frame +// carries meaningful content ([DONE]/whitespace do not count). +func responsesIsProductiveEvent(payload []byte) bool { + t := strings.TrimSpace(string(payload)) + if t == "" || t == "[DONE]" { + return false + } + return !responsesIsErrorEvent(payload) +} + +// chatIsErrorEvent reports whether a chat-completion SSE frame is an error +// (top-level "error" key on the JSON payload). +func chatIsErrorEvent(payload []byte) bool { + var evt map[string]any + if json.Unmarshal(payload, &evt) != nil { + return false + } + _, hasErr := evt["error"] + return hasErr +} + +// chatIsProductiveEvent reports whether a chat-completion SSE frame carries +// meaningful content ([DONE]/whitespace do not count). +func chatIsProductiveEvent(payload []byte) bool { + t := strings.TrimSpace(string(payload)) + if t == "" || t == "[DONE]" { + return false + } + return !chatIsErrorEvent(payload) +} + +// anthropicIsErrorEvent reports whether an Anthropic-SSE frame is an error +// (type=error). +func anthropicIsErrorEvent(payload []byte) bool { + var evt map[string]any + if json.Unmarshal(payload, &evt) != nil { + return false + } + typ, _ := evt["type"].(string) + return typ == "error" +} + +// anthropicIsProductiveEvent reports whether an Anthropic-SSE frame carries +// meaningful content ([DONE]/whitespace do not count). +func anthropicIsProductiveEvent(payload []byte) bool { + t := strings.TrimSpace(string(payload)) + if t == "" || t == "[DONE]" { + return false + } + return !anthropicIsErrorEvent(payload) +} + +var ( + // ResponsesProtocolHooks implements the Responses-protocol peek logic: + // a frame is an error when its JSON payload carries type=error / + // type=response.failed or a top-level "error" key; anything else with + // non-empty content counts as productive. + ResponsesProtocolHooks = StreamProtocolHooks{ + IsErrorEvent: responsesIsErrorEvent, + IsProductiveEvent: responsesIsProductiveEvent, + } + + // ChatProtocolHooks implements chat-completion chunk detection: + // error when the frame carries a top-level "error" key; [DONE] and + // whitespace-only frames are non-productive; anything else is productive. + ChatProtocolHooks = StreamProtocolHooks{ + IsErrorEvent: chatIsErrorEvent, + IsProductiveEvent: chatIsProductiveEvent, + } + + // AnthropicProtocolHooks implements Anthropic SSE detection: + // error when the frame carries type=error; productive otherwise. + AnthropicProtocolHooks = StreamProtocolHooks{ + IsErrorEvent: anthropicIsErrorEvent, + IsProductiveEvent: anthropicIsProductiveEvent, + } +) + +// errStreamIncompleteNoCommit：peek 窗口内上游空流 EOF / 首字节超时 / 读 +// 错误 / 上游错误事件——尚未向客户端写过任何字节，调用方可安全重试。 +var errStreamIncompleteNoCommit = errors.New("stream incomplete before first client byte") + +// PeekOutcome 是 PeekFirstFrame 的结果。Consumed 不为空时表示「已有完整 +// SSE 帧被消费」，调用方应把它原样喂回 handler 主循环。 +type PeekOutcome struct { + Consumed []streamReadResult + Err error + // Reader 是 peek 内部使用的流式 reader（已经包了一层自己的 bufio 缓 + // 冲）。调用方应把它原样续用做主循环 reader ——否则其内部 bufio 里 + // 已经预读的行会被两个独立 bufio 撕成两半。 + Reader *streamReader +} + +// PeekFirstFrame 在向上游拿到 200、但还未向客户端 WriteHeader 之前「窥 +// 视」首个**完整 SSE 帧**(空行收尾；流末尾则接受 EOF 收尾)。 +// +// 返回约定： +// - 已见完整产出帧且非错误：Reader 续用,Consumed 应回放给主循环—— +// commit。 +// - 已见完整错误帧：Err=errStreamIncompleteNoCommit,reader 已关闭—— +// 由调用方走未 commit 重试。 +// - 窗口内 EOF / 读错 / 超时且无产出：同上,Err=errStreamIncompleteNoCommit。 +// +// 关键不变量:返回时 Consumed 一定落在帧边界(空行或 EOF 行)之后, +// 绝不截断在帧中间——否则 handler 侧的 bufio 续读会把当前帧的剩余行 +// 与后续帧拼错。 +func PeekFirstFrame(ctx context.Context, rc io.Reader, timeout time.Duration, hooks StreamProtocolHooks) PeekOutcome { + reader := newStreamReader(ctx, rc, 0) + // 不要 defer Close:成功路径里调用方会续用这个 reader(它的 bufio + // 里可能已经预读了后续行);只有失败路径在这里显式关闭。 + + var timeoutCh <-chan time.Time + var timer *time.Timer + if timeout > 0 { + timer = time.NewTimer(timeout) + timeoutCh = timer.C + defer timer.Stop() + } + + var consumed []streamReadResult + var frameBuf []string + // hasFrame：已经见过至少一个完整 data 帧（含 ERROR 帧——错误帧也是「有产出」）。 + hasFrame := false + // errorEvent：已见帧里存在错误事件。 + errorEvent := false + + // flushFrame 在当前帧边界（空行或 EOF）结算 frameBuf：判定 hasFrame / + // errorEvent，并原样把整帧追加进 consumed。 + flushFrame := func() { + for _, dl := range frameBuf { + t := strings.TrimSpace(dl) + if t == "" || t == "[DONE]" { + continue + } + hasFrame = true + if !errorEvent && hooks.IsErrorEvent != nil && hooks.IsErrorEvent([]byte(t)) { + errorEvent = true + } + } + frameBuf = nil + } + + for { + select { + case <-ctx.Done(): + return PeekOutcome{Err: ctx.Err()} + case <-timeoutCh: + reader.Close() + return PeekOutcome{Err: errStreamIncompleteNoCommit} + case res := <-reader.Read(): + consumed = append(consumed, res) + trimmed := strings.TrimSpace(res.line) + switch { + case trimmed == "": + flushFrame() + case strings.HasPrefix(trimmed, ":"): + // SSE 心跳注释：不算产出。 + case strings.HasPrefix(trimmed, "data:"): + frameBuf = append(frameBuf, strings.TrimSpace(strings.TrimPrefix(trimmed, "data:"))) + case strings.HasPrefix(trimmed, "event:"): + // 仅记录属于哪个事件；data 到帧尾才结算。 + case strings.HasPrefix(trimmed, "{"): + // 非标准裸 JSON 行：直接当一帧。 + frameBuf = append(frameBuf, trimmed) + flushFrame() + } + if res.err != nil { + // EOF：流自然终止。当前若有残帧,先按帧结算;随后无论 + // 是否有产出都关闭——截在 EOF 处 consumed 已对齐帧边界。 + flushFrame() + if !hasFrame || errorEvent { + // 失败路径:把 reader 也关掉,免得泄漏 goroutine。 + reader.Close() + return PeekOutcome{Consumed: consumed, Err: errStreamIncompleteNoCommit} + } + return PeekOutcome{Consumed: consumed, Reader: reader} + } + if hasFrame { + if errorEvent { + reader.Close() + return PeekOutcome{Consumed: consumed, Err: errStreamIncompleteNoCommit} + } + return PeekOutcome{Consumed: consumed, Reader: reader} + } + } + } +} + +// DriveStreamWithRetry 负责把「peek 失败的可重试错误」翻译为最多 +// StreamEmptyRetryMax 次重试。每次 attempt（含首轮）都通过 callOnce 拿到 +// 一个已建立的流；调用方可以在 callOnce 里把「首轮复用上层已打开的 +// firstRC、后续重试再发新请求」的策略封装进去——这样 driver 不需要感知 +// first-call 与 retry 的差异。 +// +// runOnce 是协议特有的「peek + 主循环」：返回 (true, nil) 表示已 commit； +// 返回 (false, err) 且 err 非 ctx.Canceled/DeadlineExceeded 时，若还 +// 有重试额度则换 key 重试。peeked/rd 当前总是 nil——保留这两个形参是 +// 为了让后续 protocol PR 能把「peek 与 handler 解耦」时不用改本签名。 +// +// 返回约定与 runOnce 一致：(true, nil) 已 commit / (false, err) 全部 +// attempt 都未 commit。ctx.Done 与所有错误原样透传。 +func DriveStreamWithRetry( + ctx context.Context, + w http.ResponseWriter, + hooks StreamProtocolHooks, + callOnce func(ctx context.Context) (io.ReadCloser, int, error), + runOnce func(ctx context.Context, w http.ResponseWriter, rc io.Reader, peeked []streamReadResult, rd *streamReader) (bool, error), +) (bool, error) { + _ = hooks // hooks 当前由 runOnce 内部的 peek 使用;本参数预留以便后续 protocol PR 不改本签名。 + maxRetry := config.StreamEmptyRetryMax() + rc, status, err := callOnce(ctx) + if err != nil { + if rc != nil { + rc.Close() + } + return false, err + } + if status < 200 || status >= 300 { + if rc != nil { + rc.Close() + } + return false, fmt.Errorf("upstream status %d on retry", status) + } + + for attempt := 0; attempt <= maxRetry; attempt++ { + committed, runErr := runOnce(ctx, w, rc, nil, nil) + if committed { + return true, nil + } + if rc != nil { + rc.Close() + } + if errors.Is(runErr, context.Canceled) || errors.Is(runErr, context.DeadlineExceeded) { + return false, runErr + } + if attempt < maxRetry { + logging.FromContext(ctx).Warn("stream empty before commit, retrying with next key", + "attempt", attempt+1, "max_retry", maxRetry, "cause", runErr) + nextRC, status, callErr := callOnce(ctx) + if callErr != nil || status < 200 || status >= 300 { + if nextRC != nil { + nextRC.Close() + } + if callErr != nil { + return false, callErr + } + return false, fmt.Errorf("upstream status %d on retry", status) + } + rc = nextRC + continue + } + return false, runErr + } + return false, errStreamIncompleteNoCommit +} From dfb4aabb4e7bb3674fdb1387f33926d33647874a Mon Sep 17 00:00:00 2001 From: 6Kmfi6HP <179412085+6Kmfi6HP@users.noreply.github.com> Date: Mon, 28 Sep 2026 00:00:45 +0200 Subject: [PATCH 3/9] feat(chat): stream-empty retry on /v1/chat/completions direct path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit chat.go stream branch now goes through DriveStreamWithRetry with ChatProtocolHooks — empty EOF / first-byte timeout / upstream error frames observed before the first client write trigger a key-pool retry instead of being silently forwarded. The pre-retry stream loop body has been extracted into chatStreamRunOnce; both peek and main loop share the same streamReader (with deferred keepalive enabled after peek commit) so that the first-byte watchdog and the SSE keepalive ticker do not fight each other. Defer WriteHeader(http.StatusOK) until the first frame is actually written ("commit point"), so retry (= WriteHeader + JSON error) is still possible after peek reports an empty/error upstream. Also fix rawSSEReader.Close to release the source via sync.Once before acquiring r.mu — previously a watchdog-fired Close could deadlock with an in-flight Read that held the mutex while blocked inside ReadString. Tests (chat_stream_retry_test.go) mirror the claude→responses suite: empty-EOF retry once / exhausts / disabled, partial-EOF emits upstream_truncated error frame without retry, error frame triggers retry, silent upstream timeouts, hung upstream first-byte watchdog. Co-Authored-By: Claude Code --- internal/app/chat.go | 370 ++++++++++++++++++------- internal/app/chat_stream_retry_test.go | 219 +++++++++++++++ internal/app/raw_tool_calls.go | 22 +- 3 files changed, 511 insertions(+), 100 deletions(-) create mode 100644 internal/app/chat_stream_retry_test.go diff --git a/internal/app/chat.go b/internal/app/chat.go index 055c1e4..c150790 100644 --- a/internal/app/chat.go +++ b/internal/app/chat.go @@ -1,12 +1,12 @@ package app import ( - "bufio" "bytes" "context" "crypto/sha256" "encoding/hex" "encoding/json" + "errors" "fmt" "io" "log/slog" @@ -570,103 +570,34 @@ func chatCompletionsHandler(w http.ResponseWriter, r *http.Request) { upstreamBody := buildUpstreamBody(&req) if req.Stream { - upResp, status, _, err := callOpenCodeAPIStream(r.Context(), upstreamBody, req.Model, auth) - if err != nil || status < 200 || status >= 300 { - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(status) - if upResp != nil { - errBody, _ := io.ReadAll(upResp) - if len(errBody) > 0 { - w.Write(errBody) - return - } - } - json.NewEncoder(w).Encode(map[string]any{"error": map[string]any{"message": "upstream error", "type": "upstream_error"}}) - return - } - defer upResp.Close() - w.Header().Set("Content-Type", "text/event-stream") - w.Header().Set("Cache-Control", "no-cache") - w.Header().Set("Connection", "keep-alive") - w.WriteHeader(http.StatusOK) - reader := bufio.NewReader(upResp) - stats := &logging.StreamStats{Start: time.Now()} - doneSeen := false - // sendDone 幂等补发 [DONE]：正常路径上游会自带；上游提前断流（EOF - // 而未发 DONE）时由这里兜底，保证客户端总能收到终止标记。 - sendDone := func() { - if doneSeen { - return - } - doneSeen = true - stats.DoneSeen = true - w.Write([]byte("data: [DONE]\n\n")) - if f, ok := w.(http.Flusher); ok { - f.Flush() - } - } - for { - line, err := reader.ReadString('\n') - if err != nil { - if err == io.EOF { - sendDone() - break - } - logging.FromContext(r.Context()).Error("stream read error", "error", err) - // 发送错误事件通知客户端 - w.Write([]byte("data: {\"error\":\"stream read error\"}\n\n")) - sendDone() - stats.Log(r.Context(), "chat") + ctx := r.Context() + committed, driveErr := DriveStreamWithRetry(ctx, w, ChatProtocolHooks, + func(c context.Context) (io.ReadCloser, int, error) { + rc, status, _, err := callOpenCodeAPIStream(c, upstreamBody, req.Model, auth) + return rc, status, err + }, + func(c context.Context, w http.ResponseWriter, rc io.Reader, peeked []streamReadResult, rd *streamReader) (bool, error) { + return chatStreamRunOnce(c, w, rc, peeked, rd, &req, keepReasoning, clientWantsUsage) + }) + // 走完重试仍未 commit：上游错误 / 重试额度用尽——统一以 JSON + // 形式写给客户端（未 WriteHeader,还是非流式响应）。 + if !committed { + if driveErr != nil && errors.Is(driveErr, context.Canceled) { return } - if doneSeen { - // [DONE] 已发（上游自带或兜底），后续仅腾空缓冲区。 - continue - } - trimmed := strings.TrimSpace(line) - if trimmed == "data: [DONE]" { - sendDone() - continue - } - - if strings.HasPrefix(line, "data: ") { - var raw map[string]any - if json.Unmarshal([]byte(line[6:]), &raw) == nil { - if choices, ok := raw["choices"].([]any); ok && len(choices) > 0 { - if choice, ok := choices[0].(map[string]any); ok { - if delta, ok := choice["delta"].(map[string]any); ok { - stats.ObserveDelta(delta, keepReasoning) - } - if fr, ok := choice["finish_reason"].(string); ok && fr != "" { - stats.FinishReason = fr - stats.SawFinish = true - } - } - } - } - } - - out, usage := convertStreamChunkWithUsage(line, keepReasoning, clientWantsUsage) - if out == "" { - // 空choices chunk，但可能有 usage - if usage != nil { - statsx.RecordChatUsage(req.Model, usage) + msg := "upstream stream incomplete" + if driveErr != nil { + if errors.Is(driveErr, context.DeadlineExceeded) { + msg = "upstream stream timeout" + } else { + msg = msg + ": " + driveErr.Error() } - continue - } - - // 提取 usage（已在 convertStreamChunkWithUsage 中解析） - if usage != nil && !doneSeen { - statsx.RecordChatUsage(req.Model, usage) - } - - w.Write([]byte(out)) - w.Write([]byte("\n")) - if f, ok := w.(http.Flusher); ok { - f.Flush() } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusBadGateway) + json.NewEncoder(w).Encode(map[string]any{"error": map[string]any{"message": msg, "type": "upstream_error"}}) + return } - stats.Log(r.Context(), "chat") return } @@ -720,6 +651,259 @@ func chatCompletionsHandler(w http.ResponseWriter, r *http.Request) { w.Write(outBody) } +// chatStreamRunOnce 是 chat 直连流式分支的单轮执行体,与 DriveStreamWithRetry +// 配套实现「首字节前空流/错误帧可换 key 重试」。 +// +// 返回约定: +// - (true, nil):已向客户端写过至少一个字节(WriteHeader + 一条 SSE 帧), +// 即已 commit。此后即使上游再出错也仅向客户端发 error 帧收尾,不再重试。 +// - (false, err):peek 窗口内未见任何产出帧(空流 EOF / 错误帧 / 首字节超 +// 时 / 读错误)。错误通常是 errStreamIncompleteNoCommit,Drive 会换 key +// 重试。ctx.Canceled/DeadlineExceeded 原样透传不再重试。 +// +// peeked 非空时直接进入主循环(此调用已是某次 peek-commit 之后的干跑); +// 否则对 rc 做一次 PeekFirstFrame(由它先看门狗 / 吞错误帧 / 摘出首个产 +// 出帧)。WriteHeader(200) 只发生在「第一帧要真正写给客户端时」——这就 +// 是 commit 点。 +func chatStreamRunOnce( + ctx context.Context, + w http.ResponseWriter, + rc io.Reader, + peeked []streamReadResult, + rd *streamReader, + req *OpenAIRequest, + keepReasoning bool, + clientWantsUsage bool, +) (bool, error) { + // 初始化 reader:reuse rd(其 bufio 已包含 peek 预读),否则在 rc 上新建。 + var reader *streamReader + if len(peeked) == 0 { + peek := PeekFirstFrame(ctx, rc, time.Duration(config.StreamFirstByteTimeoutMs())*time.Millisecond, ChatProtocolHooks) + if peek.Err != nil { + return false, peek.Err + } + peeked = peek.Consumed + reader = peek.Reader + if reader == nil { + // 上游 EOF 但已有完整帧(极少见:单帧流)。续读 reader 落 + // 在已 EOF 的 rc 上,主循环立即收 EOF 走兜底 DONE。 + reader = newStreamReader(ctx, rc, 15*time.Second) + } else { + // peek 阶段 keepalive=0(由 timeout 当首字节看门狗);commit + // 后开启 15s 心跳,在长间隙里客户端不至于挂住。 + reader.enableKeepalive(15 * time.Second) + } + } else if rd != nil { + reader = rd + reader.enableKeepalive(15 * time.Second) + } else { + reader = newStreamReader(ctx, rc, 15*time.Second) + } + defer reader.Close() + + flusher, _ := w.(http.Flusher) + stats := &logging.StreamStats{Start: time.Now()} + doneSeen := false + // wroteHeader 标记「是否已向客户端写过任何字节」——即 commit 点。一旦 + // 写过就不可逆,后续错误只能发错误帧收尾,不可重试。 + wroteHeader := false + + // emitLine 把一段完整帧 payload 写给客户端,首次调用时先 WriteHeader(200) + // + 写 SSE 响应头,然后才写实数据。它是「commit」的唯一入口。 + emitLine := func(payload string) { + if !wroteHeader { + w.Header().Set("Content-Type", "text/event-stream") + w.Header().Set("Cache-Control", "no-cache") + w.Header().Set("Connection", "keep-alive") + w.WriteHeader(http.StatusOK) + wroteHeader = true + } + w.Write([]byte(payload)) + if flusher != nil { + flusher.Flush() + } + } + // sendDone 幂等补发 [DONE]:正常末尾上游会自带;EOF 兜底也会调它一 + // 次,保证客户端总能收到终止标记。 + sendDone := func() { + if doneSeen { + return + } + doneSeen = true + stats.DoneSeen = true + emitLine("data: [DONE]\n\n") + } + // emitError 在已 commit 的流上补一个错误帧 + [DONE]。客户端拿到完整 + // 的(带错的)终止序列。 + emitError := func(msg string) { + payload := map[string]any{ + "error": map[string]any{ + "message": msg, + "type": "upstream_truncated", + }, + "choices": []any{map[string]any{"index": 0, "finish_reason": "error"}}, + } + data, _ := json.Marshal(payload) + emitLine("data: " + string(data) + "\n\n") + sendDone() + } + + // pending 是 peek 阶段回放给主循环的行(含 peek 收的那批首帧数据),按 + // 序消费直到清空,然后转去读 reader.Read() 直读上游。 + pending := append([]streamReadResult(nil), peeked...) + popPending := func() (streamReadResult, bool) { + if len(pending) == 0 { + return streamReadResult{}, false + } + out := pending[0] + pending = pending[1:] + return out, true + } + + for { + var res streamReadResult + // 优先回放 pending;空了再去 select reader / keepalive / ctx.Done。 + if r, ok := popPending(); ok { + res = r + } else { + select { + case <-ctx.Done(): + // 客户端断开也属于「未 commit 的中断」——已写过头就仅退出, + // 未写过则返回 err 让上层理解。 + if wroteHeader { + stats.Log(ctx, "chat") + return true, nil + } + return false, ctx.Err() + case <-reader.Keepalive(): + // 15s 无新数据时主动发个心跳行,既避免客户端和中间层断连, + // 也由 emitLine 承担首次 WriteHeader(意味着「commit 点 + // 可能由一次心跳触发」——这与 claude_responses 行为一致)。 + if wroteHeader { + emitLine(": keepalive\n\n") + } + continue + case res = <-reader.Read(): + } + } + + if res.err != nil { + // 读到错(含 EOF):先结算手头这一行(可能是末尾不带 \n 的 + // 残行);再按「是否已 commit / 是否已见产出」决定 sendDone 兜底 + // 还是返回可重试错误。 + if trimmed := strings.TrimSpace(res.line); trimmed != "" { + // 残帧也算一次处理尝试:走到下面统一行处理。 + res.err = nil // 清掉让下方逻辑把残行当完整行处理 + if handleChatStreamLine(res.line, req, keepReasoning, clientWantsUsage, stats, &doneSeen, emitLine) { + // 残行内含 [DONE]:正常收尾 + stats.Log(ctx, "chat") + return true, nil + } + } + if res.err == nil || errors.Is(res.err, io.EOF) { + // EOF 分支:已 commit → emit error 帧 + [DONE] 收尾(spec: + // 上游提前断流时——即使已写过正文——也要让客户端知道这 + // 不是干净的 finish_reason=stop);未 commit → 返回 + // errStreamIncompleteNoCommit 让 Drive 重试。 + if wroteHeader { + emitError("upstream stream ended prematurely") + stats.Log(ctx, "chat") + return true, nil + } + return false, errStreamIncompleteNoCommit + } + // 非 EOF 读错(网络断等):同上,只是顺带记一条日志。 + logging.FromContext(ctx).Error("stream read error", "error", res.err) + if wroteHeader { + emitError("upstream stream ended prematurely") + stats.Log(ctx, "chat") + return true, nil + } + return false, errStreamIncompleteNoCommit + } + + if doneSeen { + // [DONE] 已发过(上游自带或我们兜底),后续仅排空缓冲。 + continue + } + + trimmed := strings.TrimSpace(res.line) + if trimmed == "" { + continue + } + // 处理一行:返回 true 表示这一行刚好是 [DONE],流应正常收尾。 + if handleChatStreamLine(res.line, req, keepReasoning, clientWantsUsage, stats, &doneSeen, emitLine) { + stats.Log(ctx, "chat") + return true, nil + } + // 检测到 payload 是顶层错误帧且尚未 commit:让上层可重试。 + if !wroteHeader && strings.HasPrefix(trimmed, "data:") { + payload := strings.TrimSpace(strings.TrimPrefix(trimmed, "data:")) + if payload != "" && payload != "[DONE]" && chatIsErrorEvent([]byte(payload)) { + return false, errStreamIncompleteNoCommit + } + } + } +} + +// handleChatStreamLine 处理一行业务数据(不含 keepalive / EOF 分支),返回 +// true 表示这一帧是 [DONE](流天然收尾),false 表示还需继续。已经经过 +// convertStreamChunkWithUsage 的改写,逐条原样 emit 给客户端。 +// +// 注意:emit 通过闭包完成,首次 emit 时由它去 WriteHeader(200) —— 即 +// 「commit 点」在第一次实际写出时发生,而不是 handler 入口。 +func handleChatStreamLine( + line string, + req *OpenAIRequest, + keepReasoning bool, + clientWantsUsage bool, + stats *logging.StreamStats, + doneSeen *bool, + emitLine func(string), +) bool { + trimmed := strings.TrimSpace(line) + if trimmed == "data: [DONE]" { + // 上游自带 DONE:落到 sendDone 等价路径(只发一次,通过 *doneSeen 幂等)。 + if !*doneSeen { + *doneSeen = true + stats.DoneSeen = true + emitLine("data: [DONE]\n\n") + } + return true + } + + if strings.HasPrefix(line, "data: ") { + var raw map[string]any + if json.Unmarshal([]byte(line[6:]), &raw) == nil { + if choices, ok := raw["choices"].([]any); ok && len(choices) > 0 { + if choice, ok := choices[0].(map[string]any); ok { + if delta, ok := choice["delta"].(map[string]any); ok { + stats.ObserveDelta(delta, keepReasoning) + } + if fr, ok := choice["finish_reason"].(string); ok && fr != "" { + stats.FinishReason = fr + stats.SawFinish = true + } + } + } + } + } + + out, usage := convertStreamChunkWithUsage(line, keepReasoning, clientWantsUsage) + if out == "" { + // 空 choices chunk,但可能有 usage。 + if usage != nil { + statsx.RecordChatUsage(req.Model, usage) + } + return false + } + if usage != nil && !*doneSeen { + statsx.RecordChatUsage(req.Model, usage) + } + emitLine(out + "\n") + return false +} + // ======================== Models Handler ======================== func listModelsHandler(w http.ResponseWriter, r *http.Request) { diff --git a/internal/app/chat_stream_retry_test.go b/internal/app/chat_stream_retry_test.go new file mode 100644 index 0000000..0eb9eb0 --- /dev/null +++ b/internal/app/chat_stream_retry_test.go @@ -0,0 +1,219 @@ +package app + +// chat 直连(/v1/chat/completions)链路的「空流兜底 + 首 token 前重试」端到 +// 端回归。镜像 claude_responses_empty_retry_test.go,针对的是 chat.go 走 +// DriveStreamWithRetry 的重构。 +// +// 覆盖三档故障形态(对照 docs/CONFIGURATION.md 的 stream_empty_retry_max): +// 1. 空流 EOF(整流 EOF、零产出)→ 重试。 +// 2. 首字节超时(StreamFirstByteTimeoutMs 内无任何 SSE 行)。 +// 3. 上游返回 200 后直接发顶层 error 帧(本应回 502 的伪 2xx)。 +// 4. 已有部分产出后 EOF——已 commit,向客户端发一个 error 帧并收尾, +// 不再重试。 +// 5. 重试关闭(retryMax=0)时首轮空流直接落 502。 + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +// chatHealthySSE 是一条完整正常结束的 chat-completion SSE 流(2 行 delta +// + [DONE])。 +const chatHealthySSE = `data: {"id":"chatcmpl_x","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"role":"assistant","content":"hel"}}]}` + "\n\n" + + `data: {"id":"chatcmpl_x","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"content":"lo"},"finish_reason":null}]}` + "\n\n" + + `data: {"id":"chatcmpl_x","object":"chat.completion.chunk","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}` + "\n\n" + + "data: [DONE]\n\n" + +// chatPartialSSE 是「半截」流:送出一个 delta 后 EOF,没有 finish_reason +// 也没有 [DONE]。 +const chatPartialSSE = `data: {"id":"chatcmpl_x","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"role":"assistant","content":"partial answer"}}]}` + "\n\n" + +// chatErrorSSE 是「首帧即顶层错误」的伪 200 流(应当触发重试而不是把错 +// 误帧甩给客户端)。 +const chatErrorSSE = `data: {"error":{"message":"upstream internal","type":"server_error"}}` + "\n\n" + +// chatSilentHeartbeatBody 仅包含 SSE 注释行(心跳)。peek 会消费完这些注 +// 释后在 EOF 判定「无产出」,应触发重试。 +const chatSilentHeartbeatBody = ": keepalive\n: keepalive\n: keepalive\n" + +func TestChatStream_EmptyEOF_RetriesOnce(t *testing.T) { + stubRetryConfig(t, 1, 5000) + + installFakeOpenCodeClient(t, []fakeUpstreamResponse{ + {status: http.StatusOK, body: ``, header: http.Header{"Content-Type": []string{"text/event-stream"}}}, + {status: http.StatusOK, body: chatHealthySSE, header: http.Header{"Content-Type": []string{"text/event-stream"}}}, + }) + + req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", + strings.NewReader(`{"model":"fallback-model-free","stream":true,"messages":[{"role":"user","content":"hi"}]}`)) + rec := httptest.NewRecorder() + chatCompletionsHandler(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200, body=%s", rec.Code, rec.Body.String()) + } + body := rec.Body.String() + if !strings.Contains(body, "hel") || !strings.Contains(body, "lo") { + t.Fatalf("expected text delta from retry, got:\n%s", body) + } + if !strings.Contains(body, "[DONE]") { + t.Fatalf("expected [DONE] sentinel, got:\n%s", body) + } + // 客户端不应看到任何 error 帧——首轮的空流应当被默默重试。 + if strings.Contains(body, `"error"`) { + t.Fatalf("client must not see retry-induced error frame:\n%s", body) + } +} + +func TestChatStream_EmptyEOF_ExhaustsRetryThenErrors(t *testing.T) { + stubRetryConfig(t, 1, 5000) + + installFakeOpenCodeClient(t, []fakeUpstreamResponse{ + {status: http.StatusOK, body: ``, header: http.Header{"Content-Type": []string{"text/event-stream"}}}, + {status: http.StatusOK, body: ``, header: http.Header{"Content-Type": []string{"text/event-stream"}}}, + }) + + req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", + strings.NewReader(`{"model":"fallback-model-free","stream":true,"messages":[{"role":"user","content":"hi"}]}`)) + rec := httptest.NewRecorder() + chatCompletionsHandler(rec, req) + + // 一直未 commit,handler 走「drive 失败」分支,应回 502 JSON。 + if rec.Code != http.StatusBadGateway { + t.Fatalf("status = %d, want 502 (retries exhausted), body=%s", rec.Code, rec.Body.String()) + } + if !strings.Contains(rec.Body.String(), "upstream") { + t.Fatalf("expected upstream error JSON, got:\n%s", rec.Body.String()) + } +} + +func TestChatStream_EmptyEOF_RetryDisabled(t *testing.T) { + stubRetryConfig(t, 0, 5000) + + installFakeOpenCodeClient(t, []fakeUpstreamResponse{ + {status: http.StatusOK, body: ``, header: http.Header{"Content-Type": []string{"text/event-stream"}}}, + }) + + req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", + strings.NewReader(`{"model":"fallback-model-free","stream":true,"messages":[{"role":"user","content":"hi"}]}`)) + rec := httptest.NewRecorder() + chatCompletionsHandler(rec, req) + + if rec.Code != http.StatusBadGateway { + t.Fatalf("status = %d, want 502 (no retry), body=%s", rec.Code, rec.Body.String()) + } +} + +func TestChatStream_PartialEOF_EmitsErrorFrameNoRetry(t *testing.T) { + stubRetryConfig(t, 1, 5000) + + installFakeOpenCodeClient(t, []fakeUpstreamResponse{ + {status: http.StatusOK, body: chatPartialSSE, header: http.Header{"Content-Type": []string{"text/event-stream"}}}, + // 即便留了「如果重试就用这条」的槽位,也绝不应被发出——上游已经 + // 写过一个 delta,handler 已 commit。 + {status: http.StatusOK, body: chatHealthySSE, header: http.Header{"Content-Type": []string{"text/event-stream"}}}, + }) + + req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", + strings.NewReader(`{"model":"fallback-model-free","stream":true,"messages":[{"role":"user","content":"hi"}]}`)) + rec := httptest.NewRecorder() + chatCompletionsHandler(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200, body=%s", rec.Code, rec.Body.String()) + } + body := rec.Body.String() + // 半截正文必须送达。 + if !strings.Contains(body, "partial answer") { + t.Fatalf("expected partial text to reach client:\n%s", body) + } + // 上游提前 EOF:必须兜底发 [DONE](不能让客户端悬挂)。 + if !strings.Contains(body, "[DONE]") { + t.Fatalf("expected [DONE] after partial EOF:\n%s", body) + } + // 上游提前 EOF:按 spec 应附加 upstream_truncated 错误帧。 + if !strings.Contains(body, "upstream_truncated") { + t.Fatalf("expected upstream_truncated error frame after partial EOF:\n%s", body) + } + // 不应触发重试——重试槽位的 hello 不应出现。 + if strings.Contains(body, "hello") { + t.Fatalf("retry slot must not have been consumed (already-committed stream):\n%s", body) + } +} + +func TestChatStream_ErrorFrame_Retries(t *testing.T) { + stubRetryConfig(t, 1, 5000) + + installFakeOpenCodeClient(t, []fakeUpstreamResponse{ + {status: http.StatusOK, body: chatErrorSSE, header: http.Header{"Content-Type": []string{"text/event-stream"}}}, + {status: http.StatusOK, body: chatHealthySSE, header: http.Header{"Content-Type": []string{"text/event-stream"}}}, + }) + + req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", + strings.NewReader(`{"model":"fallback-model-free","stream":true,"messages":[{"role":"user","content":"hi"}]}`)) + rec := httptest.NewRecorder() + chatCompletionsHandler(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200, body=%s", rec.Code, rec.Body.String()) + } + body := rec.Body.String() + if !strings.Contains(body, "hel") { + t.Fatalf("expected text from retry, got:\n%s", body) + } + // 上游首帧的顶层错误不应直接漏给客户端(已被 peek 吞掉并触发重试)。 + if strings.Contains(body, "upstream internal") { + t.Fatalf("upstream error must not leak to client when retry succeeded:\n%s", body) + } +} + +func TestChatStream_SilentUpstream_TimesOut(t *testing.T) { + stubRetryConfig(t, 1, 200) // 200ms 首字节超时 + + // 上游 body 里只有 SSE 注释行(心跳),没有任何 data: 帧,然后 EOF。 + // peek 会消费完这些注释,在 EOF 处判定「无产出」并触发重试。 + installFakeOpenCodeClient(t, []fakeUpstreamResponse{ + {status: http.StatusOK, body: chatSilentHeartbeatBody, header: http.Header{"Content-Type": []string{"text/event-stream"}}}, + {status: http.StatusOK, body: chatHealthySSE, header: http.Header{"Content-Type": []string{"text/event-stream"}}}, + }) + + req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", + strings.NewReader(`{"model":"fallback-model-free","stream":true,"messages":[{"role":"user","content":"hi"}]}`)) + rec := httptest.NewRecorder() + chatCompletionsHandler(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200, body=%s", rec.Code, rec.Body.String()) + } + if !strings.Contains(rec.Body.String(), "hel") { + t.Fatalf("expected retry to recover stream after silent-upstream:\n%s", rec.Body.String()) + } +} + +func TestChatStream_HungUpstream_FirstByteWatchdogFires(t *testing.T) { + if testing.Short() { + t.Skip("skipping hung-upstream test in -short mode") + } + stubRetryConfig(t, 1, 200) // 200ms 首字节超时 + + installFakeOpenCodeClient(t, []fakeUpstreamResponse{ + // 用 slowReader 的 Close-only 行为模拟挂死:Read 永远阻塞,直到 + // Close 才返回 EOF。peek 的 timeout 必须先于它触发。 + {status: http.StatusOK, body: "", header: http.Header{"Content-Type": []string{"text/event-stream"}}, customBody: newSlowReader()}, + {status: http.StatusOK, body: chatHealthySSE, header: http.Header{"Content-Type": []string{"text/event-stream"}}}, + }) + + req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", + strings.NewReader(`{"model":"fallback-model-free","stream":true,"messages":[{"role":"user","content":"hi"}]}`)) + rec := httptest.NewRecorder() + chatCompletionsHandler(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200, body=%s", rec.Code, rec.Body.String()) + } + if !strings.Contains(rec.Body.String(), "hel") { + t.Fatalf("expected retry to recover from hung upstream:\n%s", rec.Body.String()) + } +} diff --git a/internal/app/raw_tool_calls.go b/internal/app/raw_tool_calls.go index 91065de..78b2449 100644 --- a/internal/app/raw_tool_calls.go +++ b/internal/app/raw_tool_calls.go @@ -573,6 +573,8 @@ type rawSSEReader struct { converted bool done bool closed bool + closeOnce sync.Once + closeErr error chunkID string model string created any @@ -582,15 +584,21 @@ func wrapRawSSE(r io.ReadCloser) io.ReadCloser { return &rawSSEReader{src: r, reader: bufio.NewReader(r)} } +// Close 关闭上游源,从而解除任何进行中的 Read 阻塞。 +// +// 注意 Reed–Close 死锁:Read 在持有 r.mu 期间会进 r.reader.ReadString(实 +// 际阻塞在 src.Read)。如果 Close 也要先抢 r.mu,就会被永远无法拿到的锁卡 +// 死——这就是 chat 流路径首字节看门狗触发的真实场景。所以这里**先**用 +// closeOnce 关闭 src(解除 Read 阻塞)**再**通过 once 写 closed 标志; +// 之后的 Read 看到 src 已 EOF,自然走到 r.closed/r.done 的退出路径。 func (r *rawSSEReader) Close() error { - r.mu.Lock() - if r.closed { + r.closeOnce.Do(func() { + r.closeErr = r.src.Close() + r.mu.Lock() + r.closed = true r.mu.Unlock() - return nil - } - r.closed = true - r.mu.Unlock() - return r.src.Close() + }) + return r.closeErr } func (r *rawSSEReader) Read(p []byte) (int, error) { From c7c99820fad259b0e4be0f42ac1ea71e523f5071 Mon Sep 17 00:00:00 2001 From: 6Kmfi6HP <179412085+6Kmfi6HP@users.noreply.github.com> Date: Sun, 27 Sep 2026 23:52:50 +0200 Subject: [PATCH 4/9] feat(chat-via-responses): peek/commit split for native responses upstream responsesSSEToChatStream + forwardChatViaResponses + tests Co-Authored-By: Claude Code --- internal/app/chat_bridge_test.go | 15 +- internal/app/chat_protocol_routing_test.go | 10 +- internal/app/chat_to_responses_upstream.go | 121 +++++++++++- internal/app/chat_via_responses_retry_test.go | 172 ++++++++++++++++++ 4 files changed, 303 insertions(+), 15 deletions(-) create mode 100644 internal/app/chat_via_responses_retry_test.go diff --git a/internal/app/chat_bridge_test.go b/internal/app/chat_bridge_test.go index 15073fb..0d7d0ee 100644 --- a/internal/app/chat_bridge_test.go +++ b/internal/app/chat_bridge_test.go @@ -366,7 +366,10 @@ func TestResponsesToChat_FunctionArgumentsDoneEmitsOnlyRemainder(t *testing.T) { "event: response.function_call_arguments.done\ndata: {\"type\":\"response.function_call_arguments.done\",\"item_id\":\"fc1\",\"arguments\":\"{\\\"q\\\":\\\"beijing\\\"}\"}\n\n" + "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_1\",\"usage\":{\"input_tokens\":4,\"output_tokens\":2,\"total_tokens\":6}}}\n\n" body := drainSSEFromHandler(func(w http.ResponseWriter) { - responsesSSEToChatStream(context.Background(), w, strings.NewReader(sse), "gpt-x", false, true) + committed, err := responsesSSEToChatStream(context.Background(), w, strings.NewReader(sse), "gpt-x", false, true, nil, nil) + if err != nil || !committed { + t.Fatalf("responsesSSEToChatStream = (%v, %v), want (true, nil)", committed, err) + } }) // delta 已发 `{"q":`;done 只补发后缀 `"beijing"}`,不能重复 `{"q":`。 if !strings.Contains(body, `"arguments":"{\"q\":"`) { @@ -390,7 +393,10 @@ func TestResponsesToChat_OutputItemDoneAnnouncesUndeclaredCall(t *testing.T) { "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"type\":\"function_call\",\"id\":\"fc9\",\"call_id\":\"call_9\",\"name\":\"ping\",\"arguments\":\"{}\"}}\n\n" + "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"r\",\"usage\":{\"input_tokens\":1,\"output_tokens\":1,\"total_tokens\":2}}}\n\n" body := drainSSEFromHandler(func(w http.ResponseWriter) { - responsesSSEToChatStream(context.Background(), w, strings.NewReader(sse), "gpt-x", false, false) + committed, err := responsesSSEToChatStream(context.Background(), w, strings.NewReader(sse), "gpt-x", false, false, nil, nil) + if err != nil || !committed { + t.Fatalf("responsesSSEToChatStream = (%v, %v), want (true, nil)", committed, err) + } }) if !strings.Contains(body, `"id":"call_9"`) || !strings.Contains(body, `"name":"ping"`) { t.Fatalf("未补发首 chunk 宣告工具调用: %s", body) @@ -407,7 +413,10 @@ func TestResponsesToChat_FailedThenDoneSentinel(t *testing.T) { sse := "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"r\",\"model\":\"gpt-x\"}}\n\n" + "event: response.failed\ndata: {\"type\":\"response.failed\",\"response\":{\"id\":\"r\",\"error\":{\"message\":\"upstream blew up\"}}}\n\n" body := drainSSEFromHandler(func(w http.ResponseWriter) { - responsesSSEToChatStream(context.Background(), w, strings.NewReader(sse), "gpt-x", false, false) + committed, err := responsesSSEToChatStream(context.Background(), w, strings.NewReader(sse), "gpt-x", false, false, nil, nil) + if err != nil || !committed { + t.Fatalf("responsesSSEToChatStream = (%v, %v), want (true, nil)", committed, err) + } }) if !strings.Contains(body, `"error":{"message":"upstream blew up"}`) { t.Fatalf("缺错误事件: %s", body) diff --git a/internal/app/chat_protocol_routing_test.go b/internal/app/chat_protocol_routing_test.go index c06d2c4..8eedff2 100644 --- a/internal/app/chat_protocol_routing_test.go +++ b/internal/app/chat_protocol_routing_test.go @@ -736,7 +736,10 @@ func TestResponsesSSEToChatStream(t *testing.T) { "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_9\",\"usage\":{\"input_tokens\":8,\"output_tokens\":5,\"total_tokens\":13}}}\n\n" body := drainSSEFromHandler(func(w http.ResponseWriter) { - responsesSSEToChatStream(context.Background(), w, strings.NewReader(sse), "gpt-x", true, true) + committed, err := responsesSSEToChatStream(context.Background(), w, strings.NewReader(sse), "gpt-x", true, true, nil, nil) + if err != nil || !committed { + t.Fatalf("responsesSSEToChatStream = (%v, %v), want (true, nil)", committed, err) + } }) if !strings.Contains(body, `"role":"assistant"`) { @@ -965,7 +968,10 @@ func TestResponsesSSEToChatStream_ToolArgumentsShareIndex(t *testing.T) { "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_7\"}}\n\n" body := drainSSEFromHandler(func(w http.ResponseWriter) { - responsesSSEToChatStream(context.Background(), w, strings.NewReader(sse), "gpt-x", false, false) + committed, err := responsesSSEToChatStream(context.Background(), w, strings.NewReader(sse), "gpt-x", false, false, nil, nil) + if err != nil || !committed { + t.Fatalf("responsesSSEToChatStream = (%v, %v), want (true, nil)", committed, err) + } }) names := map[int]string{} diff --git a/internal/app/chat_to_responses_upstream.go b/internal/app/chat_to_responses_upstream.go index 4ba8c6f..8fd7ba9 100644 --- a/internal/app/chat_to_responses_upstream.go +++ b/internal/app/chat_to_responses_upstream.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "io" "net/http" @@ -256,6 +257,9 @@ func forwardChatViaResponses(w http.ResponseWriter, r *http.Request, auth Upstre log.Info("chat via responses upstream", "model", req.Model, "stream", req.Stream, "keep_reasoning", keepReasoning) if req.Stream { + // 首个请求仍在这里同步发起,以便非 2xx 时把上游错误体原样透传; + // 之后的空流/挂起重试由 DriveStreamWithRetry + ResponsesProtocolHooks + // 通过 callOnce 重新发请求(自动落到 key pool 的下一把可用 key)。 rc, status, _, err := callOpenCodeEndpoint(ctx, "responses", upstreamBody, req.Model, auth) if err != nil || status < 200 || status >= 300 { var errBody []byte @@ -275,8 +279,36 @@ func forwardChatViaResponses(w http.ResponseWriter, r *http.Request, auth Upstre writeUpstreamError(w, status, fmt.Errorf("upstream error"), "chat") return } - defer rc.Close() - responsesSSEToChatStream(ctx, w, rc, req.Model, keepReasoning, true) + // rc 的所有权交给 DriveStreamWithRetry(它在每次 attempt 结束后 + // 负责 Close),这里不能再 defer Close。 + + pending := rc + callOnce := func(ctx context.Context) (io.ReadCloser, int, error) { + if pending != nil { + first := pending + pending = nil + return first, http.StatusOK, nil + } + rc2, status2, _, err2 := callOpenCodeEndpoint(ctx, "responses", upstreamBody, req.Model, auth) + return rc2, status2, err2 + } + runOnce := func(ctx context.Context, w http.ResponseWriter, rc io.Reader, peeked []streamReadResult, rd *streamReader) (bool, error) { + return responsesSSEToChatStream(ctx, w, rc, req.Model, keepReasoning, true, peeked, rd) + } + committed, driveErr := DriveStreamWithRetry(ctx, w, ResponsesProtocolHooks, callOnce, runOnce) + if committed { + return + } + if driveErr != nil && (errors.Is(driveErr, context.Canceled) || errors.Is(driveErr, context.DeadlineExceeded)) { + // 客户端已离开:不要往已断开的连接再写错误。 + return + } + // 全部 attempt 都未 commit(空流 EOF / 上游首帧错误 / 首字节超时 + // 等):此刻还没向客户端写过任何字节,可以安全地写一个 502 JSON。 + if driveErr == nil { + driveErr = errStreamIncompleteNoCommit + } + writeUpstreamError(w, http.StatusBadGateway, fmt.Errorf("upstream error: %v", driveErr), "chat") return } @@ -761,11 +793,29 @@ type responsesToChatState struct { finalized bool // 已写 [DONE]/终态帧（幂等） } -func responsesSSEToChatStream(ctx context.Context, w http.ResponseWriter, rc io.Reader, model string, keepReasoning bool, includeUsage bool) { +// responsesSSEToChatStream 把上游 Responses SSE 翻译为 Chat SSE。 +// +// 返回约定（与 PeekFirstFrame / DriveStreamWithRetry 配合,实现「首 token 前 +// 可重试」）： +// - (true, nil)：已向客户端写过至少一个字节（首块 role chunk 即 commit 边 +// 界)。上游之后 EOF / 故障由 finalize 按既有规则合成 finish + [DONE] +// 收尾,不再触发重试。 +// - (false, err)：未向客户端写过任何字节——peek 窗口里上游给了空流/ +// 错误事件/EOF/超时,或主循环在写出 role 前遇到 EOF。调用方可安全地 +// 关闭当前 rc 并换 key 重试。 +// +// peeked/rd 配套:rd != nil 表示调用方已经完成 peek,这里直接复用其 +// reader 并把 peeked 行回放进主循环;rd == nil 表示由本函数内部先做 +// PeekFirstFrame(ResponsesProtocolHooks)再进主循环。 +// +// 首个非错误产出 chunk 由 emitChunk 隐式触发 http 库 WriteHeader(200);在此 +// 之前不向客户端写任何字节,所以 peek 失败路径调用方看到的是干净的连接。 +func responsesSSEToChatStream(ctx context.Context, w http.ResponseWriter, rc io.Reader, model string, keepReasoning bool, includeUsage bool, peeked []streamReadResult, rd *streamReader) (bool, error) { w.Header().Set("Content-Type", "text/event-stream") w.Header().Set("Cache-Control", "no-cache") w.Header().Set("Connection", "keep-alive") - w.WriteHeader(http.StatusOK) + // 注意:不在这里显式 WriteHeader。第一笔写入(无论是 ensureRole 的 + // role chunk、还是失败路径的 502 JSON)由 http 库隐式提交 header。 flusher, _ := w.(http.Flusher) st := &responsesToChatState{ @@ -789,12 +839,59 @@ func responsesSSEToChatStream(ctx context.Context, w http.ResponseWriter, rc io. st.stats.Log(ctx, "chat") }() - reader := newStreamReader(ctx, rc, 0) + var reader *streamReader + if rd != nil { + // 调用方已完成 peek:续用它的 reader(bufio 里可能已预读后续行), + // 不再二次 peek。 + reader = rd + } else { + peek := PeekFirstFrame(ctx, rc, time.Duration(config.StreamFirstByteTimeoutMs())*time.Millisecond, ResponsesProtocolHooks) + if peek.Err != nil { + // 空流 / 上游首帧错误 / 首字节超时:尚未向客户端写过任何字节, + // 让外层 DriveStreamWithRetry 换 key 重试。 + return false, peek.Err + } + peeked = peek.Consumed + reader = peek.Reader + if reader == nil { + // 极端情况:上游 EOF 之前刚好吐完一帧完整产出。reader 已经关闭, + // 这里在已 EOF 的 rc 上重启一个 reader——Read 立即返回 EOF, + // 主循环走 finalize 收尾。 + reader = newStreamReader(ctx, rc, 15*time.Second) + } else { + // 复用 peek 的 reader;顺手开 keepalive(peek 窗口期它是关的)。 + reader.enableKeepalive(15 * time.Second) + } + } defer reader.Close() + + // 回放 peek 阶段消费的行(帧边界已对齐,不会撕碎)。 + for _, res := range peeked { + if res.line != "" { + st.stats.NoteChunk() + st.handleLine(res.line) + } + } + // peeked 末位如果带 EOF(peek 在 EOF 截断时把 EOF 也计入 consumed), + // 主循环等价于立即收到 EOF——直接在回放后判定,不再进 select。 + if n := len(peeked); n > 0 && peeked[n-1].err != nil { + if !st.sentRole { + return false, errStreamIncompleteNoCommit + } + st.finalize() + return true, nil + } + for { select { case <-ctx.Done(): - return + // 客户端已断开:已 commit 就静默退出,未 commit 把 ctx 错误原样 + // 透传给 DriveStreamWithRetry(它对 ctx.Canceled/DeadlineExceeded + // 不重试)。 + if !st.sentRole { + return false, ctx.Err() + } + return true, nil case result := <-reader.Read(): pendingErr := result.err if result.line != "" { @@ -802,11 +899,15 @@ func responsesSSEToChatStream(ctx context.Context, w http.ResponseWriter, rc io. st.handleLine(result.line) } if pendingErr != nil { - // EOF / 读错误兜底:response.completed 未到达时补终态 - // finish chunk(+usage)+[DONE],保证 OpenAI SDK 不挂起 - // （幂等:已完成路径不受影响）。 + // EOF / 读错误:role 尚未发出说明上游没产出任何内容,返回未 + // commit,让外层 DriveStreamWithRetry 换 key 重试。已 commit + // 时按既有规则合成 finish chunk(+usage)+[DONE] 兜底,保证 + // OpenAI SDK 不挂起（幂等:已完成路径不受影响）。 + if !st.sentRole { + return false, errStreamIncompleteNoCommit + } st.finalize() - return + return true, nil } } } diff --git a/internal/app/chat_via_responses_retry_test.go b/internal/app/chat_via_responses_retry_test.go new file mode 100644 index 0000000..899d2ff --- /dev/null +++ b/internal/app/chat_via_responses_retry_test.go @@ -0,0 +1,172 @@ +package app + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/6Kmfi6HP/opencode2api/internal/domain" +) + +// chatViaResponsesSampleStream 一个常规原生 Responses SSE 流：created + 一段 +// output_text.delta + completed(带 usage)。就绪探针 peek 看到 created 即 +// commit;handler 收到 delta 写出正文,completed 触发 finalize。 +const chatViaResponsesSampleStream = "event: response.created\n" + + "data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_ok\",\"model\":\"test-resp\"}}\n\n" + + "event: response.output_text.delta\n" + + "data: {\"type\":\"response.output_text.delta\",\"delta\":\"hi\"}\n\n" + + "event: response.completed\n" + + "data: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_ok\",\"usage\":{\"input_tokens\":1,\"output_tokens\":1,\"total_tokens\":2}}}\n\n" + +// runChatViaResponsesStream 触发一次 chat 入站 → 原生 responses 上游的流式 +// 转发,返回 recorder 与 transport(便于断言上游调用次数)。 +func runChatViaResponsesStream(t *testing.T, responses []fakeUpstreamResponse) (*httptest.ResponseRecorder, *fakeRetryTransport) { + t.Helper() + setProtocolRulesForTest(t, []domain.ProtocolRule{{Pattern: "test-resp-*", Protocol: "responses"}}) + transport := installFakeOpenCodeClient(t, responses) + req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", + strings.NewReader(`{"model":"test-resp-model","stream":true,"messages":[{"role":"user","content":"hi"}]}`)) + rec := httptest.NewRecorder() + chatCompletionsHandler(rec, req) + return rec, transport +} + +// 空流(200 响应 + 立即 EOF,未产出任何 SSE 帧)应被 DriveStreamWithRetry +// 识别为「未 commit」并换 key 重试一次,第二次产出正常后客户端收到内容。 +func TestChatViaResponsesStream_EmptyEOF_RetriesOnce(t *testing.T) { + rec, transport := runChatViaResponsesStream(t, []fakeUpstreamResponse{ + {status: http.StatusOK, body: ""}, // attempt 1: 空 body → EOF + {status: http.StatusOK, body: chatViaResponsesSampleStream}, // attempt 2: 正常产出 + }) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String()) + } + if got := len(transport.requestedURLs); got != 2 { + t.Fatalf("upstream calls = %d, want 2 (1 empty + 1 retry); urls=%v", got, transport.requestedURLs) + } + body := rec.Body.String() + if !strings.Contains(body, `"role":"assistant"`) { + t.Fatalf("missing role chunk: %s", body) + } + if !strings.Contains(body, `"content":"hi"`) { + t.Fatalf("missing content delta: %s", body) + } + if !strings.Contains(body, "data: [DONE]") { + t.Fatalf("missing [DONE]: %s", body) + } +} + +// 重试额度耗尽（默认 1 次重试 = 共 2 次 attempt）后仍未 commit,网关 +// 应在未向客户端写过任何 SSE 字节的前提下回落 502 JSON。 +func TestChatViaResponsesStream_EmptyEOF_ExhaustsRetryThenErrors(t *testing.T) { + rec, transport := runChatViaResponsesStream(t, []fakeUpstreamResponse{ + {status: http.StatusOK, body: ""}, // attempt 1 + {status: http.StatusOK, body: ""}, // attempt 2 (重试 1 次) + }) + + if rec.Code != http.StatusBadGateway { + t.Fatalf("status = %d, want 502; body=%s", rec.Code, rec.Body.String()) + } + if got := len(transport.requestedURLs); got != 2 { + t.Fatalf("upstream calls = %d, want 2 (initial + 1 retry exhausted); urls=%v", got, transport.requestedURLs) + } + // 必须没有向客户端写过任何 SSE 帧:不能出现部分 SSE body 后又写 JSON 的夹杂。 + body := rec.Body.String() + if strings.Contains(body, "data: ") || strings.Contains(body, "[DONE]") { + t.Fatalf("client must not receive any SSE bytes when commit never happened: %q", body) + } + if ct := rec.Header().Get("Content-Type"); ct != "application/json" { + t.Fatalf("Content-Type = %q, want application/json", ct) + } +} + +// 已 commit(role chunk 已写出)之后上游 EOF 且未完成 response.completed: +// 不触发重试,handler 用 finalize 合成 finish chunk + [DONE] 收尾。 +func TestChatViaResponsesStream_PartialEOF_FinalizesNormallyNoRetry(t *testing.T) { + partialStream := "event: response.created\n" + + "data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_partial\",\"model\":\"test-resp\"}}\n\n" + + "event: response.output_text.delta\n" + + "data: {\"type\":\"response.output_text.delta\",\"delta\":\"partial\"}\n\n" + // 故意不发 response.completed / [DONE],直接 EOF。 + + rec, transport := runChatViaResponsesStream(t, []fakeUpstreamResponse{ + {status: http.StatusOK, body: partialStream}, + }) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String()) + } + if got := len(transport.requestedURLs); got != 1 { + t.Fatalf("upstream calls = %d, want 1 (no retry after commit)", got) + } + body := rec.Body.String() + if !strings.Contains(body, `"content":"partial"`) { + t.Fatalf("missing content delta: %s", body) + } + // finalize 兜底:即便没等到 response.completed,也必须补 finish + [DONE], + // 否则 OpenAI SDK 会挂在缺哨兵的流上。 + if !strings.Contains(body, `"finish_reason":"stop"`) { + t.Fatalf("missing synthesized finish_reason: %s", body) + } + if !strings.Contains(body, "data: [DONE]") { + t.Fatalf("missing [DONE]: %s", body) + } +} + +// 上游首帧就是 response.failed:被 peek 识别为错误事件,返回未 commit, +// 触发重试。重试拿到正常产出后客户端应看到正常内容,看不到第一把 key +// 的错误事件。 +func TestChatViaResponsesStream_ResponseFailed_Retries(t *testing.T) { + failedStream := "event: response.failed\n" + + "data: {\"type\":\"response.failed\",\"response\":{\"id\":\"resp_bad\",\"error\":{\"message\":\"upstream blew up\"}}}\n\n" + + rec, transport := runChatViaResponsesStream(t, []fakeUpstreamResponse{ + {status: http.StatusOK, body: failedStream}, + {status: http.StatusOK, body: chatViaResponsesSampleStream}, + }) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String()) + } + if got := len(transport.requestedURLs); got != 2 { + t.Fatalf("upstream calls = %d, want 2 (failed + retry); urls=%v", got, transport.requestedURLs) + } + body := rec.Body.String() + if !strings.Contains(body, `"content":"hi"`) { + t.Fatalf("missing content from retry attempt: %s", body) + } + // 客户端不应看到第一把 key 的错误事件——它在 peek 阶段被吞掉并整体重试。 + if strings.Contains(body, "upstream blew up") { + t.Fatalf("first-attempt error leaked into client stream: %s", body) + } +} + +// role chunk 已写出(上游发过 response.created)之后才 EOF:说明客户端 +// 已经收到首字节,不能再换 key 重试——必须用现有 partial 状态正常 finalize。 +// 与 PartialEOF 区别:本用例连一个 delta 都没产出,仅 role,验证 sentRole +// 边界本身足以抑制重试。 +func TestChatViaResponsesStream_RoleSentThenEOF_NoRetry(t *testing.T) { + roleOnlyStream := "event: response.created\n" + + "data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_role\",\"model\":\"test-resp\"}}\n\n" + // 没有 output_text.delta,也不发 completed,直接 EOF。 + + rec, transport := runChatViaResponsesStream(t, []fakeUpstreamResponse{ + {status: http.StatusOK, body: roleOnlyStream}, + }) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String()) + } + if got := len(transport.requestedURLs); got != 1 { + t.Fatalf("upstream calls = %d, want 1 (role sent → committed, no retry)", got) + } + body := rec.Body.String() + if !strings.Contains(body, `"role":"assistant"`) { + t.Fatalf("missing role chunk: %s", body) + } + if !strings.Contains(body, "data: [DONE]") { + t.Fatalf("missing [DONE] after finalize: %s", body) + } +} From 4306634800afd198a42a684717f967a389880d0f Mon Sep 17 00:00:00 2001 From: 6Kmfi6HP <179412085+6Kmfi6HP@users.noreply.github.com> Date: Sun, 27 Sep 2026 23:48:42 +0200 Subject: [PATCH 5/9] feat(chat-via-anthropic): peek/commit split for native anthropic upstream anthropicSSEToChatStream + forwardChatViaAnthropic + tests Co-Authored-By: Claude Code --- internal/app/chat_bridge_test.go | 12 +- internal/app/chat_protocol_routing_test.go | 10 +- internal/app/chat_to_anthropic.go | 164 +++++++--- internal/app/chat_via_anthropic_retry_test.go | 293 ++++++++++++++++++ 4 files changed, 427 insertions(+), 52 deletions(-) create mode 100644 internal/app/chat_via_anthropic_retry_test.go diff --git a/internal/app/chat_bridge_test.go b/internal/app/chat_bridge_test.go index 0d7d0ee..4851404 100644 --- a/internal/app/chat_bridge_test.go +++ b/internal/app/chat_bridge_test.go @@ -312,7 +312,7 @@ func TestAnthropicToChat_EOFEmitsFinishAndDone(t *testing.T) { "event: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\"},\"usage\":{\"output_tokens\":5}}\n\n" // 无 message_stop。 body := drainSSEFromHandler(func(w http.ResponseWriter) { - anthropicSSEToChatStream(context.Background(), w, strings.NewReader(sse), "claude-x", false, true) + anthropicSSEToChatStream(context.Background(), w, strings.NewReader(sse), "claude-x", false, true, nil, nil) }) if !strings.Contains(body, `"finish_reason":"stop"`) { t.Fatalf("EOF 后缺 finish chunk: %s", body) @@ -335,7 +335,7 @@ func TestAnthropicToChat_TextStartPreEmitsInitialText(t *testing.T) { "event: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0}\n\n" + "event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n" body := drainSSEFromHandler(func(w http.ResponseWriter) { - anthropicSSEToChatStream(context.Background(), w, strings.NewReader(sse), "claude-x", true, false) + anthropicSSEToChatStream(context.Background(), w, strings.NewReader(sse), "claude-x", true, false, nil, nil) }) if !strings.Contains(body, `"content":"partial-prefix"`) { t.Fatalf("start 初始 text 未预 emit: %s", body) @@ -350,7 +350,7 @@ func TestAnthropicToChat_SignatureAndRedactedAreSkipped(t *testing.T) { "event: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0}\n\n" + "event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n" body := drainSSEFromHandler(func(w http.ResponseWriter) { - anthropicSSEToChatStream(context.Background(), w, strings.NewReader(sse), "claude-x", true, false) + anthropicSSEToChatStream(context.Background(), w, strings.NewReader(sse), "claude-x", true, false, nil, nil) }) if strings.Contains(body, "sig-abc") || strings.Contains(body, "reasoning_content") { t.Fatalf("signature/redacted 不应进入 reasoning_content: %s", body) @@ -498,7 +498,7 @@ func TestAnthropicToChat_NormalStopNotDoubleFinalized(t *testing.T) { sse := "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"id\":\"m\",\"model\":\"claude-x\",\"usage\":{\"input_tokens\":2}}}\n\n" + "event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n" body := drainSSEFromHandler(func(w http.ResponseWriter) { - anthropicSSEToChatStream(context.Background(), w, strings.NewReader(sse), "claude-x", true, true) + anthropicSSEToChatStream(context.Background(), w, strings.NewReader(sse), "claude-x", true, true, nil, nil) }) if cnt := strings.Count(body, "data: [DONE]"); cnt != 1 { t.Fatalf("[DONE] 出现 %d 次, want 1", cnt) @@ -516,13 +516,13 @@ func TestAnthropicToChat_UsageChunkOnlyWhenIncludeUsage(t *testing.T) { sse := "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"id\":\"m\",\"model\":\"claude-x\",\"usage\":{\"input_tokens\":2}}}\n\n" + "event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n" noUsage := drainSSEFromHandler(func(w http.ResponseWriter) { - anthropicSSEToChatStream(context.Background(), w, strings.NewReader(sse), "claude-x", false, false) + anthropicSSEToChatStream(context.Background(), w, strings.NewReader(sse), "claude-x", false, false, nil, nil) }) if strings.Contains(noUsage, `"prompt_tokens"`) { t.Fatalf("includeUsage=false 不应发 usage 终块: %s", noUsage) } withUsage := drainSSEFromHandler(func(w http.ResponseWriter) { - anthropicSSEToChatStream(context.Background(), w, strings.NewReader(sse), "claude-x", false, true) + anthropicSSEToChatStream(context.Background(), w, strings.NewReader(sse), "claude-x", false, true, nil, nil) }) if !strings.Contains(withUsage, `"prompt_tokens":2`) { t.Fatalf("includeUsage=true 应发 usage 终块: %s", withUsage) diff --git a/internal/app/chat_protocol_routing_test.go b/internal/app/chat_protocol_routing_test.go index 8eedff2..64ad94d 100644 --- a/internal/app/chat_protocol_routing_test.go +++ b/internal/app/chat_protocol_routing_test.go @@ -322,7 +322,7 @@ func TestAnthropicSSEToChatStream(t *testing.T) { "event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n" body := drainSSEFromHandler(func(w http.ResponseWriter) { - anthropicSSEToChatStream(context.Background(), w, strings.NewReader(sse), "claude-x", true, true) + anthropicSSEToChatStream(context.Background(), w, strings.NewReader(sse), "claude-x", true, true, nil, nil) }) // role 首块。 @@ -360,7 +360,7 @@ func TestAnthropicSSEToChatStream_ToolUseInitialInputFallback(t *testing.T) { "event: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"tool_use\"}}\n\n" + "event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n" body := drainSSEFromHandler(func(w http.ResponseWriter) { - anthropicSSEToChatStream(context.Background(), w, strings.NewReader(sse), "claude-x", false, false) + anthropicSSEToChatStream(context.Background(), w, strings.NewReader(sse), "claude-x", false, false, nil, nil) }) // 首块:arguments 应为空(不等 initial,避免与后续 delta 拼接)。 // JSON 字段序由 map 序列化决定,这里只断言关键 token 同时存在。 @@ -383,7 +383,7 @@ func TestAnthropicSSEToChatStream_ToolUseInitialInputNotDoubledWhenDeltaArrives( "event: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0}\n\n" + "event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n" body := drainSSEFromHandler(func(w http.ResponseWriter) { - anthropicSSEToChatStream(context.Background(), w, strings.NewReader(sse), "claude-x", false, false) + anthropicSSEToChatStream(context.Background(), w, strings.NewReader(sse), "claude-x", false, false, nil, nil) }) // delta 到达 → initial 必须被抑制;只能看到 delta 碎片,不能出现 initial 内容。 if strings.Contains(body, `{\"k\":\"v\"}`) { @@ -404,7 +404,7 @@ func TestAnthropicSSEToChatStream_ToolUseEmptyInputFallback(t *testing.T) { "event: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0}\n\n" + "event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n" body := drainSSEFromHandler(func(w http.ResponseWriter) { - anthropicSSEToChatStream(context.Background(), w, strings.NewReader(sse), "claude-x", false, false) + anthropicSSEToChatStream(context.Background(), w, strings.NewReader(sse), "claude-x", false, false, nil, nil) }) // start 块仍先流 "arguments":""(不为空 initial 单独 emit),stop 兜底补 "{}"。 if !strings.Contains(body, `"arguments":""`) { @@ -717,7 +717,7 @@ func TestAnthropicSSEToChatStream_DropsReasoningWhenDisabled(t *testing.T) { "event: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\"}}\n\n" + "event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n" body := drainSSEFromHandler(func(w http.ResponseWriter) { - anthropicSSEToChatStream(context.Background(), w, strings.NewReader(sse), "claude-x", false, false) + anthropicSSEToChatStream(context.Background(), w, strings.NewReader(sse), "claude-x", false, false, nil, nil) }) if strings.Contains(body, "secret") { t.Fatalf("reasoning leaked: %s", body) diff --git a/internal/app/chat_to_anthropic.go b/internal/app/chat_to_anthropic.go index e3105ce..5e252fe 100644 --- a/internal/app/chat_to_anthropic.go +++ b/internal/app/chat_to_anthropic.go @@ -3,6 +3,7 @@ package app import ( "context" "encoding/json" + "errors" "fmt" "io" "log/slog" @@ -487,32 +488,26 @@ func forwardChatViaAnthropic(w http.ResponseWriter, r *http.Request, auth Upstre "model", req.Model, "stream", req.Stream, "keep_reasoning", keepReasoning) if req.Stream { - rc, status, _, err := callOpenCodeAnthropicEndpoint(ctx, upstreamBody, req.Model, auth) - if err != nil || status < 200 || status >= 300 { - var errBody []byte - if rc != nil { - errBody, _ = io.ReadAll(io.LimitReader(rc, 64*1024)) - rc.Close() - } - if status < 100 || status >= 600 { - status = http.StatusBadGateway - } - if len(errBody) > 0 { - if ape, ok := parseAnthropicErrorBody(errBody); ok { - writeUpstreamError(w, status, ape, "chat") - return - } - // 上游错误体非 Anthropic 形状：原样透传，保真状态码。 - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(status) - w.Write(errBody) - return - } - writeUpstreamError(w, status, fmt.Errorf("upstream error"), "chat") + // DriveStreamWithRetry 内部首轮调 callOnce 建立流；peek 失败（空流 + // EOF/读错/首字节看门狗）时按 StreamEmptyRetryMax 换 key 重试。 + callOnce := func(callCtx context.Context) (io.ReadCloser, int, error) { + rc, status, _, err := callOpenCodeAnthropicEndpoint(callCtx, upstreamBody, req.Model, auth) + return rc, status, err + } + runOnce := func(runCtx context.Context, rw http.ResponseWriter, rc io.Reader, peeked []streamReadResult, rd *streamReader) (bool, error) { + return anthropicSSEToChatStream(runCtx, rw, rc, req.Model, keepReasoning, true, peeked, rd) + } + committed, streamErr := DriveStreamWithRetry(ctx, w, AnthropicProtocolHooks, callOnce, runOnce) + if committed { + return + } + // 全部 attempt 都未 commit：尚未向客户端写过任何字节,落 502, + // 不伪装半截流。ctx 取消由调用方按客户端断开处理,原样透传。 + if errors.Is(streamErr, context.Canceled) || errors.Is(streamErr, context.DeadlineExceeded) { return } - defer rc.Close() - anthropicSSEToChatStream(ctx, w, rc, req.Model, keepReasoning, true) + log.Warn("chat via anthropic stream empty after retries", "model", req.Model, "err", streamErr) + writeUpstreamError(w, http.StatusBadGateway, fmt.Errorf("upstream stream empty after retries"), "chat") return } @@ -618,11 +613,22 @@ func (t *anthropicToolState) initialArguments() string { } } -func anthropicSSEToChatStream(ctx context.Context, w http.ResponseWriter, rc io.Reader, model string, keepReasoning bool, includeUsage bool) { +// anthropicSSEToChatStream 把上游 Anthropic Messages SSE 翻译为 Chat SSE。 +// +// 返回值的约定（与 PeekFirstFrame 配合，实现「首 token 前可重试」）： +// - (true, nil)：已向客户端写过至少一个字节（含 finish 后的 [DONE]），上游流 +// 正常翻译完毕或按已有规则合成收尾。这是唯一「已 commit」的返回。 +// - (false, err)：未向客户端写过任何字节——peek 窗口里上游给了空流/ +// 错误事件/EOF/超时或 message_start 之前 EOF。调用方可以安全地关闭当前 +// rc、换 key 重发请求。 +// +// peeked 非空时直接进入主循环（此调用已是某次 peek-commit 之后的干跑）， +// 不再二次 peek、不再做首字节看门狗（窗口已在第一次调用里耗尽）。rd 是 +// peek 主循环里续用的 reader；为 nil 时函数自己在 rc 上新建。 +func anthropicSSEToChatStream(ctx context.Context, w http.ResponseWriter, rc io.Reader, model string, keepReasoning bool, includeUsage bool, peeked []streamReadResult, rd *streamReader) (bool, error) { w.Header().Set("Content-Type", "text/event-stream") w.Header().Set("Cache-Control", "no-cache") w.Header().Set("Connection", "keep-alive") - w.WriteHeader(http.StatusOK) flusher, _ := w.(http.Flusher) st := &anthropicToChatState{ @@ -646,27 +652,103 @@ func anthropicSSEToChatStream(ctx context.Context, w http.ResponseWriter, rc io. st.stats.Log(ctx, "chat") }() - reader := newStreamReader(ctx, rc, 0) + // WriteHeader 延迟到首字节真正写出（避免 peek 阶段占用了 200，让那次 + // 空流的调用方可安全重试而不污染客户端）。 + wroteHeader := false + writeHeaderOnce := func() { + if wroteHeader { + return + } + w.WriteHeader(http.StatusOK) + wroteHeader = true + } + + var reader *streamReader + if len(peeked) == 0 { + peek := PeekFirstFrame(ctx, rc, time.Duration(config.StreamFirstByteTimeoutMs())*time.Millisecond, AnthropicProtocolHooks) + if peek.Err != nil { + return false, peek.Err + } + peeked = peek.Consumed + reader = peek.Reader + if reader == nil { + // 上游 EOF 但已见完整帧（极少见：单帧流）。续读的 reader 直接 + // 落在已 EOF 的 rc 上,主循环立即收 EOF 并走 finalize。 + reader = newStreamReader(ctx, rc, 15*time.Second) + } else { + // 复用 peek 的 reader(它的 bufio 可能已预读后续行);顺手开 + // keepalive(此前为 0,看门狗由 timeout 承担)。 + reader.enableKeepalive(15 * time.Second) + } + } else { + // 已有 peeked：此调用即某次 peek-commit 之后的干跑；调用方透传 + // 了自己的 reader（rd）就用，否则在 rc 上新建。 + if rd != nil { + reader = rd + } else { + reader = newStreamReader(ctx, rc, 15*time.Second) + } + } defer reader.Close() + + // flushPending 按序回放 peek 阶段攒下的行（peek 阶段已按 SSE 行结构 + // 验证过帧完整性），随后转为 nil 直读上游。 + pending := append([]streamReadResult(nil), peeked...) + consumePending := func() []streamReadResult { + out := pending + pending = nil + return out + } + + // processResult 处理一行上游 SSE（行可能为空字符串收尾一帧,err 非空 + // 表示当前行已是最后一行）。committed=true 表示已经向客户端写出过至 + // 少一字节。 + processResult := func(result streamReadResult) (committed bool, done bool, retErr error) { + if result.line != "" { + st.stats.NoteChunk() + writeHeaderOnce() + st.handleLine(result.line) + } + if result.err != nil { + // EOF / 读错误兜底。 + if !st.sentRole { + // message_start 都未到达 = 完全空流,从未 commit。 + return false, true, errStreamIncompleteNoCommit + } + // 已有 role 输出后再 EOF:按finalize合成 stop chunk+[DONE], + // 保证 OpenAI SDK 不挂起（幂等）。 + st.finalize() + if st.skippedSig > 0 || st.skippedRedacted > 0 { + slog.Debug("chat stream: dropped unrepresentable anthropic deltas", + "model", model, "signature_delta", st.skippedSig, "redacted_thinking", st.skippedRedacted) + } + return true, true, nil + } + return wroteHeader, false, nil + } + + // 先回放 peeked，再进入主循环。 + for _, res := range consumePending() { + _, done, err := processResult(res) + if done { + if err != nil { + return false, err + } + return true, nil + } + } + for { select { case <-ctx.Done(): - return + return wroteHeader, ctx.Err() case result := <-reader.Read(): - pendingErr := result.err - if result.line != "" { - st.stats.NoteChunk() - st.handleLine(result.line) - } - if pendingErr != nil { - // EOF / 读错误兜底:message_stop 未到达时补 finish+usage 终块 - // +[DONE],保证 OpenAI SDK 不挂起（幂等）。 - st.finalize() - if st.skippedSig > 0 || st.skippedRedacted > 0 { - slog.Debug("chat stream: dropped unrepresentable anthropic deltas", - "model", model, "signature_delta", st.skippedSig, "redacted_thinking", st.skippedRedacted) + _, done, err := processResult(result) + if done { + if err != nil { + return false, err } - return + return true, nil } } } diff --git a/internal/app/chat_via_anthropic_retry_test.go b/internal/app/chat_via_anthropic_retry_test.go new file mode 100644 index 0000000..e7da5b6 --- /dev/null +++ b/internal/app/chat_via_anthropic_retry_test.go @@ -0,0 +1,293 @@ +package app + +// chat→anthropic 流式链路的「空流兜底 + 首 token 前重试」端到端回归。 +// +// 覆盖以下故障形态(对照 docs/CONFIGURATION.md 的 stream_empty_retry_max): +// 1. 空流 EOF(整流 EOF、零产出)→ 双层兜底,首 token 前换 key 重试。 +// 2. 上游 200 后立刻发 error 帧(本应回 502 的活失败的 2xx 形态)→ 同上重试。 +// 3. 首字节看门狗:连接接受后一字节不发,纯挂死→超过 stream_first_byte_timeout_ms +// 兜底换 key 重试。 +// 4. 已有 message_start 之后 EOF(message_start 已 emit 出 role chunk,即 +// 已 commit):按「合成 stop」正常关流,不再重试。 +// 5. message_start 之前 EOF:返回未 commit,走重试。 +// 6. 重试额度用尽:落 502,不伪装半截流。 + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/6Kmfi6HP/opencode2api/internal/domain" +) + +// chatViaAnthropicHealthySSE 是一份「正常完结」的最小 anthropic SSE 流。 +const chatViaAnthropicHealthySSE = "event: message_start\n" + + `data: {"type":"message_start","message":{"id":"msg_ok","model":"go-anthropic-model","role":"assistant","usage":{"input_tokens":3,"output_tokens":1}}}` + "\n\n" + + "event: content_block_start\n" + + `data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}` + "\n\n" + + "event: content_block_delta\n" + + `data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"hello"}}` + "\n\n" + + "event: content_block_stop\n" + + `data: {"type":"content_block_stop","index":0}` + "\n\n" + + "event: message_delta\n" + + `data: {"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"output_tokens":5}}` + "\n\n" + + "event: message_stop\n" + + `data: {"type":"message_stop"}` + "\n\n" + +// stubAnthropicRoute 把 modelPattern(例如 "go-anthropic-*")绑定为 anthropic +// 协议,让 chat 入站走到 forwardChatViaAnthropic。返回清理。 +func stubAnthropicRoute(t *testing.T, pattern string) { + t.Helper() + setProtocolRulesForTest(t, []domain.ProtocolRule{{Pattern: pattern, Protocol: "anthropic"}}) +} + +// drainRecorder 提取 recorder 的所有 SSE 行(便于断言)。 +func drainRecorder(rec *httptest.ResponseRecorder) string { + return rec.Body.String() +} + +// 1. 首轮空流 EOF,重试拿到正常流:客户端只看到一条干净的成功流,不应 +// 出现 error 事件、不应看到前半截残流。 +func TestChatViaAnthropicStream_EmptyEOF_RetriesOnce(t *testing.T) { + stubRetryConfig(t, 1, 5000) + stubAnthropicRoute(t, "go-anthropic-*") + + installFakeOpenCodeClient(t, []fakeUpstreamResponse{ + {status: http.StatusOK, body: ``, header: http.Header{"Content-Type": []string{"text/event-stream"}}}, + {status: http.StatusOK, body: chatViaAnthropicHealthySSE, header: http.Header{"Content-Type": []string{"text/event-stream"}}}, + }) + + req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", + strings.NewReader(`{"model":"go-anthropic-model","stream":true,"messages":[{"role":"user","content":"hi"}]}`)) + rec := httptest.NewRecorder() + chatCompletionsHandler(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200, body=%s", rec.Code, rec.Body.String()) + } + body := drainRecorder(rec) + // 首轮空流不应残留任何字节;客户端只看到重试后的干净流。 + if !strings.Contains(body, `"finish_reason":"stop"`) { + t.Fatalf("expected finish_reason=stop from retry, got:\n%s", body) + } + if !strings.Contains(body, "hello") { + t.Fatalf("expected 'hello' text delta from retry, got:\n%s", body) + } + if !strings.Contains(body, "data: [DONE]") { + t.Fatalf("expected [DONE] terminator, got:\n%s", body) + } + if strings.Contains(body, `"error"`) { + t.Fatalf("client must not see retry-induced error:\n%s", body) + } + // 顶部 role chunk 只能来自重试后的首帧 message_start; 只有一个 role。 + if got := strings.Count(body, `"role":"assistant"`); got != 1 { + t.Fatalf("expected exactly 1 role chunk (from retried stream), got %d:\n%s", got, body) + } +} + +// 2. 重试额度用尽,首轮 + 重试都空流:客户端收到 502,不伪装成功流。 +func TestChatViaAnthropicStream_EmptyEOF_ExhaustsRetryThenErrors(t *testing.T) { + stubRetryConfig(t, 1, 5000) + stubAnthropicRoute(t, "go-anthropic-*") + + installFakeOpenCodeClient(t, []fakeUpstreamResponse{ + {status: http.StatusOK, body: ``, header: http.Header{"Content-Type": []string{"text/event-stream"}}}, + {status: http.StatusOK, body: ``, header: http.Header{"Content-Type": []string{"text/event-stream"}}}, + }) + + req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", + strings.NewReader(`{"model":"go-anthropic-model","stream":true,"messages":[{"role":"user","content":"hi"}]}`)) + rec := httptest.NewRecorder() + chatCompletionsHandler(rec, req) + + if rec.Code != http.StatusBadGateway { + t.Fatalf("status = %d, want 502 (retries exhausted, nothing committed), body=%s", rec.Code, rec.Body.String()) + } + // 失败响应必须是 JSON 形状错误,而不是半截 SSE。 + if ct := rec.Header().Get("Content-Type"); !strings.Contains(ct, "application/json") { + t.Fatalf("failure response must be JSON, got content-type %q; body=%s", ct, rec.Body.String()) + } +} + +// 3. 部分产出后 EOF(已有 role chunk + text delta,但没有 message_stop): +// 按 ParalonCloud Rule 2 不重试,按现有「合成 stop」逻辑正常关流——既 +// 不让 agent 拿半截,也不多达一次重算 input。 +func TestChatViaAnthropicStream_PartialEOF_FinalizesNormallyNoRetry(t *testing.T) { + stubRetryConfig(t, 1, 5000) + stubAnthropicRoute(t, "go-anthropic-*") + + partialSSE := "event: message_start\n" + + `data: {"type":"message_start","message":{"id":"msg_partial","model":"go-anthropic-model","role":"assistant","usage":{"input_tokens":1}}}` + "\n\n" + + "event: content_block_start\n" + + `data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}` + "\n\n" + + "event: content_block_delta\n" + + `data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"partial answer"}}` + "\n\n" + // 没有 message_stop / message_delta ——上游在 text delta 后死掉。 + + installFakeOpenCodeClient(t, []fakeUpstreamResponse{ + {status: http.StatusOK, body: partialSSE, header: http.Header{"Content-Type": []string{"text/event-stream"}}}, + // 即便是当 retry 候选位准备的一条正常流,也绝不应被发出。 + {status: http.StatusOK, body: chatViaAnthropicHealthySSE, header: http.Header{"Content-Type": []string{"text/event-stream"}}}, + }) + + req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", + strings.NewReader(`{"model":"go-anthropic-model","stream":true,"messages":[{"role":"user","content":"hi"}]}`)) + rec := httptest.NewRecorder() + chatCompletionsHandler(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200, body=%s", rec.Code, rec.Body.String()) + } + body := drainRecorder(rec) + // 半截正文必须送达。 + if !strings.Contains(body, "partial answer") { + t.Fatalf("expected partial text to reach client:\n%s", body) + } + // 合成正常收尾,不发 error。 + if !strings.Contains(body, `"finish_reason":"stop"`) { + t.Fatalf("expected synthesized finish_reason=stop, got:\n%s", body) + } + if !strings.Contains(body, "data: [DONE]") { + t.Fatalf("expected [DONE] after synthesized stop, got:\n%s", body) + } + if strings.Contains(body, `"error"`) { + t.Fatalf("must not emit error on partial EOF (already committed):\n%s", body) + } + // 重试不应被触发:fallback 槽位里的正常流不应出现在响应里。 + if strings.Contains(body, "hello") { + t.Fatalf("retry slot must not have been consumed (already-committed stream):\n%s", body) + } +} + +// 4. 上游 200 但首帧就是 type=error:按未 commit 处理,走重试,而不 +// 是把半截 SSE 流甩给客户端。 +func TestChatViaAnthropicStream_ErrorEvent_Retries(t *testing.T) { + stubRetryConfig(t, 1, 5000) + stubAnthropicRoute(t, "go-anthropic-*") + + errorOnlySSE := "event: error\n" + + `data: {"type":"error","error":{"type":"api_error","message":"upstream internal"}}` + "\n\n" + + installFakeOpenCodeClient(t, []fakeUpstreamResponse{ + {status: http.StatusOK, body: errorOnlySSE, header: http.Header{"Content-Type": []string{"text/event-stream"}}}, + {status: http.StatusOK, body: chatViaAnthropicHealthySSE, header: http.Header{"Content-Type": []string{"text/event-stream"}}}, + }) + + req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", + strings.NewReader(`{"model":"go-anthropic-model","stream":true,"messages":[{"role":"user","content":"hi"}]}`)) + rec := httptest.NewRecorder() + chatCompletionsHandler(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200, body=%s", rec.Code, rec.Body.String()) + } + body := drainRecorder(rec) + if !strings.Contains(body, "hello") { + t.Fatalf("expected text from retry, got:\n%s", body) + } + // 上游 error 帧必须被 peek 吞掉并触发重试,不应直接翻译成 client-visible error。 + if strings.Contains(body, "upstream internal") { + t.Fatalf("upstream error must not leak to client when retry succeeded:\n%s", body) + } +} + +// 5. 上游发了 message_start 但随后立刻 EOF,没有任何 content_block。按 +// 「已 commit」处理(已经向客户端写过 role chunk),合成 stop 关闭, +// 不重试。这相当于 topic_1 「Thinking-Only or Empty Post-Start」的 +// message_start-only 边界场景。 +func TestChatViaAnthropicStream_MessageStartThenEOF_NoRetry(t *testing.T) { + stubRetryConfig(t, 1, 5000) + stubAnthropicRoute(t, "go-anthropic-*") + + startOnlySSE := "event: message_start\n" + + `data: {"type":"message_start","message":{"id":"msg_only","model":"go-anthropic-model","role":"assistant","usage":{"input_tokens":2}}}` + "\n\n" + // 没有任何 content_block / message_stop,直接 EOF。 + + installFakeOpenCodeClient(t, []fakeUpstreamResponse{ + {status: http.StatusOK, body: startOnlySSE, header: http.Header{"Content-Type": []string{"text/event-stream"}}}, + {status: http.StatusOK, body: chatViaAnthropicHealthySSE, header: http.Header{"Content-Type": []string{"text/event-stream"}}}, + }) + + req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", + strings.NewReader(`{"model":"go-anthropic-model","stream":true,"messages":[{"role":"user","content":"hi"}]}`)) + rec := httptest.NewRecorder() + chatCompletionsHandler(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200 (committed by role chunk), body=%s", rec.Code, rec.Body.String()) + } + body := drainRecorder(rec) + // 已 commit:message_start 已翻译为 role chunk;synthesize stop+[DONE]。 + if !strings.Contains(body, `"role":"assistant"`) { + t.Fatalf("expected role chunk from message_start:\n%s", body) + } + if !strings.Contains(body, `"finish_reason":"stop"`) { + t.Fatalf("expected synthesized finish_reason=stop after EOF, got:\n%s", body) + } + if !strings.Contains(body, "data: [DONE]") { + t.Fatalf("expected [DONE] after synthesized stop:\n%s", body) + } + // 重试不应被触发。 + if strings.Contains(body, "hello") { + t.Fatalf("retry slot must not have been consumed (already-committed by message_start):\n%s", body) + } +} + +// 6. 完全死锁(连接接受后一字节都不发,纯挂在 socket 上)——由 +// stream_first_byte_timeout_ms 兜出,不应傻等到外层 ctx 超时。 +func TestChatViaAnthropicStream_HungUpstream_TimesOut(t *testing.T) { + if testing.Short() { + t.Skip("skipping hung-upstream test in -short mode") + } + stubRetryConfig(t, 1, 200) // 200ms 首字节超时 + stubAnthropicRoute(t, "go-anthropic-*") + + installFakeOpenCodeClient(t, []fakeUpstreamResponse{ + // 用 slowReader 的 Close-only 行为模拟挂死:Read 永远阻塞,直 + // 到 Close 才返回 EOF。peek 的 timeout 必须先于它触发。 + {status: http.StatusOK, body: "", header: http.Header{"Content-Type": []string{"text/event-stream"}}, customBody: newSlowReader()}, + {status: http.StatusOK, body: chatViaAnthropicHealthySSE, header: http.Header{"Content-Type": []string{"text/event-stream"}}}, + }) + + req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", + strings.NewReader(`{"model":"go-anthropic-model","stream":true,"messages":[{"role":"user","content":"hi"}]}`)) + rec := httptest.NewRecorder() + + start := time.Now() + chatCompletionsHandler(rec, req) + elapsed := time.Since(start) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200 (retry succeeded after watchdog), body=%s (took %s)", rec.Code, rec.Body.String(), elapsed) + } + if !strings.Contains(rec.Body.String(), "hello") { + t.Fatalf("expected retry to recover from hung upstream (took %s):\n%s", elapsed, rec.Body.String()) + } + // 看门狗应当在远小于默认值的时间内触发(默认 5s,我们设了 200ms)。 + if elapsed > 5*time.Second { + t.Fatalf("watchdog should have fired well within 5s, took %s", elapsed) + } +} + +// 7. 单元级:anthropicSSEToChatStream 在 sentRole=false 时 EOF,应返回未 +// commit + errStreamIncompleteNoCommit(供 DriveStreamWithRetry 重试)。 +func TestAnthropicSSEToChatStream_NoRoleEOF_ReturnsNoCommit(t *testing.T) { + // 一个空 data 帧都没有的上游流(连 message_start 都没发)。 + sse := "" + rec := httptest.NewRecorder() + committed, err := anthropicSSEToChatStream(context.Background(), rec, io.NopCloser(strings.NewReader(sse)), "go-anthropic-model", false, true, nil, nil) + if committed { + t.Fatalf("expected not-committed on empty upstream before any message_start") + } + if err == nil { + t.Fatalf("expected errStreamIncompleteNoCommit on empty upstream before any message_start") + } + // 客户端不应有任何字节。 + if rec.Body.Len() != 0 { + t.Fatalf("no bytes should have been written to client, got %q", rec.Body.String()) + } +} From c3509b3249f47fb63b93924ddff6b9db78cec1f2 Mon Sep 17 00:00:00 2001 From: 6Kmfi6HP <179412085+6Kmfi6HP@users.noreply.github.com> Date: Mon, 28 Sep 2026 00:19:48 +0200 Subject: [PATCH 6/9] feat(responses): ensureCreated lazy WriteHeader + retry responsesStreamHandler + responsesHandler call site + tests Co-Authored-By: Claude Code --- internal/app/anthropic_decode_test.go | 3 +- internal/app/protocol_regression_test.go | 6 +- internal/app/raw_tool_calls.go | 4 +- internal/app/raw_tool_calls_test.go | 2 +- internal/app/responses.go | 240 ++++++++++++++----- internal/app/responses_handler_retry_test.go | 201 ++++++++++++++++ internal/app/stream_integrity_test.go | 26 +- 7 files changed, 405 insertions(+), 77 deletions(-) create mode 100644 internal/app/responses_handler_retry_test.go diff --git a/internal/app/anthropic_decode_test.go b/internal/app/anthropic_decode_test.go index e9ae464..efa6341 100644 --- a/internal/app/anthropic_decode_test.go +++ b/internal/app/anthropic_decode_test.go @@ -1838,8 +1838,7 @@ func TestResponsesStream_NormalizesUpstreamID(t *testing.T) { ``, }, "\n") rr := httptest.NewRecorder() - resp := &http.Response{StatusCode: 200, Body: io.NopCloser(strings.NewReader(upstream)), Header: make(http.Header)} - responsesStreamHandler(rr, nil, resp, "m", "m", false, nil, nil, ResponsesAPIRequest{}) + responsesStreamHandler(rr, nil, strings.NewReader(upstream), "m", "m", false, nil, nil, ResponsesAPIRequest{}, nil, nil) events := parseSSEEvents(t, rr.Body.String()) for _, e := range events { if e.Name == "response.created" { diff --git a/internal/app/protocol_regression_test.go b/internal/app/protocol_regression_test.go index d012752..afe5d5a 100644 --- a/internal/app/protocol_regression_test.go +++ b/internal/app/protocol_regression_test.go @@ -74,7 +74,7 @@ func TestResponsesStreamLengthEndsIncompleteAndFunctionDoneHasName(t *testing.T) }, "\n") rr := httptest.NewRecorder() resp := &http.Response{StatusCode: 200, Body: io.NopCloser(strings.NewReader(upstream)), Header: make(http.Header)} - responsesStreamHandler(rr, nil, resp, "m", "m", true, nil, nil, ResponsesAPIRequest{}) + responsesStreamHandler(rr, nil, resp.Body, "m", "m", true, nil, nil, ResponsesAPIRequest{}, nil, nil) out := rr.Body.String() if !strings.Contains(out, "event: response.incomplete") || strings.Contains(out, "event: response.completed") { t.Fatalf("wrong terminal event:\n%s", out) @@ -148,7 +148,7 @@ func TestResponsesStreamAllocatesUniqueIndicesWhenToolPrecedesText(t *testing.T) `data: [DONE]`, "", }, "\n") rr := httptest.NewRecorder() - responsesStreamHandler(rr, nil, &http.Response{Body: io.NopCloser(strings.NewReader(upstream))}, "m", "m", false, nil, nil, ResponsesAPIRequest{}) + responsesStreamHandler(rr, nil, io.NopCloser(strings.NewReader(upstream)), "m", "m", false, nil, nil, ResponsesAPIRequest{}, nil, nil) var added []map[string]any for _, block := range strings.Split(rr.Body.String(), "\n\n") { if !strings.HasPrefix(block, "event: response.output_item.added") { @@ -725,7 +725,7 @@ func TestResponsesStreamUsageAddsCachedTokensWhenPromptDetailsLackIt(t *testing. ``, }, "\n") rr := httptest.NewRecorder() - responsesStreamHandler(rr, nil, &http.Response{Body: io.NopCloser(strings.NewReader(upstream))}, "m", "m", false, nil, nil, ResponsesAPIRequest{}) + responsesStreamHandler(rr, nil, io.NopCloser(strings.NewReader(upstream)), "m", "m", false, nil, nil, ResponsesAPIRequest{}, nil, nil) for _, event := range parseSSEEvents(t, rr.Body.String()) { if event.Name != "response.completed" { continue diff --git a/internal/app/raw_tool_calls.go b/internal/app/raw_tool_calls.go index 78b2449..8eff89c 100644 --- a/internal/app/raw_tool_calls.go +++ b/internal/app/raw_tool_calls.go @@ -586,9 +586,9 @@ func wrapRawSSE(r io.ReadCloser) io.ReadCloser { // Close 关闭上游源,从而解除任何进行中的 Read 阻塞。 // -// 注意 Reed–Close 死锁:Read 在持有 r.mu 期间会进 r.reader.ReadString(实 +// 注意 Read–Close 死锁:Read 在持有 r.mu 期间会进 r.reader.ReadString(实 // 际阻塞在 src.Read)。如果 Close 也要先抢 r.mu,就会被永远无法拿到的锁卡 -// 死——这就是 chat 流路径首字节看门狗触发的真实场景。所以这里**先**用 +// 死——这就是 peek 首字节看门狗触发关闭时的真实场景。所以这里**先**用 // closeOnce 关闭 src(解除 Read 阻塞)**再**通过 once 写 closed 标志; // 之后的 Read 看到 src 已 EOF,自然走到 r.closed/r.done 的退出路径。 func (r *rawSSEReader) Close() error { diff --git a/internal/app/raw_tool_calls_test.go b/internal/app/raw_tool_calls_test.go index 2439ef2..5ea5361 100644 --- a/internal/app/raw_tool_calls_test.go +++ b/internal/app/raw_tool_calls_test.go @@ -481,7 +481,7 @@ func TestResponsesStreamHandlerRawDSML(t *testing.T) { }, "\n") rr := httptest.NewRecorder() resp := &http.Response{StatusCode: 200, Body: wrapRawSSE(io.NopCloser(strings.NewReader(upstream))), Header: make(http.Header)} - responsesStreamHandler(rr, nil, resp, "m", "m", false, nil, nil, ResponsesAPIRequest{}) + responsesStreamHandler(rr, nil, resp.Body, "m", "m", false, nil, nil, ResponsesAPIRequest{}, nil, nil) events := parseSSEEvents(t, rr.Body.String()) if !hasEvent(events, "response.function_call_arguments.done") { t.Fatalf("missing function_call done:\n%s", rr.Body.String()) diff --git a/internal/app/responses.go b/internal/app/responses.go index aa7e7b6..3b75fbc 100644 --- a/internal/app/responses.go +++ b/internal/app/responses.go @@ -3,6 +3,7 @@ package app import ( "context" "encoding/json" + "errors" "github.com/6Kmfi6HP/opencode2api/internal/config" "github.com/6Kmfi6HP/opencode2api/internal/logging" statsx "github.com/6Kmfi6HP/opencode2api/internal/stats" @@ -1607,13 +1608,18 @@ func responsesHandler(w http.ResponseWriter, r *http.Request) { // （prompt_cache_key/retention 保留），并记住该模型不再注入。 if rc, st, rerr := retryChatCompletionsWithoutCacheControl(r.Context(), upstreamBody, chatReq.Model, auth, status, transErrBody, true); rc != nil || rerr != nil || st != 0 { if rerr == nil && st >= 200 && st < 300 { - defer rc.Close() - resp := &http.Response{ - StatusCode: st, - Body: rc, - Header: make(http.Header), + // 这条 cache_control 回退路径不再走 DriveStreamWithRetry —— 它本身 + // 已是一次重发;空流兜底由 responsesStreamHandler 的内部 peek 给 + // 出 (false, err),然后我们回落 502 JSON。rc 由 handler 内部 + // reader 关闭,所以这里不能再 defer Close。 + ctx := r.Context() + committed, driveErr := responsesStreamHandler(w, r, rc, chatReq.Model, chatReq.Model, wantReasoning, respReq.Tools, respReq.ToolChoice, respReq, nil, nil) + if !committed && driveErr != nil && !errors.Is(driveErr, context.Canceled) && !errors.Is(driveErr, context.DeadlineExceeded) { + logging.FromContext(ctx).Warn("cache_control fallback stream empty before commit", "err", driveErr) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusBadGateway) + json.NewEncoder(w).Encode(map[string]any{"error": map[string]any{"message": "upstream stream incomplete", "type": "upstream_error"}}) } - responsesStreamHandler(w, r, resp, chatReq.Model, chatReq.Model, wantReasoning, respReq.Tools, respReq.ToolChoice, respReq) return } if rerr != nil { @@ -1638,14 +1644,39 @@ func responsesHandler(w http.ResponseWriter, r *http.Request) { json.NewEncoder(w).Encode(map[string]any{"error": map[string]any{"message": "upstream error"}}) return } - defer upResp.Close() - - resp := &http.Response{ - StatusCode: status, - Body: upResp, - Header: make(http.Header), + // rc 的所有权交给 DriveStreamWithRetry(它在每次 attempt 结束后负责 + // Close),这里不能再 defer Close。 + // + // 上游是 OpenAI chat completions —— 用 ChatProtocolHooks;首个请求已 + // 在这里同步拿到,首轮复用 upResp、后续 attempt 让 callOnce 重发以便 + // key pool 切到下一把可用 key。空流 / EOF / 错误帧 / 首字节超时都会 + // 走到 DriveStreamWithRetry 内部按 errStreamIncompleteNoCommit 重试; + // 全部 attempt 失败时,因为尚未向客户端写过任何字节,可以干净地写 + // 一个 502 JSON。 + ctx := r.Context() + pending := upResp + callOnce := func(c context.Context) (io.ReadCloser, int, error) { + if pending != nil { + rc := pending + pending = nil + return rc, http.StatusOK, nil + } + nrc, nstatus, _, nerr := callOpenCodeAPIStream(c, upstreamBody, chatReq.Model, auth) + return nrc, nstatus, nerr + } + runOnce := func(c context.Context, w http.ResponseWriter, rc io.Reader, peeked []streamReadResult, rd *streamReader) (bool, error) { + return responsesStreamHandler(w, r, rc, chatReq.Model, chatReq.Model, wantReasoning, respReq.Tools, respReq.ToolChoice, respReq, peeked, rd) + } + committed, driveErr := DriveStreamWithRetry(ctx, w, ChatProtocolHooks, callOnce, runOnce) + if !committed && driveErr != nil { + if errors.Is(driveErr, context.Canceled) || errors.Is(driveErr, context.DeadlineExceeded) { + return + } + logging.FromContext(ctx).Warn("responses translate stream exhausted retries", "model", chatReq.Model, "err", driveErr) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusBadGateway) + json.NewEncoder(w).Encode(map[string]any{"error": map[string]any{"message": "upstream stream incomplete", "type": "upstream_error"}}) } - responsesStreamHandler(w, r, resp, chatReq.Model, chatReq.Model, wantReasoning, respReq.Tools, respReq.ToolChoice, respReq) return } @@ -1728,15 +1759,55 @@ func responsesInputTokensDetails(details any) map[string]any { return map[string]any{"cached_tokens": 0} } -func responsesStreamHandler(w http.ResponseWriter, r *http.Request, resp *http.Response, model string, _ string, wantReasoning bool, tools []ResponsesTool, toolChoice any, originalReq ResponsesAPIRequest) { +// responsesStreamHandler 把上游 OpenAI chat-completions SSE 翻译为 +// Responses SSE。返回值的约定(与 PeekFirstFrame / DriveStreamWithRetry +// 配合,实现「首 token 前可重试」): +// - (true, nil):已向客户端写过至少一个字节(SSE 头 + response.created +// 已写出),即已 commit。此后即使上游再出错也仅向客户端发 +// response.failed 收尾,不再重试。 +// - (false, err):未向客户端写过任何字节——peek 窗口里上游给了空流/ +// 错误事件/EOF/超时。调用方可以安全地关闭当前 reader、换 key 重发。 +// +// peeked 非空时,把这些行回放进主循环(此调用已是某次 peek-commit 之后 +// 的干跑),不再二次 peek、不再做首字节看门狗(窗口已在第一次调用里 +// 耗尽);状态(responseID/msgID 等)在本函数顶部已初始化,回放不会再 +// 触发重新随机化——ensureCreated 内部 createdSent 会保证只产生一次。 +func responsesStreamHandler(w http.ResponseWriter, r *http.Request, rc io.Reader, model string, _ string, wantReasoning bool, tools []ResponsesTool, toolChoice any, originalReq ResponsesAPIRequest, peeked []streamReadResult, rd *streamReader) (bool, error) { ctx := context.Background() if r != nil { ctx = r.Context() } - w.Header().Set("Content-Type", "text/event-stream") - w.Header().Set("Cache-Control", "no-cache") - w.Header().Set("Connection", "keep-alive") - w.WriteHeader(http.StatusOK) + + // 初始化 reader:reuse rd(其 bufio 已包含 peek 预读),否则在 rc 上 + // 新建并先做首字节 peek。peek 失败等价于「commit 前空流」,返回 + // (false, errStreamIncompleteNoCommit) 交给外层 Drive 重试。 + var reader *streamReader + if len(peeked) == 0 { + peek := PeekFirstFrame(ctx, rc, time.Duration(config.StreamFirstByteTimeoutMs())*time.Millisecond, ChatProtocolHooks) + if peek.Err != nil { + if peek.Reader == nil { + // peek 内部已经关闭了它的 reader(失败路径),无需再关。 + } + return false, peek.Err + } + peeked = peek.Consumed + reader = peek.Reader + if reader == nil { + // 上游 EOF 但已有完整帧(极少见:单帧流)。续读 reader 落 + // 在已 EOF 的 rc 上,主循环立即收 EOF 走 finalize。 + reader = newStreamReader(ctx, rc, 15*time.Second) + } else { + // 复用 peek 的 reader(它的 bufio 可能已预读后续行);顺手开 + // keepalive(此前为 0,看门狗由 timeout 承担)。 + reader.enableKeepalive(15 * time.Second) + } + } else if rd != nil { + reader = rd + reader.enableKeepalive(15 * time.Second) + } else { + reader = newStreamReader(ctx, rc, 15*time.Second) + } + defer reader.Close() flusher, _ := w.(http.Flusher) stats := &logging.StreamStats{Start: time.Now()} @@ -1772,7 +1843,33 @@ func responsesStreamHandler(w http.ResponseWriter, r *http.Request, resp *http.R reasoningOutputIndex := -1 messageIndex := -1 - reader := newStreamReader(ctx, resp.Body, 0) + // wroteHeader 标记「是否已向客户端写过任何字节」——即 commit 点。一旦 + // 写过就不可逆,后续错误只能发错误帧收尾,不可重试。 + wroteHeader := false + // writeEvent 包装一层惰性 WriteHeader——首次真正写字节时才 WriteHeader(200), + // 这样 peek 失败路径从未碰过 w,客户端能拿到干净的 JSON 错误。 + writeEvent := func(event string, data any) { + if !wroteHeader { + w.Header().Set("Content-Type", "text/event-stream") + w.Header().Set("Cache-Control", "no-cache") + w.Header().Set("Connection", "keep-alive") + w.WriteHeader(http.StatusOK) + wroteHeader = true + } + writeSSEEvent(w, flusher, event, data) + } + + // pending 是 peek 阶段回放给主循环的行(含 peek 收的那批首帧数据),按 + // 序消费直到清空,然后转去读 reader.Read() 直读上游。 + pending := append([]streamReadResult(nil), peeked...) + popPending := func() (streamReadResult, bool) { + if len(pending) == 0 { + return streamReadResult{}, false + } + out := pending[0] + pending = pending[1:] + return out, true + } defer func() { stats.TextChars = len(fullText) @@ -1780,8 +1877,6 @@ func responsesStreamHandler(w http.ResponseWriter, r *http.Request, resp *http.R stats.ToolCallCount = len(toolOrder) stats.Log(ctx, "responses") }() - // Reader cleanup: signal goroutine, unblock any pending read, wait for exit. - defer reader.Close() messageOutputIndex := func() int { if messageIndex < 0 { @@ -1836,7 +1931,7 @@ func responsesStreamHandler(w http.ResponseWriter, r *http.Request, resp *http.R return } seq++ - writeSSEEvent(w, flusher, "response.reasoning_summary_text.done", map[string]any{ + writeEvent("response.reasoning_summary_text.done", map[string]any{ "type": "response.reasoning_summary_text.done", "sequence_number": seq, "item_id": reasoningID, @@ -1845,7 +1940,7 @@ func responsesStreamHandler(w http.ResponseWriter, r *http.Request, resp *http.R "text": fullReasoning, }) seq++ - writeSSEEvent(w, flusher, "response.reasoning_summary_part.done", map[string]any{ + writeEvent("response.reasoning_summary_part.done", map[string]any{ "type": "response.reasoning_summary_part.done", "sequence_number": seq, "item_id": reasoningID, @@ -1854,7 +1949,7 @@ func responsesStreamHandler(w http.ResponseWriter, r *http.Request, resp *http.R "part": map[string]any{"type": "summary_text", "text": fullReasoning}, }) seq++ - writeSSEEvent(w, flusher, "response.output_item.done", map[string]any{ + writeEvent("response.output_item.done", map[string]any{ "type": "response.output_item.done", "sequence_number": seq, "output_index": reasoningOutputIndex, @@ -1869,7 +1964,7 @@ func responsesStreamHandler(w http.ResponseWriter, r *http.Request, resp *http.R } idx := messageOutputIndex() seq++ - writeSSEEvent(w, flusher, "response.output_text.done", map[string]any{ + writeEvent("response.output_text.done", map[string]any{ "type": "response.output_text.done", "sequence_number": seq, "item_id": msgID, @@ -1879,7 +1974,7 @@ func responsesStreamHandler(w http.ResponseWriter, r *http.Request, resp *http.R "logprobs": []any{}, }) seq++ - writeSSEEvent(w, flusher, "response.content_part.done", map[string]any{ + writeEvent("response.content_part.done", map[string]any{ "type": "response.content_part.done", "sequence_number": seq, "item_id": msgID, @@ -1888,7 +1983,7 @@ func responsesStreamHandler(w http.ResponseWriter, r *http.Request, resp *http.R "part": map[string]any{"type": "output_text", "annotations": []any{}, "logprobs": []any{}, "text": fullText}, }) seq++ - writeSSEEvent(w, flusher, "response.output_item.done", map[string]any{ + writeEvent("response.output_item.done", map[string]any{ "type": "response.output_item.done", "sequence_number": seq, "output_index": idx, @@ -1903,7 +1998,7 @@ func responsesStreamHandler(w http.ResponseWriter, r *http.Request, resp *http.R } idx := messageOutputIndex() seq++ - writeSSEEvent(w, flusher, "response.refusal.done", map[string]any{ + writeEvent("response.refusal.done", map[string]any{ "type": "response.refusal.done", "sequence_number": seq, "item_id": msgID, @@ -1923,7 +2018,7 @@ func responsesStreamHandler(w http.ResponseWriter, r *http.Request, resp *http.R name, _ := call["name"].(string) args, _ := call["arguments"].(string) seq++ - writeSSEEvent(w, flusher, "response.function_call_arguments.done", map[string]any{ + writeEvent("response.function_call_arguments.done", map[string]any{ "type": "response.function_call_arguments.done", "sequence_number": seq, "item_id": itemID, @@ -1941,7 +2036,7 @@ func responsesStreamHandler(w http.ResponseWriter, r *http.Request, resp *http.R item["namespace"] = ns } item["status"] = itemStatus - writeSSEEvent(w, flusher, "response.output_item.done", map[string]any{ + writeEvent("response.output_item.done", map[string]any{ "type": "response.output_item.done", "sequence_number": seq, "output_index": idx, @@ -1964,13 +2059,13 @@ func responsesStreamHandler(w http.ResponseWriter, r *http.Request, resp *http.R } } seq++ - writeSSEEvent(w, flusher, "response.created", map[string]any{ + writeEvent("response.created", map[string]any{ "type": "response.created", "sequence_number": seq, "response": map[string]any{"id": responseID, "object": "response", "created_at": createdAt, "status": "in_progress", "background": false, "error": nil, "output": []any{}}, }) seq++ - writeSSEEvent(w, flusher, "response.in_progress", map[string]any{ + writeEvent("response.in_progress", map[string]any{ "type": "response.in_progress", "sequence_number": seq, "response": map[string]any{"id": responseID, "object": "response", "created_at": createdAt, "status": "in_progress"}, @@ -1996,7 +2091,7 @@ func responsesStreamHandler(w http.ResponseWriter, r *http.Request, resp *http.R } applyResponsesRequestEcho(failedResponse, originalReq) seq++ - writeSSEEvent(w, flusher, "response.failed", map[string]any{ + writeEvent("response.failed", map[string]any{ "type": "response.failed", "sequence_number": seq, "response": failedResponse, @@ -2008,11 +2103,22 @@ func responsesStreamHandler(w http.ResponseWriter, r *http.Request, resp *http.R loop: for { - select { - case <-ctx.Done(): - // Client cancelled: quiet exit, no error writes. - return - case result := <-reader.Read(): + var result streamReadResult + // 优先回放 peeked 行;空了再 select reader/ctx.Done。 + if r0, ok := popPending(); ok { + result = r0 + } else { + select { + case <-ctx.Done(): + // Client cancelled: quiet exit, no error writes. + if wroteHeader { + return true, nil + } + return false, ctx.Err() + case result = <-reader.Read(): + } + } + { // bufio.ReadString may return both a non-empty line and an error // (e.g. the last line without a trailing newline + io.EOF). Process // the line first, then handle the accompanying error via pendingErr. @@ -2029,8 +2135,12 @@ loop: finished = true break loop } + if !wroteHeader { + // [DONE] 但从未 commit 过——上游没有给我们任何产出帧。 + return false, errStreamIncompleteNoCommit + } emitResponseFailed("stream ended with [DONE] but no finish_reason") - return + return true, nil } break loop } @@ -2039,8 +2149,11 @@ loop: if strings.TrimSpace(payload) != "" { var chunk map[string]any if err := json.Unmarshal([]byte(payload), &chunk); err != nil { + if !wroteHeader { + return false, errStreamIncompleteNoCommit + } emitResponseFailed("stream received malformed JSON data") - return + return true, nil } else { // In-band error from upstream. if errVal, ok := chunk["error"]; ok && errVal != nil { @@ -2052,8 +2165,11 @@ loop: } else if errStr, ok := errVal.(string); ok && errStr != "" { errMsg = errStr } + if !wroteHeader { + return false, errStreamIncompleteNoCommit + } emitResponseFailed(errMsg) - return + return true, nil } else { stats.NoteChunk() ensureCreated(chunk) @@ -2087,14 +2203,14 @@ loop: if !reasoningStarted { reasoningOutputIndex = indexAllocator.Allocate() seq++ - writeSSEEvent(w, flusher, "response.output_item.added", map[string]any{ + writeEvent("response.output_item.added", map[string]any{ "type": "response.output_item.added", "sequence_number": seq, "output_index": reasoningOutputIndex, "item": reasoningItem("in_progress"), }) seq++ - writeSSEEvent(w, flusher, "response.reasoning_summary_part.added", map[string]any{ + writeEvent("response.reasoning_summary_part.added", map[string]any{ "type": "response.reasoning_summary_part.added", "sequence_number": seq, "item_id": reasoningID, @@ -2106,7 +2222,7 @@ loop: } fullReasoning += rcStr seq++ - writeSSEEvent(w, flusher, "response.reasoning_summary_text.delta", map[string]any{ + writeEvent("response.reasoning_summary_text.delta", map[string]any{ "type": "response.reasoning_summary_text.delta", "sequence_number": seq, "item_id": reasoningID, @@ -2137,14 +2253,14 @@ loop: if !messageStarted { idx := messageOutputIndex() seq++ - writeSSEEvent(w, flusher, "response.output_item.added", map[string]any{ + writeEvent("response.output_item.added", map[string]any{ "type": "response.output_item.added", "sequence_number": seq, "output_index": idx, "item": map[string]any{"id": msgID, "type": "message", "status": "in_progress", "content": []any{}, "role": "assistant"}, }) seq++ - writeSSEEvent(w, flusher, "response.content_part.added", map[string]any{ + writeEvent("response.content_part.added", map[string]any{ "type": "response.content_part.added", "sequence_number": seq, "item_id": msgID, @@ -2156,7 +2272,7 @@ loop: } fullText += contentStr seq++ - writeSSEEvent(w, flusher, "response.output_text.delta", map[string]any{ + writeEvent("response.output_text.delta", map[string]any{ "type": "response.output_text.delta", "sequence_number": seq, "item_id": msgID, @@ -2173,7 +2289,7 @@ loop: } fullRefusal += refusalStr seq++ - writeSSEEvent(w, flusher, "response.refusal.delta", map[string]any{ + writeEvent("response.refusal.delta", map[string]any{ "type": "response.refusal.delta", "sequence_number": seq, "item_id": msgID, @@ -2225,7 +2341,7 @@ loop: if ns != "" { addedItem["namespace"] = ns } - writeSSEEvent(w, flusher, "response.output_item.added", map[string]any{ + writeEvent("response.output_item.added", map[string]any{ "type": "response.output_item.added", "sequence_number": seq, "output_index": outputIndex, @@ -2247,7 +2363,7 @@ loop: if argDelta, _ := fn["arguments"].(string); argDelta != "" { call["arguments"] = call["arguments"].(string) + argDelta seq++ - writeSSEEvent(w, flusher, "response.function_call_arguments.delta", map[string]any{ + writeEvent("response.function_call_arguments.delta", map[string]any{ "type": "response.function_call_arguments.delta", "sequence_number": seq, "item_id": call["item_id"], @@ -2294,14 +2410,24 @@ loop: finished = true break loop } - emitResponseFailed("stream ended without finish_reason") - return + if !wroteHeader { + // EOF 但从未 commit——上游空流/EOF,可重试。 + return false, errStreamIncompleteNoCommit + } + // 已 commit 但 EOF 时上游还未给 finish_reason:合 + // 成一个 response.failed 收尾(reason 标记 upstream_ + // truncated),客户端能拿到完整错误事件。 + emitResponseFailed("upstream_truncated: stream ended without finish_reason") + return true, nil } break loop } logging.FromContext(ctx).Error("stream read error", "error", pendingErr) + if !wroteHeader { + return false, errStreamIncompleteNoCommit + } emitResponseFailed("stream read error") - return + return true, nil } } } @@ -2312,14 +2438,14 @@ loop: if !messageStarted && len(toolCalls) == 0 { idx := messageOutputIndex() seq++ - writeSSEEvent(w, flusher, "response.output_item.added", map[string]any{ + writeEvent("response.output_item.added", map[string]any{ "type": "response.output_item.added", "sequence_number": seq, "output_index": idx, "item": map[string]any{"id": msgID, "type": "message", "status": "in_progress", "content": []any{}, "role": "assistant"}, }) seq++ - writeSSEEvent(w, flusher, "response.content_part.added", map[string]any{ + writeEvent("response.content_part.added", map[string]any{ "type": "response.content_part.added", "sequence_number": seq, "item_id": msgID, @@ -2410,7 +2536,7 @@ loop: } seq++ - writeSSEEvent(w, flusher, terminalEvent, map[string]any{ + writeEvent(terminalEvent, map[string]any{ "type": terminalEvent, "sequence_number": seq, "response": completedResponse, @@ -2420,6 +2546,8 @@ loop: flusher.Flush() } storeResponseState(completedResponse, originalReq) + // 正常 finalize:已经写过 response.created,必然 commit 过。 + return true, nil } func convertChatToResponses(chatBody []byte, model string, wantReasoning bool, tools []ResponsesTool, toolChoice any, include []string) []byte { diff --git a/internal/app/responses_handler_retry_test.go b/internal/app/responses_handler_retry_test.go new file mode 100644 index 0000000..c592258 --- /dev/null +++ b/internal/app/responses_handler_retry_test.go @@ -0,0 +1,201 @@ +package app + +// responses 入站 → 上游 chat-completions 翻译路径的「空流兜底 + 首 token +// 前重试」端到端回归。镜像 chat_stream_retry_test.go 与 +// claude_responses_empty_retry_test.go,针对的是 responses.go 中 +// responsesStreamHandler 接上 DriveStreamWithRetry 后的行为。 +// +// 覆盖五档故障形态(对照 docs/CONFIGURATION.md 的 stream_empty_retry_max / +// stream_first_byte_timeout_ms): +// 1. 空流 EOF(整流 EOF、零产出)→ 重试一次,客户端只看到正常流。 +// 2. 重试额度耗尽——未 commit,客户端收到一个干净的 502 JSON,而不是 +// 半截 SSE 后再切到错误响应。 +// 3. 已 commit(role/created 帧已写出)后 EOF——不再重试,handler 合成 +// response.failed(reason=upstream_truncated)收尾。 +// 4. 上游首帧就是顶层 error 帧——peek 吞掉并触发重试,客户端只看到 +// 重试后的正常流。 +// 5. 上游挂死,首字节超时——Drive 切换到重试,客户端恢复正常产出。 + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +// responsesHandlerHealthySSE 是一条「正常完结」的 chat-completion SSE 流(这 +// 条入站是 /v1/responses,上游是 OpenAI chat completions,经 +// responsesStreamHandler 翻译为 responses SSE)。heartbeat + 两个 delta + +// stop finish_reason + [DONE]。 +const responsesHandlerHealthySSE = ": heartbeat\n\n" + + `data: {"id":"chatcmpl_resp_ok","created":1,"choices":[{"index":0,"delta":{"role":"assistant","content":"hel"}}]}` + "\n\n" + + `data: {"id":"chatcmpl_resp_ok","choices":[{"index":0,"delta":{"content":"lo"},"finish_reason":null}]}` + "\n\n" + + `data: {"id":"chatcmpl_resp_ok","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}` + "\n\n" + + "data: [DONE]\n\n" + +// responsesHandlerPartialSSE 是「半截」流:role+一个 delta 后 EOF,没有 +// finish_reason 也没有 [DONE]。已 commit(response.created + output_item +// 都已写)后,handler 应合成 response.failed 兜底,而不是走外层重试。 +const responsesHandlerPartialSSE = `data: {"id":"chatcmpl_resp_partial","created":1,"choices":[{"index":0,"delta":{"role":"assistant","content":"partial"}}]}` + "\n\n" + +// responsesHandlerErrorSSE 是「首帧就顶层 error」的伪 200 流(应当触发 +// 重试而不是把错误帧甩给客户端)。 +const responsesHandlerErrorSSE = `data: {"error":{"message":"upstream internal","type":"server_error"}}` + "\n\n" + +// responsesHandlerSilentBody 仅含 SSE 注释行,peek 在 EOF 处判「无产出」, +// 应触发首字节超时/EOF 兜底重试。 +const responsesHandlerSilentBody = ": keepalive\n: keepalive\n: keepalive\n" + +// runResponsesHandlerStream 用给定上游响应队列触发一次 /v1/responses 的流式 +// 请求,返回 recorder 与 transport(便于断言上游调用次数与请求模型)。 +func runResponsesHandlerStream(t *testing.T, responses []fakeUpstreamResponse) (*httptest.ResponseRecorder, *fakeRetryTransport) { + t.Helper() + transport := installFakeOpenCodeClient(t, responses) + req := httptest.NewRequest(http.MethodPost, "/v1/responses", + strings.NewReader(`{"model":"fallback-model-free","stream":true,"input":[{"role":"user","content":[{"type":"input_text","text":"hi"}]}]}`)) + rec := httptest.NewRecorder() + responsesHandler(rec, req) + return rec, transport +} + +// 空流(200 + 立即 EOF,零产出)应被 DriveStreamWithRetry 识别为「未 +// commit」并换 key 重试一次;第二次产出正常后客户端只看到干净的成功流。 +func TestResponsesHandler_StreamEmptyEOF_RetriesOnce(t *testing.T) { + stubRetryConfig(t, 1, 5000) + + rec, transport := runResponsesHandlerStream(t, []fakeUpstreamResponse{ + {status: http.StatusOK, body: ``, header: http.Header{"Content-Type": []string{"text/event-stream"}}}, + {status: http.StatusOK, body: responsesHandlerHealthySSE, header: http.Header{"Content-Type": []string{"text/event-stream"}}}, + }) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200, body=%s", rec.Code, rec.Body.String()) + } + if got := len(transport.requestedURLs); got != 2 { + t.Fatalf("upstream calls = %d, want 2 (1 empty + 1 retry); urls=%v", got, transport.requestedURLs) + } + body := rec.Body.String() + if !strings.Contains(body, `"response.created"`) { + t.Fatalf("expected response.created from retry, got:\n%s", body) + } + if !strings.Contains(body, "hel") || !strings.Contains(body, "lo") { + t.Fatalf("expected content delta from retry, got:\n%s", body) + } + // 客户端不应看到「upstream_truncated」等重试引发的错误事件——首轮空流 + // 应被默默吞掉。 + if strings.Contains(body, `"response.failed"`) || strings.Contains(body, "upstream_truncated") { + t.Fatalf("client must not see retry-induced failure event:\n%s", body) + } +} + +// 重试额度耗尽(retryMax=1,共 2 次 attempt)仍未 commit 时,网关应在未 +// 向客户端写过任何 SSE 字节的前提下回落 502 JSON。 +func TestResponsesHandler_StreamEmptyEOF_ExhaustsRetry(t *testing.T) { + stubRetryConfig(t, 1, 5000) + + rec, transport := runResponsesHandlerStream(t, []fakeUpstreamResponse{ + {status: http.StatusOK, body: ``, header: http.Header{"Content-Type": []string{"text/event-stream"}}}, + {status: http.StatusOK, body: ``, header: http.Header{"Content-Type": []string{"text/event-stream"}}}, + }) + + if rec.Code != http.StatusBadGateway { + t.Fatalf("status = %d, want 502 (retries exhausted), body=%s", rec.Code, rec.Body.String()) + } + if got := len(transport.requestedURLs); got != 2 { + t.Fatalf("upstream calls = %d, want 2 (initial + 1 retry exhausted); urls=%v", got, transport.requestedURLs) + } + body := rec.Body.String() + // 必须没有向客户端写过任何 SSE 帧:不能出现部分 SSE body 后又写 JSON 的夹杂。 + if strings.Contains(body, "event: ") || strings.Contains(body, `data:.+"response"`) { + t.Fatalf("client must not receive any SSE bytes when commit never happened: %q", body) + } + if ct := rec.Header().Get("Content-Type"); ct != "application/json" { + t.Fatalf("Content-Type = %q, want application/json", ct) + } +} + +// 已 commit(response.created + 一段 output_text.delta 已写出)后 EOF 且 +// 未完成 finish:不触发重试,handler 必须合成 response.failed(reason= +// upstream_truncated)收尾——responses API 没有 [DONE] 哨兵。 +func TestResponsesHandler_PartialEOF_SynthesizesFailedNoRetry(t *testing.T) { + stubRetryConfig(t, 1, 5000) + + rec, transport := runResponsesHandlerStream(t, []fakeUpstreamResponse{ + {status: http.StatusOK, body: responsesHandlerPartialSSE, header: http.Header{"Content-Type": []string{"text/event-stream"}}}, + // 即便留了「如果重试就用这条」的槽位,也绝不应被发出——已经 commit。 + {status: http.StatusOK, body: responsesHandlerHealthySSE, header: http.Header{"Content-Type": []string{"text/event-stream"}}}, + }) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200, body=%s", rec.Code, rec.Body.String()) + } + if got := len(transport.requestedURLs); got != 1 { + t.Fatalf("upstream calls = %d, want 1 (no retry after commit)", got) + } + body := rec.Body.String() + if !strings.Contains(body, `"response.created"`) { + t.Fatalf("missing response.created (must have committed before EOF):\n%s", body) + } + if !strings.Contains(body, "partial") { + t.Fatalf("missing partial content delta:\n%s", body) + } + // 收尾必须是 response.failed 而不是静默收尾——客户端才能感知半截流。 + if !strings.Contains(body, `"response.failed"`) { + t.Fatalf("missing synthesized response.failed:\n%s", body) + } + if !strings.Contains(body, "upstream_truncated") { + t.Fatalf("expected upstream_truncated reason in response.failed:\n%s", body) + } +} + +// 上游首帧就发顶层 error 帧:被 peek 识别为错误,返回未 commit,触发换 +// key 重试。重试拿到正常产出后客户端只看到正常流,看不到第一把 key 的错 +// 误帧。 +func TestResponsesHandler_ErrorChunk_Retries(t *testing.T) { + stubRetryConfig(t, 1, 5000) + + rec, transport := runResponsesHandlerStream(t, []fakeUpstreamResponse{ + {status: http.StatusOK, body: responsesHandlerErrorSSE, header: http.Header{"Content-Type": []string{"text/event-stream"}}}, + {status: http.StatusOK, body: responsesHandlerHealthySSE, header: http.Header{"Content-Type": []string{"text/event-stream"}}}, + }) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200, body=%s", rec.Code, rec.Body.String()) + } + if got := len(transport.requestedURLs); got != 2 { + t.Fatalf("upstream calls = %d, want 2 (failed + retry); urls=%v", got, transport.requestedURLs) + } + body := rec.Body.String() + if !strings.Contains(body, "hel") { + t.Fatalf("expected retry content, got:\n%s", body) + } + // 客户端不应看到第一把 key 的错误信息——它已被 peek 吞掉并整体重试。 + if strings.Contains(body, "upstream internal") { + t.Fatalf("first-attempt upstream error leaked into client stream:\n%s", body) + } +} + +// 上游挂死(body 永远阻塞 Read):首字节看门狗必须在 timeout 内放弃等 +// 待并触发重试,客户端从第二把 key 拿到正常流。 +func TestResponsesHandler_HungUpstream_TimesOut(t *testing.T) { + if testing.Short() { + t.Skip("skipping hung-upstream test in -short mode") + } + stubRetryConfig(t, 1, 200) // 200ms 首字节超时 + + rec, transport := runResponsesHandlerStream(t, []fakeUpstreamResponse{ + // slowReader 的 Read 永远阻塞,直到 Close 才返回 EOF,模拟挂死上游。 + {status: http.StatusOK, body: "", header: http.Header{"Content-Type": []string{"text/event-stream"}}, customBody: newSlowReader()}, + {status: http.StatusOK, body: responsesHandlerHealthySSE, header: http.Header{"Content-Type": []string{"text/event-stream"}}}, + }) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200, body=%s", rec.Code, rec.Body.String()) + } + if got := len(transport.requestedURLs); got != 2 { + t.Fatalf("upstream calls = %d, want 2 (hung + retry); urls=%v", got, transport.requestedURLs) + } + if !strings.Contains(rec.Body.String(), "hel") { + t.Fatalf("expected retry to recover from hung upstream:\n%s", rec.Body.String()) + } +} diff --git a/internal/app/stream_integrity_test.go b/internal/app/stream_integrity_test.go index c52838a..ff7e5fb 100644 --- a/internal/app/stream_integrity_test.go +++ b/internal/app/stream_integrity_test.go @@ -427,7 +427,7 @@ func TestResponsesStream_UsageOnlyEnd_SynthesizesCompleted(t *testing.T) { }, "\n") rr := httptest.NewRecorder() resp := &http.Response{StatusCode: 200, Body: io.NopCloser(strings.NewReader(upstream)), Header: make(http.Header)} - responsesStreamHandler(rr, nil, resp, "m", "m", false, nil, nil, ResponsesAPIRequest{}) + responsesStreamHandler(rr, nil, resp.Body, "m", "m", false, nil, nil, ResponsesAPIRequest{}, nil, nil) events := parseSSEEvents(t, rr.Body.String()) if hasEvent(events, "response.failed") { t.Fatalf("must not emit response.failed on usage-terminated stream:\n%s", rr.Body.String()) @@ -448,7 +448,7 @@ func TestResponsesStream_DoneNoFinishWithUsage_SynthesizesCompleted(t *testing.T }, "\n") rr := httptest.NewRecorder() resp := &http.Response{StatusCode: 200, Body: io.NopCloser(strings.NewReader(upstream)), Header: make(http.Header)} - responsesStreamHandler(rr, nil, resp, "m", "m", false, nil, nil, ResponsesAPIRequest{}) + responsesStreamHandler(rr, nil, resp.Body, "m", "m", false, nil, nil, ResponsesAPIRequest{}, nil, nil) events := parseSSEEvents(t, rr.Body.String()) if hasEvent(events, "response.failed") { t.Fatalf("must not emit response.failed on [DONE] with usage-terminated stream:\n%s", rr.Body.String()) @@ -465,7 +465,7 @@ func TestResponsesStream_PartialEOF_ResponseFailed(t *testing.T) { }, "\n") rr := httptest.NewRecorder() resp := &http.Response{StatusCode: 200, Body: io.NopCloser(strings.NewReader(upstream)), Header: make(http.Header)} - responsesStreamHandler(rr, nil, resp, "m", "m", false, nil, nil, ResponsesAPIRequest{}) + responsesStreamHandler(rr, nil, resp.Body, "m", "m", false, nil, nil, ResponsesAPIRequest{}, nil, nil) events := parseSSEEvents(t, rr.Body.String()) if !hasEvent(events, "response.failed") { t.Fatalf("expected response.failed on partial EOF:\n%s", rr.Body.String()) @@ -500,7 +500,7 @@ func TestResponsesStream_InBandError_ResponseFailed(t *testing.T) { }, "\n") rr := httptest.NewRecorder() resp := &http.Response{StatusCode: 200, Body: io.NopCloser(strings.NewReader(upstream)), Header: make(http.Header)} - responsesStreamHandler(rr, nil, resp, "m", "m", false, nil, nil, ResponsesAPIRequest{}) + responsesStreamHandler(rr, nil, resp.Body, "m", "m", false, nil, nil, ResponsesAPIRequest{}, nil, nil) events := parseSSEEvents(t, rr.Body.String()) if !hasEvent(events, "response.failed") { t.Fatalf("expected response.failed on in-band error:\n%s", rr.Body.String()) @@ -516,7 +516,7 @@ func TestResponsesStream_ReaderError_ResponseFailed(t *testing.T) { } rr := httptest.NewRecorder() resp := &http.Response{StatusCode: 200, Body: er, Header: make(http.Header)} - responsesStreamHandler(rr, nil, resp, "m", "m", false, nil, nil, ResponsesAPIRequest{}) + responsesStreamHandler(rr, nil, resp.Body, "m", "m", false, nil, nil, ResponsesAPIRequest{}, nil, nil) events := parseSSEEvents(t, rr.Body.String()) if !hasEvent(events, "response.failed") { t.Fatalf("expected response.failed on reader error:\n%s", rr.Body.String()) @@ -535,7 +535,7 @@ func TestResponsesStream_DoneNoFinish_ResponseFailed(t *testing.T) { }, "\n") rr := httptest.NewRecorder() resp := &http.Response{StatusCode: 200, Body: io.NopCloser(strings.NewReader(upstream)), Header: make(http.Header)} - responsesStreamHandler(rr, nil, resp, "m", "m", false, nil, nil, ResponsesAPIRequest{}) + responsesStreamHandler(rr, nil, resp.Body, "m", "m", false, nil, nil, ResponsesAPIRequest{}, nil, nil) events := parseSSEEvents(t, rr.Body.String()) if !hasEvent(events, "response.failed") { t.Fatalf("expected response.failed on [DONE] without finish:\n%s", rr.Body.String()) @@ -554,7 +554,7 @@ func TestResponsesStream_BadJSON_ResponseFailed(t *testing.T) { }, "\n") rr := httptest.NewRecorder() resp := &http.Response{StatusCode: 200, Body: io.NopCloser(strings.NewReader(upstream)), Header: make(http.Header)} - responsesStreamHandler(rr, nil, resp, "m", "m", false, nil, nil, ResponsesAPIRequest{}) + responsesStreamHandler(rr, nil, resp.Body, "m", "m", false, nil, nil, ResponsesAPIRequest{}, nil, nil) events := parseSSEEvents(t, rr.Body.String()) if !hasEvent(events, "response.failed") { t.Fatalf("expected response.failed on bad JSON:\n%s", rr.Body.String()) @@ -575,7 +575,7 @@ func TestResponsesStream_NormalFinish(t *testing.T) { }, "\n") rr := httptest.NewRecorder() resp := &http.Response{StatusCode: 200, Body: io.NopCloser(strings.NewReader(upstream)), Header: make(http.Header)} - responsesStreamHandler(rr, nil, resp, "m", "m", false, nil, nil, ResponsesAPIRequest{}) + responsesStreamHandler(rr, nil, resp.Body, "m", "m", false, nil, nil, ResponsesAPIRequest{}, nil, nil) events := parseSSEEvents(t, rr.Body.String()) if hasEvent(events, "response.failed") { t.Fatalf("must not emit response.failed on normal finish:\n%s", rr.Body.String()) @@ -596,7 +596,7 @@ func TestResponsesStream_LengthFinish_Incomplete(t *testing.T) { }, "\n") rr := httptest.NewRecorder() resp := &http.Response{StatusCode: 200, Body: io.NopCloser(strings.NewReader(upstream)), Header: make(http.Header)} - responsesStreamHandler(rr, nil, resp, "m", "m", false, nil, nil, ResponsesAPIRequest{}) + responsesStreamHandler(rr, nil, resp.Body, "m", "m", false, nil, nil, ResponsesAPIRequest{}, nil, nil) events := parseSSEEvents(t, rr.Body.String()) if hasEvent(events, "response.failed") { t.Fatalf("must not emit response.failed on length finish:\n%s", rr.Body.String()) @@ -623,7 +623,7 @@ func TestResponsesStream_FinishNoTrailingNewline_EOF(t *testing.T) { }, "\n") rr := httptest.NewRecorder() resp := &http.Response{StatusCode: 200, Body: io.NopCloser(strings.NewReader(upstream)), Header: make(http.Header)} - responsesStreamHandler(rr, nil, resp, "m", "m", false, nil, nil, ResponsesAPIRequest{}) + responsesStreamHandler(rr, nil, resp.Body, "m", "m", false, nil, nil, ResponsesAPIRequest{}, nil, nil) events := parseSSEEvents(t, rr.Body.String()) if hasEvent(events, "response.failed") { t.Fatalf("must not emit response.failed when finish chunk has no trailing newline:\n%s", rr.Body.String()) @@ -640,7 +640,7 @@ func TestResponsesStream_PartialDeltaThenReaderError(t *testing.T) { } rr := httptest.NewRecorder() resp := &http.Response{StatusCode: 200, Body: er, Header: make(http.Header)} - responsesStreamHandler(rr, nil, resp, "m", "m", false, nil, nil, ResponsesAPIRequest{}) + responsesStreamHandler(rr, nil, resp.Body, "m", "m", false, nil, nil, ResponsesAPIRequest{}, nil, nil) body := rr.Body.String() // The partial delta should be present (as output_text.delta) if !strings.Contains(body, "partial delta") { @@ -724,7 +724,7 @@ func TestResponsesStream_FinishThenInBandError(t *testing.T) { }, "\n") rr := httptest.NewRecorder() resp := &http.Response{StatusCode: 200, Body: io.NopCloser(strings.NewReader(upstream)), Header: make(http.Header)} - responsesStreamHandler(rr, nil, resp, "m", "m", false, nil, nil, ResponsesAPIRequest{}) + responsesStreamHandler(rr, nil, resp.Body, "m", "m", false, nil, nil, ResponsesAPIRequest{}, nil, nil) events := parseSSEEvents(t, rr.Body.String()) if !hasEvent(events, "response.failed") { t.Fatalf("expected response.failed after finish+in-band error: %s", rr.Body.String()) @@ -749,7 +749,7 @@ func TestResponsesStream_FinishThenReaderError(t *testing.T) { } rr := httptest.NewRecorder() resp := &http.Response{StatusCode: 200, Body: er, Header: make(http.Header)} - responsesStreamHandler(rr, nil, resp, "m", "m", false, nil, nil, ResponsesAPIRequest{}) + responsesStreamHandler(rr, nil, resp.Body, "m", "m", false, nil, nil, ResponsesAPIRequest{}, nil, nil) events := parseSSEEvents(t, rr.Body.String()) if !hasEvent(events, "response.failed") { t.Fatalf("expected response.failed after finish+reader error: %s", rr.Body.String()) From eecc1f726726f60c72a2fbbc114f9d22f29f2300 Mon Sep 17 00:00:00 2001 From: 6Kmfi6HP <179412085+6Kmfi6HP@users.noreply.github.com> Date: Mon, 28 Sep 2026 00:00:20 +0200 Subject: [PATCH 7/9] feat(passthrough): peek + EOF synthesis for byte-relay streams relayResponsesStream + pipeAnthropicStream + tests Co-Authored-By: Claude Code --- internal/app/anthropic_upstream.go | 187 +++++++++++++------ internal/app/anthropic_upstream_test.go | 5 +- internal/app/chat_protocol_routing_test.go | 13 +- internal/app/passthrough_retry_test.go | 205 +++++++++++++++++++++ internal/app/responses_passthrough.go | 131 +++++++++++-- internal/app/stream_retry.go | 15 ++ 6 files changed, 482 insertions(+), 74 deletions(-) create mode 100644 internal/app/passthrough_retry_test.go diff --git a/internal/app/anthropic_upstream.go b/internal/app/anthropic_upstream.go index d605055..921f99f 100644 --- a/internal/app/anthropic_upstream.go +++ b/internal/app/anthropic_upstream.go @@ -58,34 +58,55 @@ func forwardClaudeViaAnthropic(ctx context.Context, w http.ResponseWriter, auth // 非 2xx 即使请求方要求 stream 也统一走 buffered JSON 错误直转 // （上游未建立 SSE 流，tee 会把错误 JSON 包进 data frame 破坏客户端解析）。 if status >= 200 && status < 300 && stream { - pipeAnthropicStream(ctx, w, rc, status, header, modelID) + // 用 DriveStreamWithRetry 在 peek 失败时切换 key 重试,空流 / EOF / + // 首字节超时被翻译为可重试的 errStreamIncompleteNoCommit。首轮复用 + // 调用方已打开的 rc,后续重试通过 callOpenCodeAnthropicEndpoint 让 + // key pool 切到下一把可用 key。 + pending := rc + callOnce := func(c context.Context) (io.ReadCloser, int, error) { + if pending != nil { + r := pending + pending = nil + return r, status, nil + } + nrc, nstatus, _, nerr := callOpenCodeAnthropicEndpoint(c, upstreamBody, modelID, auth) + return nrc, nstatus, nerr + } + runOnce := func(c context.Context, w http.ResponseWriter, nrc io.Reader, _ []streamReadResult, _ *streamReader) (bool, error) { + return pipeAnthropicStream(c, w, nrc, status, header, modelID) + } + committed, driveErr := DriveStreamWithRetry(ctx, w, AnthropicProtocolHooks, callOnce, runOnce) + if !committed { + // 一直未 commit,让上层走 chat 翻译路径;不写任何字节给客户端。 + if driveErr != nil { + log.Warn("anthropic passthrough stream exhausted retries", "model", modelID, "err", driveErr) + } + return false + } return true } relayAnthropicBuffered(ctx, w, rc, status, header, modelID) return true } -// flushWriter 在每次 Write 后立即 Flush,保证 SSE 以事件粒度实时下发; -// http.ResponseWriter 内部带 bufio 缓冲,不显式 Flush 会把事件攒批到 EOF -// (与 responses_passthrough.go relayResponsesStream 的逐行 Flush 同一约定)。 -type flushWriter struct { - w io.Writer - f http.Flusher -} - -func (fw flushWriter) Write(p []byte) (int, error) { - n, err := fw.w.Write(p) - if n > 0 { - fw.f.Flush() - } - return n, err -} - // pipeAnthropicStream 把上游 Anthropic SSE 流字节级原样转发给客户端,同时 // 旁路 tee 解析 message_start / message_delta 中的 usage 记入 token 统计。 // 行边界、CRLF/LF、空行均不做改写,确保下游收到与上游完全一致的字节流。 // 仅在上游 2xx(真 SSE)时被调用；错误响应一律走 relayAnthropicBuffered。 -func pipeAnthropicStream(ctx context.Context, w http.ResponseWriter, rc io.Reader, status int, header http.Header, modelID string) { +// +// 返回 (true, nil)：已 commit（首帧已 peek + 写入）。EOF 时若未见过 +// message_stop 但见过 message_start,合成一条 message_stop 保证客户端正常 +// 关流；若两者皆无（不应发生：peek 至少要看到一帧）返回 false 供调用方 +// 走未 commit 重试。返回 (false, err)：peek 未 commit(空流 / EOF / 错误帧 / +// 首字节超时),由 DriveStreamWithRetry 决定是否换 key 重发。 +func pipeAnthropicStream(ctx context.Context, w http.ResponseWriter, rc io.Reader, status int, header http.Header, modelID string) (bool, error) { + // peek 首帧:在 WriteHeader 之前约束 commit 边界,空流 / EOF / 错误帧 / + // 首字节超时都返回 errStreamIncompleteNoCommit,由调用方驱动重试。 + peek := PeekFirstFrame(ctx, rc, time.Duration(config.StreamFirstByteTimeoutMs())*time.Millisecond, AnthropicProtocolHooks) + if peek.Err != nil { + return false, peek.Err + } + filtered := filterResponseHeaders(header) for k, v := range filtered { w.Header().Set(k, v[0]) @@ -97,61 +118,113 @@ func pipeAnthropicStream(ctx context.Context, w http.ResponseWriter, rc io.Reade stats := &logging.StreamStats{Start: time.Now()} fullUsage := map[string]any{} - - // tee 管道:旁路解析走 pipeWriter,主流走 io.Copy 直透;两组无背压, - // io.Copy 返回(EOF、rc 读取失败、pw.Write 失败)时主动 pw.Close() - // 告知解析端收尾;ctx 取消则先 close 上游 rc 解锁 io.Copy,再 - // pw.CloseWithError(ctx.Err()) 让 pr.Read 立刻返回。 - pr, pw := io.Pipe() - copyDone := make(chan struct{}) - go func() { - defer close(copyDone) - // MultiWriter 把每个 read 同步写给客户端与旁路解析端;flushWriter - // 让每片上游数据即时下发(不攒批)。任一侧写失败 io.Copy 立即返回, - // 随后 pw.Close 告知解析端收尾,最终 close(copyDone) 供主循环 join。 - flusher, _ := w.(http.Flusher) - cw := io.Writer(w) - if flusher != nil { - cw = flushWriter{w: w, f: flusher} - } - _, _ = io.Copy(io.MultiWriter(cw, pw), rc) - _ = pw.Close() - }() - - // tee 解析流:复用 newStreamReader 的协程,读到行就 observe,不写出。 - reader := newStreamReader(ctx, pr, 0) defer func() { if len(fullUsage) > 0 { statsx.RecordChatUsage(modelID, anthropicUsageToChat(fullUsage)) } stats.Log(ctx, "claude") }() + + flusher, _ := w.(http.Flusher) + // 写 peek 出的原始字节(完整保留 \r\n / 换行 / 空行),同时喂给 + // observeAnthropicStreamEvent 让 stats 与 message_start/stop 计数正确 + // 累计——peek 消费过的帧不再二次进 reader.Read() 通道,所以这里必须补 + // 一次观察。 + if err := FlushPeekedBytes(w, peek.Consumed); err != nil { + return true, err + } + sawMessageStart := false + sawMessageStop := false + observeLine := func(line string) { + stats.NoteChunk() + observeAnthropicStreamEvent(stats, fullUsage, line) + payload, ok := strings.CutPrefix(line, "data: ") + if !ok { + return + } + var evt map[string]any + if json.Unmarshal([]byte(strings.TrimSpace(payload)), &evt) != nil { + return + } + switch typ, _ := evt["type"].(string); typ { + case "message_start": + sawMessageStart = true + case "message_stop": + sawMessageStop = true + } + } + // 先用 peek 消费过的行回填 sawMessageStart / sawMessageStop——后续 EOF + // 兜底合成 message_stop 需要知道是否已见过 message_start / message_stop。 + for _, res := range peek.Consumed { + if res.line != "" { + observeLine(res.line) + } + } + if flusher != nil { + flusher.Flush() + } + + // 续用 peek 内部 streamReader(它的 bufio 已预读后续行),按 SSE 帧聚合 + // 再写客户端——与原 io.Copy 的「一次上游 chunk ≈ 一次 Write+Flush」 + // 节奏对齐,保留逐事件的打字机效果,而不是退回到 line-at-a-time。 + reader := peek.Reader + if reader == nil { + // EOF 收尾的 peek 没留下 reader——主循环立即结束。 + reader = newStreamReader(ctx, rc, 0) + } defer reader.Close() + var frameBuf strings.Builder + flushFrame := func() error { + if frameBuf.Len() == 0 { + return nil + } + _, err := io.WriteString(w, frameBuf.String()) + frameBuf.Reset() + if err != nil { + return err + } + if flusher != nil { + flusher.Flush() + } + return nil + } + for { select { case <-ctx.Done(): - // 先 close 上游,让 io.Copy 立刻读到错误退出(不再卡在 w.Write), - // 再 close pipe 让旁路解析收尾,这样 copyDone 不会等慢客户端。 - if c, ok := rc.(io.Closer); ok { - _ = c.Close() - } - _ = pw.CloseWithError(ctx.Err()) - <-copyDone - return + return true, ctx.Err() case result := <-reader.Read(): - pendingErr := result.err line := result.line + pendingErr := result.err if line != "" { - stats.NoteChunk() - observeAnthropicStreamEvent(stats, fullUsage, line) + observeLine(line) + frameBuf.WriteString(line) + // 空行 = 帧边界:整帧一次写出再 Flush。 + if strings.TrimRight(line, "\r\n") == "" { + if err := flushFrame(); err != nil { + return true, err + } + } } if pendingErr != nil { - // pr 的错误只可能来自 pw.Close(),即 copy 协程已越过 io.Copy, - // 此处 join 必然立即返回;保证协程不再于 handler 返回后触碰 - // 已交还的 http.ResponseWriter(net/http 禁止这种并发使用)。 - <-copyDone - return + // EOF / 上游读取失败。先把残帧(无空行收尾)吐出去,再看是否 + // 需要补 message_stop / 走重试。 + if err := flushFrame(); err != nil { + return true, err + } + if !sawMessageStop { + if !sawMessageStart { + return false, errStreamIncompleteNoCommit + } + if _, err := io.WriteString(w, "event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n"); err != nil { + return true, err + } + if flusher != nil { + flusher.Flush() + } + } + return true, nil } } } diff --git a/internal/app/anthropic_upstream_test.go b/internal/app/anthropic_upstream_test.go index 5281ea8..ab45d19 100644 --- a/internal/app/anthropic_upstream_test.go +++ b/internal/app/anthropic_upstream_test.go @@ -102,7 +102,10 @@ func TestPipeAnthropicStream_PreservesHeaderAndBytes(t *testing.T) { rec := httptest.NewRecorder() header := http.Header{} header.Set("Content-Type", "text/event-stream") - pipeAnthropicStream(context.Background(), rec, io.NopCloser(strings.NewReader(upstreamBody)), http.StatusOK, header, "m") + committed, perr := pipeAnthropicStream(context.Background(), rec, io.NopCloser(strings.NewReader(upstreamBody)), http.StatusOK, header, "m") + if !committed || perr != nil { + t.Fatalf("pipeAnthropicStream = (%v, %v), want (true, nil)", committed, perr) + } if rec.Code != http.StatusOK { t.Fatalf("status = %d, want 200", rec.Code) diff --git a/internal/app/chat_protocol_routing_test.go b/internal/app/chat_protocol_routing_test.go index 64ad94d..b06b71a 100644 --- a/internal/app/chat_protocol_routing_test.go +++ b/internal/app/chat_protocol_routing_test.go @@ -427,7 +427,10 @@ func TestPipeAnthropicStream_ByteIdentityPassthrough(t *testing.T) { rec := httptest.NewRecorder() header := http.Header{} header.Set("Content-Type", "text/event-stream") - pipeAnthropicStream(context.Background(), rec, io.NopCloser(strings.NewReader(upstreamBody)), http.StatusOK, header, "m") + committed, perr := pipeAnthropicStream(context.Background(), rec, io.NopCloser(strings.NewReader(upstreamBody)), http.StatusOK, header, "m") + if !committed || perr != nil { + t.Fatalf("pipeAnthropicStream = (%v, %v), want (true, nil)", committed, perr) + } // 字节完全一致。 if rec.Body.String() != upstreamBody { @@ -476,8 +479,14 @@ func TestPipeAnthropicStream_FlushesEachUpstreamChunk(t *testing.T) { chunk2 := "event: message_stop\r\ndata: {\"type\":\"message_stop\"}\r\n\r\n" fcr := &flushCountingRecorder{ResponseRecorder: httptest.NewRecorder()} header := http.Header{} - pipeAnthropicStream(context.Background(), fcr, &chunkedReader{chunks: []string{chunk1, chunk2}}, http.StatusOK, header, "m") + committed, perr := pipeAnthropicStream(context.Background(), fcr, &chunkedReader{chunks: []string{chunk1, chunk2}}, http.StatusOK, header, "m") + if !committed || perr != nil { + t.Fatalf("pipeAnthropicStream = (%v, %v), want (true, nil)", committed, perr) + } + // 上游 chunk1 + chunk2 各占一次 flush;peek 阶段已刷过 chunk1。新流式 + // 路径先写 peek 字节、Flush,再逐行写;如果 peek / 主循环都发生过 Flush, + // 总数应与上游 chunk 数一致(2)。 if fcr.flushes != 2 { t.Fatalf("expected one flush per upstream chunk (2), got %d", fcr.flushes) } diff --git a/internal/app/passthrough_retry_test.go b/internal/app/passthrough_retry_test.go new file mode 100644 index 0000000..a3a2858 --- /dev/null +++ b/internal/app/passthrough_retry_test.go @@ -0,0 +1,205 @@ +package app + +// passthrough_retry_test.go 覆盖 byte-passthrough 路径(relayResponsesStream +// + pipeAnthropicStream)在 peek 失败时的「空流重试」与在 EOF 时的「半截合成 +// 收尾」两条行为契约。 +// +// 不同于 claude_responses_empty_retry_test.go 的协议翻译路径,这里被测函 +// 数自己就是「写字节到客户端」的最后一公里——peek 拿到的原始字节必须原 +// 样写回 w,不能 unmarshal→remarshal(避免 key 顺序、tag 数字、空格被改 +// 写)。 + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/6Kmfi6HP/opencode2api/internal/domain" +) + +// responsesPassthroughHealthySSE 一份「正常完结」的最小原生 Responses SSE 流。 +const responsesPassthroughHealthySSE = "event: response.output_text.delta\n" + + `data: {"type":"response.output_text.delta","output_index":0,"item_id":"msg_1","delta":"hello"}` + "\n\n" + + "event: response.completed\n" + + `data: {"type":"response.completed","response":{"id":"resp_ok","status":"completed","output":[{"id":"msg_1","type":"message","content":[{"type":"output_text","text":"hello"}]}],"usage":{"input_tokens":1,"output_tokens":1,"total_tokens":2}}}` + "\n\n" + +// TestRelayResponsesStream_EmptyEOF_RetriesOnce：上游首轮 200 但 body 立刻 +// EOF(prefill 阶段 tunnel 被宰,干净 EOF)。peek 应判为未 commit,Drive +// StreamWithRetry 切下一把 key 重试拿到正常流,客户端只见干净成功流。 +func TestRelayResponsesStream_EmptyEOF_RetriesOnce(t *testing.T) { + stubRetryConfig(t, 1, 5000) + stubNativeModel(t, "gpt-resp-retry-a") + + callCount := 0 + upstreamCall := func(c context.Context, body []byte) (io.ReadCloser, int, http.Header, error) { + callCount++ + hdr := http.Header{"Content-Type": []string{"text/event-stream"}} + // 首轮 rc 由 relayResponsesToClient 调用方提供(空流);callOnce 后续 + // 重试才走这里。所以本闭包首次调用就该返回健康 SSE。 + return io.NopCloser(strings.NewReader(responsesPassthroughHealthySSE)), http.StatusOK, hdr, nil + } + + rec := httptest.NewRecorder() + firstRC := io.NopCloser(strings.NewReader(``)) // 由 relayResponsesToClient 拿到 + hdr := http.Header{"Content-Type": []string{"text/event-stream"}} + // 首轮 rc 已建立(空流),走带 DriveStreamWithRetry 的外层包装:首轮进入 + // relayResponsesStream 会 peek 出 EOF,返回 errStreamIncompleteNoCommit, + // driver 用 upstreamCall 拿到第二份(健康)流再跑一遍。 + relayResponsesToClient(context.Background(), rec, firstRC, http.StatusOK, hdr, "gpt-resp-retry-a", true, ResponsesAPIRequest{}, newResponsesNameRewrites(), UpstreamAuth{}, []byte(`{"model":"gpt-resp-retry-a","stream":true,"input":"hi"}`), upstreamCall) + + if callCount != 1 { + t.Fatalf("expected exactly one retry upstream call, got callCount=%d", callCount) + } + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200, body=%s", rec.Code, rec.Body.String()) + } + body := rec.Body.String() + if !strings.Contains(body, `"response.output_text.delta"`) || !strings.Contains(body, "hello") { + t.Fatalf("expected retry to deliver healthy SSE, got:\n%s", body) + } +} + +// TestRelayResponsesStream_PartialEOF_NoRetry：上游已发过 data 帧后 EOF,无 +// completed——已 commit,不再触发空流重试;末尾合成 incomplete + [DONE]。 +func TestRelayResponsesStream_PartialEOF_NoRetry(t *testing.T) { + stubRetryConfig(t, 1, 5000) + partial := "event: response.output_text.delta\n" + + `data: {"type":"response.output_text.delta","output_index":0,"item_id":"msg_1","delta":"partial answer"}` + "\n\n" + // 故意没有 response.completed / [DONE] —— EOF 兜底走的是「合成 incom + // plete + [DONE]」,不应触发空流重试(那个分支只在首帧之前)。 + callCount := 0 + upstreamCall := func(c context.Context, body []byte) (io.ReadCloser, int, http.Header, error) { + callCount++ + // retry 槽放一条健康 SSE——不应被消费。 + return io.NopCloser(strings.NewReader(responsesPassthroughHealthySSE)), http.StatusOK, http.Header{"Content-Type": []string{"text/event-stream"}}, nil + } + + rec := httptest.NewRecorder() + hdr := http.Header{"Content-Type": []string{"text/event-stream"}} + relayResponsesToClient(context.Background(), rec, + io.NopCloser(strings.NewReader(partial)), http.StatusOK, hdr, "gpt-resp-retry-b", true, + ResponsesAPIRequest{}, newResponsesNameRewrites(), UpstreamAuth{}, []byte(`{"model":"gpt-resp-retry-b","stream":true,"input":"hi"}`), upstreamCall) + + if callCount != 0 { + t.Fatalf("retry must not be consumed on partial EOF (already committed), got callCount=%d", callCount) + } + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200, body=%s", rec.Code, rec.Body.String()) + } + body := rec.Body.String() + if !strings.Contains(body, "partial answer") { + t.Fatalf("expected partial body delivered:\n%s", body) + } + if !strings.Contains(body, "response.incomplete") { + t.Fatalf("expected synthesized response.incomplete on EOF:\n%s", body) + } + if strings.Contains(body, "hello") { + t.Fatalf("must not consume retry slot when already committed:\n%s", body) + } +} + +// anthropicHealthySSE 一份「正常完结」的最小 Anthropic SSE 流。 +const anthropicHealthySSE = "event: message_start\n" + + `data: {"type":"message_start","message":{"id":"msg_a","role":"assistant","usage":{"input_tokens":1}}}` + "\n\n" + + "event: content_block_delta\n" + + `data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"hi"}}` + "\n\n" + + "event: message_stop\n" + + `data: {"type":"message_stop"}` + "\n\n" + +// TestPipeAnthropicStream_EmptyEOF_RetriesOnce：/messages → Anthropic 直通, +// 上游首轮 200 但 body 立刻 EOF,经 DriveStreamWithRetry 重试拿到正常流。 +func TestPipeAnthropicStream_EmptyEOF_RetriesOnce(t *testing.T) { + stubRetryConfig(t, 1, 5000) + setProtocolRulesForTest(t, []domain.ProtocolRule{{Pattern: "claude-ant-retry-*", Protocol: "anthropic"}}) + installFakeOpenCodeClient(t, []fakeUpstreamResponse{ + {status: http.StatusOK, body: ``, header: http.Header{"Content-Type": []string{"text/event-stream"}}}, + {status: http.StatusOK, body: anthropicHealthySSE, header: http.Header{"Content-Type": []string{"text/event-stream"}}}, + }) + + req := httptest.NewRequest(http.MethodPost, "/v1/messages", + strings.NewReader(`{"model":"claude-ant-retry-a","max_tokens":64,"stream":true,"messages":[{"role":"user","content":"hi"}]}`)) + rec := httptest.NewRecorder() + claudeMessagesHandler(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200, body=%s", rec.Code, rec.Body.String()) + } + body := rec.Body.String() + if !strings.Contains(body, `"type":"message_start"`) || !strings.Contains(body, "hi") { + t.Fatalf("expected retry to deliver healthy SSE, got:\n%s", body) + } + if !strings.Contains(body, `"type":"message_stop"`) { + t.Fatalf("expected message_stop from retry, got:\n%s", body) + } +} + +// TestPipeAnthropicStream_EOFAfterMessageStart_SynthesizesMessageStop：已见过 +// message_start 但 EOF 前没有 message_stop——已 commit,合成一条 message_stop +// 让客户端正常关流;不重试。 +func TestPipeAnthropicStream_EOFAfterMessageStart_SynthesizesMessageStop(t *testing.T) { + stubRetryConfig(t, 1, 5000) + setProtocolRulesForTest(t, []domain.ProtocolRule{{Pattern: "claude-ant-partial-*", Protocol: "anthropic"}}) + partialSSE := "event: message_start\n" + + `data: {"type":"message_start","message":{"id":"msg_p","role":"assistant","usage":{"input_tokens":1}}}` + "\n\n" + + "event: content_block_delta\n" + + `data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"partial"}}` + "\n\n" + installFakeOpenCodeClient(t, []fakeUpstreamResponse{ + {status: http.StatusOK, body: partialSSE, header: http.Header{"Content-Type": []string{"text/event-stream"}}}, + // 重试候选位的健康流不应被消费(已 commit)。 + {status: http.StatusOK, body: anthropicHealthySSE, header: http.Header{"Content-Type": []string{"text/event-stream"}}}, + }) + + req := httptest.NewRequest(http.MethodPost, "/v1/messages", + strings.NewReader(`{"model":"claude-ant-partial-a","max_tokens":64,"stream":true,"messages":[{"role":"user","content":"hi"}]}`)) + rec := httptest.NewRecorder() + claudeMessagesHandler(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200, body=%s", rec.Code, rec.Body.String()) + } + body := rec.Body.String() + if !strings.Contains(body, "partial") { + t.Fatalf("expected partial body delivered, got:\n%s", body) + } + if !strings.Contains(body, `"type":"message_stop"`) { + t.Fatalf("expected synthesized message_stop on EOF, got:\n%s", body) + } + // 重试不应被触发(retry 槽位的 msg_a 来自健康流,partial SSE 用的是 + // msg_p)。 + if strings.Contains(body, "msg_a") { + t.Fatalf("retry slot must not have been consumed (already committed):\n%s", body) + } +} + +// TestPipeAnthropicStream_ErrorEvent_Retries：上游 200 但首帧就是 type=error +// 的 SSE 事件,peek 视为未 commit,DriveStreamWithRetry 切 key 重试;最终的 +// retry 槽返回正常流。 +func TestPipeAnthropicStream_ErrorEvent_Retries(t *testing.T) { + stubRetryConfig(t, 1, 5000) + setProtocolRulesForTest(t, []domain.ProtocolRule{{Pattern: "claude-ant-error-*", Protocol: "anthropic"}}) + errSSE := "event: error\n" + + `data: {"type":"error","error":{"type":"overloaded_error","message":"boom"}}` + "\n\n" + installFakeOpenCodeClient(t, []fakeUpstreamResponse{ + {status: http.StatusOK, body: errSSE, header: http.Header{"Content-Type": []string{"text/event-stream"}}}, + {status: http.StatusOK, body: anthropicHealthySSE, header: http.Header{"Content-Type": []string{"text/event-stream"}}}, + }) + + req := httptest.NewRequest(http.MethodPost, "/v1/messages", + strings.NewReader(`{"model":"claude-ant-error-a","max_tokens":64,"stream":true,"messages":[{"role":"user","content":"hi"}]}`)) + rec := httptest.NewRecorder() + claudeMessagesHandler(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200, body=%s", rec.Code, rec.Body.String()) + } + body := rec.Body.String() + if !strings.Contains(body, `"type":"message_start"`) { + t.Fatalf("expected retry message_start, got:\n%s", body) + } + if strings.Contains(body, "overloaded_error") { + t.Fatalf("upstream error frame must not leak to client when retry succeeded:\n%s", body) + } +} diff --git a/internal/app/responses_passthrough.go b/internal/app/responses_passthrough.go index 61dc7ed..9ba2dd1 100644 --- a/internal/app/responses_passthrough.go +++ b/internal/app/responses_passthrough.go @@ -1,7 +1,6 @@ package app import ( - "bufio" "bytes" "context" "crypto/sha256" @@ -16,6 +15,7 @@ import ( "net/http" "strings" "sync" + "time" ) // ======================== function_call 参数浮点归一化 ======================== @@ -910,7 +910,26 @@ func relayResponsesToClient(ctx context.Context, w http.ResponseWriter, rc io.Re } if stream && status >= 200 && status < 300 { - relayResponsesStream(ctx, w, rc, status, modelID, req, rewrites, auth, rawBody, upstreamCall) + // 用 DriveStreamWithRetry 在 peek 失败时切换 key 重试,空流/超时/上游 + // 错误帧都被翻译为可重试的 errStreamIncompleteNoCommit。首轮复用 + // 调用方已打开的 rc,后续重试走 upstreamCall 让 key pool 换下一把 key。 + pending := rc.(io.ReadCloser) + callOnce := func(c context.Context) (io.ReadCloser, int, error) { + if pending != nil { + r := pending + pending = nil + return r, status, nil + } + nrc, nstatus, _, nerr := upstreamCall(c, rawBody) + return nrc, nstatus, nerr + } + runOnce := func(c context.Context, w http.ResponseWriter, nrc io.Reader, _ []streamReadResult, _ *streamReader) (bool, error) { + return relayResponsesStream(c, w, nrc, status, modelID, req, rewrites, auth, rawBody, upstreamCall) + } + committed, driveErr := DriveStreamWithRetry(ctx, w, ResponsesProtocolHooks, callOnce, runOnce) + if !committed && driveErr != nil { + logging.FromContext(ctx).Warn("relayResponsesStream exhausted retries", "model", modelID, "err", driveErr) + } return } @@ -965,25 +984,80 @@ func relayResponsesToClient(ctx context.Context, w http.ResponseWriter, rc io.Re // relayResponsesStream 逐行透传 SSE 并在每个事件行后 Flush，保证打字机效果； // 同时从 response.completed / usage 事件中提取 usage 做 Token 统计，并保存 // 完整响应对象以维持 previous_response_id 会话链条。 -func relayResponsesStream(ctx context.Context, w http.ResponseWriter, rc io.Reader, status int, modelID string, req ResponsesAPIRequest, rewrites *responsesNameRewrites, auth UpstreamAuth, rawBody []byte, upstreamCall func(context.Context, []byte) (io.ReadCloser, int, http.Header, error)) { +// +// 返回 (true, nil)：已 commit 且本轮处理完毕（写完或合成了收尾），调用方 +// 无需重试。返回 (false, err)：peek 窗口内未 commit（空流 / EOF / 错误帧 / +// 首字节超时），由 DriveStreamWithRetry 决定是否换 key 重发。已 commit 后 +// 主循环返回 (true, nil)——半截流由 continuation / 末尾 [DONE] 兜底处理。 +func relayResponsesStream(ctx context.Context, w http.ResponseWriter, rc io.Reader, status int, modelID string, req ResponsesAPIRequest, rewrites *responsesNameRewrites, auth UpstreamAuth, rawBody []byte, upstreamCall func(context.Context, []byte) (io.ReadCloser, int, http.Header, error)) (bool, error) { w.Header().Set("Content-Type", "text/event-stream") w.Header().Set("Cache-Control", "no-cache") w.Header().Set("Connection", "keep-alive") w.Header().Set("X-Accel-Buffering", "no") - w.WriteHeader(status) + // peek 首帧：在 WriteHeader 之前约束 commit 边界，空流 / EOF / 错误帧 / + // 首字节超时返回 errStreamIncompleteNoCommit,由调用方驱动重试。 + peek := PeekFirstFrame(ctx, rc, time.Duration(config.StreamFirstByteTimeoutMs())*time.Millisecond, ResponsesProtocolHooks) + if peek.Err != nil { + return false, peek.Err + } + + w.WriteHeader(status) flusher, _ := w.(http.Flusher) if flusher != nil { flusher.Flush() } - _ = ctx + // byte-level 透传：peek 已消费的字节保持原样直写 w,与主循环读取的 + // 后续字节拼成完整 SSE 流。 + if err := FlushPeekedBytes(w, peek.Consumed); err != nil { + return false, err + } + if flusher != nil { + flusher.Flush() + } + + // 续用 peek 内部的 reader（它的 bufio 可能已预读后续行）。 + sr := peek.Reader + if sr == nil { + // EOF 收尾的 peek 没留下 reader——上游已 EOF,主循环立即结束。 + sr = newStreamReader(ctx, rc, 0) + } + defer sr.Close() + var lastUsage map[string]any var lastResponse map[string]any - sawData := false + sawData := len(peek.Consumed) > 0 doneSeen := false writeFailed := false terminalSeen := false + // peek 阶段已看到完整帧，但终端事件（completed/failed/incomplete / [DONE]） + // 要逐行扫过目前 consumed 才知道——这里只预先回填终端标志，让末尾判定 + // 与原有逻辑一致。 + for _, res := range peek.Consumed { + trimmed := bytes.TrimSpace([]byte(res.line)) + if bytes.Equal(trimmed, []byte("data: [DONE]")) || bytes.Equal(trimmed, []byte("[DONE]")) { + doneSeen = true + continue + } + if !bytes.HasPrefix(trimmed, []byte("data: ")) { + continue + } + payload := trimmed[6:] + if len(payload) == 0 || payload[0] != '{' { + continue + } + var evt map[string]any + if json.Unmarshal(payload, &evt) != nil { + continue + } + if _, response := extractStreamEventUsage([]byte(res.line)); response != nil { + lastResponse = response + if s, _ := response["status"].(string); s == "completed" || s == "failed" || s == "incomplete" { + terminalSeen = true + } + } + } argStates := map[int]*argsNormState{} argItemToOutput := map[string]int{} @@ -997,18 +1071,43 @@ func relayResponsesStream(ctx context.Context, w http.ResponseWriter, rc io.Read // pendingTerminalLine 保存本轮的终结事件行（incomplete + max_output_tokens 时暂缓写入） var pendingTerminalLine []byte + // 首轮主循环复用 peek 的 streamReader；续写轮重新发起 upstreamCall 后, + // 用新响应的 rc 新建 streamReader。 + currentReader := sr + for round := 0; round <= maxContinuations && !doneSeen && !writeFailed; round++ { - reader := bufio.NewReader(currentRC) var contLastResponse map[string]any isTruncatedByMaxTokens = false pendingTerminalLine = nil + reader := currentReader + needClose := false + if round > 0 { + // 续写轮：用新 rc 起一个 streamReader(它的内部协程按行投递,与 + // 主循环的 select 模型对齐),不再用裸 bufio.NewReader——前者同时 + // 兼容 rc 后台 Close 触发 EOF。本轮结束后立刻 Close,不堆积协程。 + reader = newStreamReader(ctx, currentRC, 0) + needClose = true + } + + lineLoop: for { - line, err := reader.ReadBytes('\n') + var ( + line string + err error + ) + select { + case <-ctx.Done(): + return false, ctx.Err() + case res := <-reader.Read(): + line = res.line + err = res.err + } if len(line) > 0 { - outLine := line + lineBytes := []byte(line) + outLine := lineBytes if isMuseSparkModel(modelID) { - if normalized, ok := normalizeResponsesStreamLine(line, argStates, argItemToOutput, rewrites); ok { + if normalized, ok := normalizeResponsesStreamLine(lineBytes, argStates, argItemToOutput, rewrites); ok { outLine = normalized } } @@ -1016,7 +1115,7 @@ func relayResponsesStream(ctx context.Context, w http.ResponseWriter, rc io.Read isDoneSentinel := bytes.Equal(trimmed, []byte("data: [DONE]")) || bytes.Equal(trimmed, []byte("[DONE]")) if isDoneSentinel && !terminalSeen { if err != nil { - break + break lineLoop } continue } @@ -1055,7 +1154,7 @@ func relayResponsesStream(ctx context.Context, w http.ResponseWriter, rc io.Read } else if !isTerminalLine { if _, werr := w.Write(outLine); werr != nil { writeFailed = true - break + break lineLoop } if flusher != nil { flusher.Flush() @@ -1064,7 +1163,7 @@ func relayResponsesStream(ctx context.Context, w http.ResponseWriter, rc io.Read // completed / failed / 其他 incomplete：直写 if _, werr := w.Write(outLine); werr != nil { writeFailed = true - break + break lineLoop } if flusher != nil { flusher.Flush() @@ -1094,9 +1193,12 @@ func relayResponsesStream(ctx context.Context, w http.ResponseWriter, rc io.Read } } if err != nil { - break + break lineLoop } } + if needClose { + reader.Close() + } if flusher != nil { flusher.Flush() } @@ -1241,6 +1343,7 @@ func relayResponsesStream(ctx context.Context, w http.ResponseWriter, rc io.Read flusher.Flush() } } + return true, nil } // buildContinuationBody 构造续写请求：input 设为已收到的 output 数组， diff --git a/internal/app/stream_retry.go b/internal/app/stream_retry.go index 3e10515..5522adb 100644 --- a/internal/app/stream_retry.go +++ b/internal/app/stream_retry.go @@ -123,6 +123,21 @@ var ( // 错误 / 上游错误事件——尚未向客户端写过任何字节，调用方可安全重试。 var errStreamIncompleteNoCommit = errors.New("stream incomplete before first client byte") +// FlushPeekedBytes 把 peek 阶段攒的 SSE 行原样写回 w（行字节、换行符都 +// 不动）。返回首个写错误。byte-level 透传路径用它把 peek 消费掉的字节 +// 回放给客户端。 +func FlushPeekedBytes(w io.Writer, peeked []streamReadResult) error { + for _, res := range peeked { + if res.line == "" { + continue + } + if _, err := io.WriteString(w, res.line); err != nil { + return err + } + } + return nil +} + // PeekOutcome 是 PeekFirstFrame 的结果。Consumed 不为空时表示「已有完整 // SSE 帧被消费」，调用方应把它原样喂回 handler 主循环。 type PeekOutcome struct { From a27ad6d94b59e6892c0120665f0e951aba888ee8 Mon Sep 17 00:00:00 2001 From: 6Kmfi6HP <179412085+6Kmfi6HP@users.noreply.github.com> Date: Mon, 28 Sep 2026 01:20:41 +0200 Subject: [PATCH 8/9] =?UTF-8?q?fix(stream):=20address=20review=20findings?= =?UTF-8?q?=20=E2=80=94=20commit=20boundary,=20goroutine=20leak,=20status?= =?UTF-8?q?=20passthrough?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review pass over the 6-commit retry rollout surfaced five structural bugs: F1 chat streaming swallowed upstream 4xx/5xx (status + body) into a generic 502 message. Wrap callOnce with UpstreamErrorCapture so non-2xx responses are kept (RC not closed by driver) and replayed verbatim to the client. F2 pipeAnthropicStream returned (false, errStreamIncompleteNoCommit) on EOF-after-peek even though WriteHeader + peeked bytes were already on the wire — Drive would then WriteHeader+replay again on the same connection. After commit, always finish with a synthesized message_stop; never return committed=false. F3 anthropicSSEToChatStream had the symmetric bug via writeHeaderOnce firing on the first non-empty line (even a ping or content_block_start that does not tick sentRole). Returning (false, ...) after that lets the driver run a duplicate WriteHeader+replay. Switched the EOF branch's early-return guard from !sentRole to !wroteHeader. F4 PeekFirstFrame leaked the streamReader goroutine when ctx was canceled mid-peek (readCh never receives, done never closed). Now explicitly reader.Close() on the ctx.Done branch like the other early exits. F5 chatStreamRunOnce recorded usage-bearing chunks regardless of whether commit had happened. When the attempt was then aborted (peek-EOF path), the next attempt would re-stat the same upstream usage. Gate RecordChatUsage on wasCommitted. F6 relayResponsesStream returned (false, ctx.Err()) from its lineLoop ctx branch even when WriteHeader had already been issued. Drive short-circuits on ctx errors so this didn't double-write in practice, but the contract violation was structural — the function now returns (true, ctx.Err()). Also fixes a lower-severity case where FlushPeekedBytes errors after WriteHeader were propagated as (false, err), for the same reason. Co-Authored-By: Claude Code --- internal/app/anthropic_upstream.go | 25 ++++-- internal/app/chat.go | 32 +++++-- internal/app/chat_to_anthropic.go | 22 ++++- internal/app/chat_to_responses_upstream.go | 9 +- internal/app/responses.go | 9 +- internal/app/responses_passthrough.go | 10 ++- internal/app/stream_retry.go | 100 +++++++++++++++++++-- 7 files changed, 174 insertions(+), 33 deletions(-) diff --git a/internal/app/anthropic_upstream.go b/internal/app/anthropic_upstream.go index 921f99f..4290341 100644 --- a/internal/app/anthropic_upstream.go +++ b/internal/app/anthropic_upstream.go @@ -70,6 +70,13 @@ func forwardClaudeViaAnthropic(ctx context.Context, w http.ResponseWriter, auth return r, status, nil } nrc, nstatus, _, nerr := callOpenCodeAnthropicEndpoint(c, upstreamBody, modelID, auth) + // Drive 拿到非 2xx 会立即返回(不再 close)。retry 路径里我们已 + // 在上层只能是「走 chat 翻译兜底」,rc 在这里直接 close 掉防泄漏。 + if nerr == nil && (nstatus < 200 || nstatus >= 300) { + if nrc != nil { + nrc.Close() + } + } return nrc, nstatus, nerr } runOnce := func(c context.Context, w http.ResponseWriter, nrc io.Reader, _ []streamReadResult, _ *streamReader) (bool, error) { @@ -133,7 +140,6 @@ func pipeAnthropicStream(ctx context.Context, w http.ResponseWriter, rc io.Reade if err := FlushPeekedBytes(w, peek.Consumed); err != nil { return true, err } - sawMessageStart := false sawMessageStop := false observeLine := func(line string) { stats.NoteChunk() @@ -146,10 +152,7 @@ func pipeAnthropicStream(ctx context.Context, w http.ResponseWriter, rc io.Reade if json.Unmarshal([]byte(strings.TrimSpace(payload)), &evt) != nil { return } - switch typ, _ := evt["type"].(string); typ { - case "message_start": - sawMessageStart = true - case "message_stop": + if typ, _ := evt["type"].(string); typ == "message_stop" { sawMessageStop = true } } @@ -213,10 +216,16 @@ func pipeAnthropicStream(ctx context.Context, w http.ResponseWriter, rc io.Reade if err := flushFrame(); err != nil { return true, err } + // 关键不变量:此时 WriteHeader + peeked 首帧字节已经发出去了, + // 客户端连接已经处于 SSE 数据段。**绝不可再返回 (false, ...)** + // 否则 DriveStreamWithRetry 会用同一个 ResponseWriter 二次 + // WriteHeader + 二次 replay peeked 字节,流被污染(I4/I7)。 if !sawMessageStop { - if !sawMessageStart { - return false, errStreamIncompleteNoCommit - } + // 上游 EOF 但没关 message:展开成「合成 message_stop」让 + // Claude SDK 正常关流。sawMessageStart=false 也照发——客 + // 户端拿到「没 message_start 直接 message_stop」虽不规范, + // 但比重复写 header 安全(对一个非法流,SDK 通常仅丢弃该 + // 事件,而不是报错)。 if _, err := io.WriteString(w, "event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n"); err != nil { return true, err } diff --git a/internal/app/chat.go b/internal/app/chat.go index c150790..e717493 100644 --- a/internal/app/chat.go +++ b/internal/app/chat.go @@ -571,11 +571,15 @@ func chatCompletionsHandler(w http.ResponseWriter, r *http.Request) { if req.Stream { ctx := r.Context() + // 用 UpstreamErrorCapture 包住 callOnce:非 2xx 时 rc 暂存到 cap + // (Drive 不会 close),让外层在后面把上游真实 status+body 回写给客户 + // 端,而不是吞掉换成通用 502。 + upstreamCap := &UpstreamErrorCapture{} committed, driveErr := DriveStreamWithRetry(ctx, w, ChatProtocolHooks, - func(c context.Context) (io.ReadCloser, int, error) { + upstreamCap.WrapCallOnce(func(c context.Context) (io.ReadCloser, int, error) { rc, status, _, err := callOpenCodeAPIStream(c, upstreamBody, req.Model, auth) return rc, status, err - }, + }), func(c context.Context, w http.ResponseWriter, rc io.Reader, peeked []streamReadResult, rd *streamReader) (bool, error) { return chatStreamRunOnce(c, w, rc, peeked, rd, &req, keepReasoning, clientWantsUsage) }) @@ -585,6 +589,12 @@ func chatCompletionsHandler(w http.ResponseWriter, r *http.Request) { if driveErr != nil && errors.Is(driveErr, context.Canceled) { return } + // 优先把上游真实 status + body 透回去:4xx/5xx 客户端能从 body + // 里看到具体错误信息(max_tokens 越界、rate limit 等),比 502 + // 通用错误更有用。仅在没拿到上游 body 时落在通用 502。 + if upstreamCap.WriteUpstreamErrorTo(w) { + return + } msg := "upstream stream incomplete" if driveErr != nil { if errors.Is(driveErr, context.DeadlineExceeded) { @@ -794,7 +804,7 @@ func chatStreamRunOnce( if trimmed := strings.TrimSpace(res.line); trimmed != "" { // 残帧也算一次处理尝试:走到下面统一行处理。 res.err = nil // 清掉让下方逻辑把残行当完整行处理 - if handleChatStreamLine(res.line, req, keepReasoning, clientWantsUsage, stats, &doneSeen, emitLine) { + if handleChatStreamLine(res.line, req, keepReasoning, clientWantsUsage, stats, &doneSeen, emitLine, wroteHeader) { // 残行内含 [DONE]:正常收尾 stats.Log(ctx, "chat") return true, nil @@ -832,7 +842,7 @@ func chatStreamRunOnce( continue } // 处理一行:返回 true 表示这一行刚好是 [DONE],流应正常收尾。 - if handleChatStreamLine(res.line, req, keepReasoning, clientWantsUsage, stats, &doneSeen, emitLine) { + if handleChatStreamLine(res.line, req, keepReasoning, clientWantsUsage, stats, &doneSeen, emitLine, wroteHeader) { stats.Log(ctx, "chat") return true, nil } @@ -850,7 +860,11 @@ func chatStreamRunOnce( // true 表示这一帧是 [DONE](流天然收尾),false 表示还需继续。已经经过 // convertStreamChunkWithUsage 的改写,逐条原样 emit 给客户端。 // -// 注意:emit 通过闭包完成,首次 emit 时由它去 WriteHeader(200) —— 即 +// wasCommitted 表示「本行处理之前 HTTP 头是否已发出」——只在已 commit +// 之后才允许 RecordChatUsage,否则 attempt 在 commit 前夭折时,Drive 的 +// 下一次 attempt 会重复统计同一笔上游 usage(double-count)。 +// +// emit 通过闭包完成,首次 emit 时由它去 WriteHeader(200) —— 即 // 「commit 点」在第一次实际写出时发生,而不是 handler 入口。 func handleChatStreamLine( line string, @@ -860,6 +874,7 @@ func handleChatStreamLine( stats *logging.StreamStats, doneSeen *bool, emitLine func(string), + wasCommitted bool, ) bool { trimmed := strings.TrimSpace(line) if trimmed == "data: [DONE]" { @@ -891,13 +906,14 @@ func handleChatStreamLine( out, usage := convertStreamChunkWithUsage(line, keepReasoning, clientWantsUsage) if out == "" { - // 空 choices chunk,但可能有 usage。 - if usage != nil { + // 空 choices chunk,但可能有 usage。仅在已 commit 后才记录 usage; + // 否则本次 attempt 由 retry 取消时,usage 仍会被错误地统计进账号。 + if usage != nil && wasCommitted { statsx.RecordChatUsage(req.Model, usage) } return false } - if usage != nil && !*doneSeen { + if usage != nil && !*doneSeen && wasCommitted { statsx.RecordChatUsage(req.Model, usage) } emitLine(out + "\n") diff --git a/internal/app/chat_to_anthropic.go b/internal/app/chat_to_anthropic.go index 5e252fe..8662f60 100644 --- a/internal/app/chat_to_anthropic.go +++ b/internal/app/chat_to_anthropic.go @@ -490,10 +490,13 @@ func forwardChatViaAnthropic(w http.ResponseWriter, r *http.Request, auth Upstre if req.Stream { // DriveStreamWithRetry 内部首轮调 callOnce 建立流；peek 失败（空流 // EOF/读错/首字节看门狗）时按 StreamEmptyRetryMax 换 key 重试。 - callOnce := func(callCtx context.Context) (io.ReadCloser, int, error) { + // 用 UpstreamErrorCapture 暂存非 2xx 响应,这样 retry 全失败后能 + // 把上游真实 status+body 透回去给客户端,而不是吞掉换成通用 502。 + upstreamCap := &UpstreamErrorCapture{} + callOnce := upstreamCap.WrapCallOnce(func(callCtx context.Context) (io.ReadCloser, int, error) { rc, status, _, err := callOpenCodeAnthropicEndpoint(callCtx, upstreamBody, req.Model, auth) return rc, status, err - } + }) runOnce := func(runCtx context.Context, rw http.ResponseWriter, rc io.Reader, peeked []streamReadResult, rd *streamReader) (bool, error) { return anthropicSSEToChatStream(runCtx, rw, rc, req.Model, keepReasoning, true, peeked, rd) } @@ -507,6 +510,10 @@ func forwardChatViaAnthropic(w http.ResponseWriter, r *http.Request, auth Upstre return } log.Warn("chat via anthropic stream empty after retries", "model", req.Model, "err", streamErr) + // 优先回写上游真实错误(4xx/5xx 的 status+body),比通用 502 更利于调试。 + if upstreamCap.WriteUpstreamErrorTo(w) { + return + } writeUpstreamError(w, http.StatusBadGateway, fmt.Errorf("upstream stream empty after retries"), "chat") return } @@ -711,10 +718,17 @@ func anthropicSSEToChatStream(ctx context.Context, w http.ResponseWriter, rc io. } if result.err != nil { // EOF / 读错误兜底。 - if !st.sentRole { - // message_start 都未到达 = 完全空流,从未 commit。 + // 关键不变量:一旦 writeHeaderOnce 触发(任何行非空就调),HTTP + // 头已发出,**绝不能返回 (false, ...)** 否则 DriveStreamWithRetry + // 会用同一个 ResponseWriter 二次 WriteHeader 并 retry,流被污染 + // (I4/I7)。message_start 未到但 header 已写的「伪 commit」场景: + // 让 st.finalize() 合成 finish 终止,对客户端是干净流末尾。 + if !wroteHeader { + // 真未 commit(message_start 都未达且一字未写)= 让 Drive retry。 + // 此分支只在 peeked 全空且首行就是 EOF 时进入。 return false, true, errStreamIncompleteNoCommit } + // 已写过任意字节(无论是否到 message_start) —— 收尾 finalize。 // 已有 role 输出后再 EOF:按finalize合成 stop chunk+[DONE], // 保证 OpenAI SDK 不挂起（幂等）。 st.finalize() diff --git a/internal/app/chat_to_responses_upstream.go b/internal/app/chat_to_responses_upstream.go index 8fd7ba9..ef1a6be 100644 --- a/internal/app/chat_to_responses_upstream.go +++ b/internal/app/chat_to_responses_upstream.go @@ -283,7 +283,8 @@ func forwardChatViaResponses(w http.ResponseWriter, r *http.Request, auth Upstre // 负责 Close),这里不能再 defer Close。 pending := rc - callOnce := func(ctx context.Context) (io.ReadCloser, int, error) { + upstreamCap := &UpstreamErrorCapture{} + callOnce := upstreamCap.WrapCallOnce(func(ctx context.Context) (io.ReadCloser, int, error) { if pending != nil { first := pending pending = nil @@ -291,7 +292,7 @@ func forwardChatViaResponses(w http.ResponseWriter, r *http.Request, auth Upstre } rc2, status2, _, err2 := callOpenCodeEndpoint(ctx, "responses", upstreamBody, req.Model, auth) return rc2, status2, err2 - } + }) runOnce := func(ctx context.Context, w http.ResponseWriter, rc io.Reader, peeked []streamReadResult, rd *streamReader) (bool, error) { return responsesSSEToChatStream(ctx, w, rc, req.Model, keepReasoning, true, peeked, rd) } @@ -303,6 +304,10 @@ func forwardChatViaResponses(w http.ResponseWriter, r *http.Request, auth Upstre // 客户端已离开:不要往已断开的连接再写错误。 return } + // 优先回写上游真实错误(4xx/5xx 的 status+body),比通用 502 更利于调试。 + if upstreamCap.WriteUpstreamErrorTo(w) { + return + } // 全部 attempt 都未 commit(空流 EOF / 上游首帧错误 / 首字节超时 // 等):此刻还没向客户端写过任何字节,可以安全地写一个 502 JSON。 if driveErr == nil { diff --git a/internal/app/responses.go b/internal/app/responses.go index 3b75fbc..60bd88d 100644 --- a/internal/app/responses.go +++ b/internal/app/responses.go @@ -1655,7 +1655,8 @@ func responsesHandler(w http.ResponseWriter, r *http.Request) { // 一个 502 JSON。 ctx := r.Context() pending := upResp - callOnce := func(c context.Context) (io.ReadCloser, int, error) { + upstreamCap := &UpstreamErrorCapture{} + callOnce := upstreamCap.WrapCallOnce(func(c context.Context) (io.ReadCloser, int, error) { if pending != nil { rc := pending pending = nil @@ -1663,7 +1664,7 @@ func responsesHandler(w http.ResponseWriter, r *http.Request) { } nrc, nstatus, _, nerr := callOpenCodeAPIStream(c, upstreamBody, chatReq.Model, auth) return nrc, nstatus, nerr - } + }) runOnce := func(c context.Context, w http.ResponseWriter, rc io.Reader, peeked []streamReadResult, rd *streamReader) (bool, error) { return responsesStreamHandler(w, r, rc, chatReq.Model, chatReq.Model, wantReasoning, respReq.Tools, respReq.ToolChoice, respReq, peeked, rd) } @@ -1673,6 +1674,10 @@ func responsesHandler(w http.ResponseWriter, r *http.Request) { return } logging.FromContext(ctx).Warn("responses translate stream exhausted retries", "model", chatReq.Model, "err", driveErr) + // 优先回写上游真实错误(4xx/5xx 的 status+body),比通用 502 更利于调试。 + if upstreamCap.WriteUpstreamErrorTo(w) { + return + } w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusBadGateway) json.NewEncoder(w).Encode(map[string]any{"error": map[string]any{"message": "upstream stream incomplete", "type": "upstream_error"}}) diff --git a/internal/app/responses_passthrough.go b/internal/app/responses_passthrough.go index 9ba2dd1..daa7abb 100644 --- a/internal/app/responses_passthrough.go +++ b/internal/app/responses_passthrough.go @@ -1010,8 +1010,11 @@ func relayResponsesStream(ctx context.Context, w http.ResponseWriter, rc io.Read // byte-level 透传：peek 已消费的字节保持原样直写 w,与主循环读取的 // 后续字节拼成完整 SSE 流。 + // 关键不变量:WriteHeader(status) 已发出 —— 此后绝对不能再返回 + // (false, ...) 让 Drive 二次 WriteHeader。flush 写错(连接已断)也 + // 同样视为已 commit。 if err := FlushPeekedBytes(w, peek.Consumed); err != nil { - return false, err + return true, err } if flusher != nil { flusher.Flush() @@ -1098,7 +1101,10 @@ func relayResponsesStream(ctx context.Context, w http.ResponseWriter, rc io.Read ) select { case <-ctx.Done(): - return false, ctx.Err() + // 已写过 WriteHeader + peeked 字节,不能再返回 (false, ...) 让 + // Drive 二次 WriteHeader / retry。返回 (true, ...) 透传 ctx + // 错误,Drive 看到 committed=true 立即结束循环。 + return true, ctx.Err() case res := <-reader.Read(): line = res.line err = res.err diff --git a/internal/app/stream_retry.go b/internal/app/stream_retry.go index 5522adb..c8ef96b 100644 --- a/internal/app/stream_retry.go +++ b/internal/app/stream_retry.go @@ -123,6 +123,87 @@ var ( // 错误 / 上游错误事件——尚未向客户端写过任何字节，调用方可安全重试。 var errStreamIncompleteNoCommit = errors.New("stream incomplete before first client byte") +// UpstreamErrorCapture 捕获最近一次非 2xx 响应的 (rc, status),让调用 +// 方在 Drive 返回 (!committed, ...) 后还能把真实 status+body 回写给客 +// 户端——而不是吞掉换成通用 502。 +// +// 使用模式: +// +// cap := &UpstreamErrorCapture{} +// committed, err := DriveStreamWithRetry(ctx, w, hooks, +// cap.WrapCallOnce(rawCallOnce), +// runOnce) +// if !committed { +// if cap.Status != 0 && cap.RC != nil { +// // write the buffered status+body to w +// } +// } +// +// WrapCallOnce 同时在「Drive 内部 close 不到非 2xx rc」的前提下兜底: +// 非 2xx 时不交给 Drive Close,由 WrapCallOnce 把 rc 暂存到字段里;调用 +// 方在后面读取并 close。 +type UpstreamErrorCapture struct { + RC io.ReadCloser + Status int +} + +// WrapCallOnce 包装一个原始 callOnce:首轮/重试都过Wrap。 +// - 进 Drive 的 rc 仅在 2xx 时由 Drive 拥有; +// - 非 2xx 时 Wrap 把 rc 暂存到 Cap,返回 (nil, status, nil)——Drive 看到 +// nil rc + 非 2xx status 就直接放弃(不再 close 一个已经是 nil 的 rc)。 +func (cap *UpstreamErrorCapture) WrapCallOnce( + callOnce func(ctx context.Context) (io.ReadCloser, int, error), +) func(ctx context.Context) (io.ReadCloser, int, error) { + return func(ctx context.Context) (io.ReadCloser, int, error) { + rc, status, err := callOnce(ctx) + if err != nil { + if rc != nil { + rc.Close() + } + return nil, status, err + } + if status < 200 || status >= 300 { + // 由 Wrap 持有 rc,Drive 不会 Close——调用方在 committed=false 后 + // 读出 body 回写给客户端,然后 Close。 + if cap.RC != nil { + cap.RC.Close() + } + cap.RC = rc + cap.Status = status + return nil, status, nil + } + return rc, status, nil + } +} + +// WriteUpstreamErrorTo 把捕获到的非 2xx 上游响应原样回写给客户端。返回 +// 是否成功表达了一段上游错误(若 cap 里没有捕获,返回 false)。调用方在 +// false 时应回退到自己的通用错误。 +func (cap *UpstreamErrorCapture) WriteUpstreamErrorTo(w http.ResponseWriter) bool { + if cap.RC == nil || cap.Status < 200 { + return false + } + defer cap.RC.Close() + errBody, _ := io.ReadAll(io.LimitReader(cap.RC, 32*1024*1024)) + status := cap.Status + if status < 100 || status >= 600 { + status = http.StatusBadGateway + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + if len(errBody) > 0 { + _, _ = w.Write(errBody) + } else { + _ = json.NewEncoder(w).Encode(map[string]any{ + "error": map[string]any{ + "message": fmt.Sprintf("upstream status %d", status), + "type": "upstream_error", + }, + }) + } + return true +} + // FlushPeekedBytes 把 peek 阶段攒的 SSE 行原样写回 w（行字节、换行符都 // 不动）。返回首个写错误。byte-level 透传路径用它把 peek 消费掉的字节 // 回放给客户端。 @@ -201,6 +282,10 @@ func PeekFirstFrame(ctx context.Context, rc io.Reader, timeout time.Duration, ho for { select { case <-ctx.Done(): + // 关键:必须 Close reader,否则它的 goroutine 会永阻塞在 readCh + // 上(readCh 已无接收方,<-r.done 又从未被 close)——每客户端断开 + // 一次的连接就 leak 一个 goroutine。 + reader.Close() return PeekOutcome{Err: ctx.Err()} case <-timeoutCh: reader.Close() @@ -274,9 +359,8 @@ func DriveStreamWithRetry( return false, err } if status < 200 || status >= 300 { - if rc != nil { - rc.Close() - } + // 不在这里 close rc —— 调用方可能还要把上游错误体透传给客户端。 + // 由调用方负责 close(或在 callOnce 内留好暂存句柄)。 return false, fmt.Errorf("upstream status %d on retry", status) } @@ -295,13 +379,15 @@ func DriveStreamWithRetry( logging.FromContext(ctx).Warn("stream empty before commit, retrying with next key", "attempt", attempt+1, "max_retry", maxRetry, "cause", runErr) nextRC, status, callErr := callOnce(ctx) - if callErr != nil || status < 200 || status >= 300 { + if callErr != nil { + // 调用方有 wrap 时它已 close;无 wrap 时这里 close 兜底。 if nextRC != nil { nextRC.Close() } - if callErr != nil { - return false, callErr - } + return false, callErr + } + if status < 200 || status >= 300 { + // 与顶部路径一致:不 close,让调用方 WrapCallOnce 处理。 return false, fmt.Errorf("upstream status %d on retry", status) } rc = nextRC From 7301f87fa0f3fc91278e4a93b19346b61c62b5c7 Mon Sep 17 00:00:00 2001 From: 6Kmfi6HP <179412085+6Kmfi6HP@users.noreply.github.com> Date: Mon, 28 Sep 2026 01:33:43 +0200 Subject: [PATCH 9/9] fix(chat): SSE frame separator must be \n\n, not \n MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pre-existing latent bug amplified by the chat retry refactor's emitLine path: each chunk was being terminated with a single \n, which is inside- frame continuation line boundary in SSE — OpenAI's strict client (python sdk ≥1.x) parses `data:` blocks separated by `\n\n` only and would loudly JSONDecodeError "Extra data" once a stream had more than one frame. Caught by running the new end-to-end OpenAI SDK E2E test against the real running gateway. Verified: 7/7 chat SDK tests now pass against a live upstream (mimo-v2.6-flash). Also tested: - launch claude → /v1/messages (anthropic frames incl. message_start, content_block, message_delta, message_stop) → works - launch codex exec → /v1/responses → works Co-Authored-By: Claude Code --- internal/app/chat.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/app/chat.go b/internal/app/chat.go index e717493..d3118cc 100644 --- a/internal/app/chat.go +++ b/internal/app/chat.go @@ -916,7 +916,9 @@ func handleChatStreamLine( if usage != nil && !*doneSeen && wasCommitted { statsx.RecordChatUsage(req.Model, usage) } - emitLine(out + "\n") + // SSE 帧分隔必须是 \n\n(\n 单换行是同一帧内的多行 data)。否则严格 + // 解析的 OpenAI SDK 会把多个 data 帧拼成一个,触发 "Extra data"。 + emitLine(out + "\n\n") return false }