test(app): route prompt e2e through mock llm (#20383)

Author: kitlangton

packages/opencode/test/lib/llm-server.ts

@@ -8,6 +8,13 @@ export type Usage = { input: number; output: number }
 
 type Line = Record<string, unknown>
 
+type Flow =
+  | { type: "text"; text: string }
+  | { type: "reason"; text: string }
+  | { type: "tool-start"; id: string; name: string }
+  | { type: "tool-args"; text: string }
+  | { type: "usage"; usage: Usage }
+
 type Hit = {
   url: URL
   body: Record<string, unknown>
@@ -119,6 +126,276 @@ function bytes(input: Iterable<unknown>) {
   return Stream.fromIterable([...input].map(line)).pipe(Stream.encodeText)
 }
 
+function responseCreated(model: string) {
+  return {
+    type: "response.created",
+    sequence_number: 1,
+    response: {
+      id: "resp_test",
+      created_at: Math.floor(Date.now() / 1000),
+      model,
+      service_tier: null,
+    },
+  }
+}
+
+function responseCompleted(input: { seq: number; usage?: Usage }) {
+  return {
+    type: "response.completed",
+    sequence_number: input.seq,
+    response: {
+      incomplete_details: null,
+      service_tier: null,
+      usage: {
+        input_tokens: input.usage?.input ?? 0,
+        input_tokens_details: { cached_tokens: null },
+        output_tokens: input.usage?.output ?? 0,
+        output_tokens_details: { reasoning_tokens: null },
+      },
+    },
+  }
+}
+
+function responseMessage(id: string, seq: number) {
+  return {
+    type: "response.output_item.added",
+    sequence_number: seq,
+    output_index: 0,
+    item: { type: "message", id },
+  }
+}
+
+function responseText(id: string, text: string, seq: number) {
+  return {
+    type: "response.output_text.delta",
+    sequence_number: seq,
+    item_id: id,
+    delta: text,
+    logprobs: null,
+  }
+}
+
+function responseMessageDone(id: string, seq: number) {
+  return {
+    type: "response.output_item.done",
+    sequence_number: seq,
+    output_index: 0,
+    item: { type: "message", id },
+  }
+}
+
+function responseReason(id: string, seq: number) {
+  return {
+    type: "response.output_item.added",
+    sequence_number: seq,
+    output_index: 0,
+    item: { type: "reasoning", id, encrypted_content: null },
+  }
+}
+
+function responseReasonPart(id: string, seq: number) {
+  return {
+    type: "response.reasoning_summary_part.added",
+    sequence_number: seq,
+    item_id: id,
+    summary_index: 0,
+  }
+}
+
+function responseReasonText(id: string, text: string, seq: number) {
+  return {
+    type: "response.reasoning_summary_text.delta",
+    sequence_number: seq,
+    item_id: id,
+    summary_index: 0,
+    delta: text,
+  }
+}
+
+function responseReasonDone(id: string, seq: number) {
+  return {
+    type: "response.output_item.done",
+    sequence_number: seq,
+    output_index: 0,
+    item: { type: "reasoning", id, encrypted_content: null },
+  }
+}
+
+function responseTool(id: string, item: string, name: string, seq: number) {
+  return {
+    type: "response.output_item.added",
+    sequence_number: seq,
+    output_index: 0,
+    item: {
+      type: "function_call",
+      id: item,
+      call_id: id,
+      name,
+      arguments: "",
+      status: "in_progress",
+    },
+  }
+}
+
+function responseToolArgs(id: string, text: string, seq: number) {
+  return {
+    type: "response.function_call_arguments.delta",
+    sequence_number: seq,
+    output_index: 0,
+    item_id: id,
+    delta: text,
+  }
+}
+
+function responseToolDone(tool: { id: string; item: string; name: string; args: string }, seq: number) {
+  return {
+    type: "response.output_item.done",
+    sequence_number: seq,
+    output_index: 0,
+    item: {
+      type: "function_call",
+      id: tool.item,
+      call_id: tool.id,
+      name: tool.name,
+      arguments: tool.args,
+      status: "completed",
+    },
+  }
+}
+
+function choices(part: unknown) {
+  if (!part || typeof part !== "object") return
+  if (!("choices" in part) || !Array.isArray(part.choices)) return
+  const choice = part.choices[0]
+  if (!choice || typeof choice !== "object") return
+  return choice
+}
+
+function flow(item: Sse) {
+  const out: Flow[] = []
+  for (const part of [...item.head, ...item.tail]) {
+    const choice = choices(part)
+    const delta =
+      choice && "delta" in choice && choice.delta && typeof choice.delta === "object" ? choice.delta : undefined
+
+    if (delta && "content" in delta && typeof delta.content === "string") {
+      out.push({ type: "text", text: delta.content })
+    }
+
+    if (delta && "reasoning_content" in delta && typeof delta.reasoning_content === "string") {
+      out.push({ type: "reason", text: delta.reasoning_content })
+    }
+
+    if (delta && "tool_calls" in delta && Array.isArray(delta.tool_calls)) {
+      for (const tool of delta.tool_calls) {
+        if (!tool || typeof tool !== "object") continue
+        const fn = "function" in tool && tool.function && typeof tool.function === "object" ? tool.function : undefined
+        if ("id" in tool && typeof tool.id === "string" && fn && "name" in fn && typeof fn.name === "string") {
+          out.push({ type: "tool-start", id: tool.id, name: fn.name })
+        }
+        if (fn && "arguments" in fn && typeof fn.arguments === "string" && fn.arguments) {
+          out.push({ type: "tool-args", text: fn.arguments })
+        }
+      }
+    }
+
+    if (part && typeof part === "object" && "usage" in part && part.usage && typeof part.usage === "object") {
+      const raw = part.usage as Record<string, unknown>
+      if (typeof raw.prompt_tokens === "number" && typeof raw.completion_tokens === "number") {
+        out.push({
+          type: "usage",
+          usage: { input: raw.prompt_tokens, output: raw.completion_tokens },
+        })
+      }
+    }
+  }
+  return out
+}
+
+function responses(item: Sse, model: string) {
+  let seq = 1
+  let msg: string | undefined
+  let reason: string | undefined
+  let hasMsg = false
+  let hasReason = false
+  let call:
+    | {
+        id: string
+        item: string
+        name: string
+        args: string
+      }
+    | undefined
+  let usage: Usage | undefined
+  const lines: unknown[] = [responseCreated(model)]
+
+  for (const part of flow(item)) {
+    if (part.type === "text") {
+      msg ??= "msg_1"
+      if (!hasMsg) {
+        hasMsg = true
+        seq += 1
+        lines.push(responseMessage(msg, seq))
+      }
+      seq += 1
+      lines.push(responseText(msg, part.text, seq))
+      continue
+    }
+
+    if (part.type === "reason") {
+      reason ||= "rs_1"
+      if (!hasReason) {
+        hasReason = true
+        seq += 1
+        lines.push(responseReason(reason, seq))
+        seq += 1
+        lines.push(responseReasonPart(reason, seq))
+      }
+      seq += 1
+      lines.push(responseReasonText(reason, part.text, seq))
+      continue
+    }
+
+    if (part.type === "tool-start") {
+      call ||= { id: part.id, item: "fc_1", name: part.name, args: "" }
+      seq += 1
+      lines.push(responseTool(call.id, call.item, call.name, seq))
+      continue
+    }
+
+    if (part.type === "tool-args") {
+      if (!call) continue
+      call.args += part.text
+      seq += 1
+      lines.push(responseToolArgs(call.item, part.text, seq))
+      continue
+    }
+
+    usage = part.usage
+  }
+
+  if (msg) {
+    seq += 1
+    lines.push(responseMessageDone(msg, seq))
+  }
+  if (reason) {
+    seq += 1
+    lines.push(responseReasonDone(reason, seq))
+  }
+  if (call && !item.hang && !item.error) {
+    seq += 1
+    lines.push(responseToolDone(call, seq))
+  }
+  if (!item.hang && !item.error) lines.push(responseCompleted({ seq: seq + 1, usage }))
+  return { ...item, head: lines, tail: [] } satisfies Sse
+}
+
+function modelFrom(body: unknown) {
+  if (!body || typeof body !== "object") return "test-model"
+  if (!("model" in body) || typeof body.model !== "string") return "test-model"
+  return body.model
+}
+
 function send(item: Sse) {
   const head = bytes(item.head)
   const tail = bytes([...item.tail, ...(item.hang || item.error ? [] : [done])])
@@ -293,6 +570,13 @@ function item(input: Item | Reply) {
   return input instanceof Reply ? input.item() : input
 }
 
+function hit(url: string, body: unknown) {
+  return {
+    url: new URL(url, "http://localhost"),
+    body: body && typeof body === "object" ? (body as Record<string, unknown>) : {},
+  } satisfies Hit
+}
+
 namespace TestLLMServer {
   export interface Service {
     readonly url: string
@@ -342,30 +626,24 @@ export class TestLLMServer extends ServiceMap.Service<TestLLMServer, TestLLMServ
         return first
       }
 
-      yield* router.add(
-        "POST",
-        "/v1/chat/completions",
-        Effect.gen(function* () {
-          const req = yield* HttpServerRequest.HttpServerRequest
-          const next = pull()
-          if (!next) return HttpServerResponse.text("unexpected request", { status: 500 })
-          const body = yield* req.json.pipe(Effect.orElseSucceed(() => ({})))
-          hits = [
-            ...hits,
-            {
-              url: new URL(req.originalUrl, "http://localhost"),
-              body: body && typeof body === "object" ? (body as Record<string, unknown>) : {},
-            },
-          ]
-          yield* notify()
-          if (next.type === "sse" && next.reset) {
-            yield* reset(next)
-            return HttpServerResponse.empty()
-          }
-          if (next.type === "sse") return send(next)
-          return fail(next)
-        }),
-      )
+      const handle = Effect.fn("TestLLMServer.handle")(function* (mode: "chat" | "responses") {
+        const req = yield* HttpServerRequest.HttpServerRequest
+        const next = pull()
+        if (!next) return HttpServerResponse.text("unexpected request", { status: 500 })
+        const body = yield* req.json.pipe(Effect.orElseSucceed(() => ({})))
+        hits = [...hits, hit(req.originalUrl, body)]
+        yield* notify()
+        if (next.type !== "sse") return fail(next)
+        if (mode === "responses") return send(responses(next, modelFrom(body)))
+        if (next.reset) {
+          yield* reset(next)
+          return HttpServerResponse.empty()
+        }
+        return send(next)
+      })
+
+      yield* router.add("POST", "/v1/chat/completions", handle("chat"))
+      yield* router.add("POST", "/v1/responses", handle("responses"))
 
       yield* server.serve(router.asHttpEffect())