test(app): add mock llm e2e fixture (#20375)

kitlangton · web-flow · commit c8ecd6402203 · 2026-03-31T21:24:39.000-04:00
diff --git a/packages/app/e2e/fixtures.ts b/packages/app/e2e/fixtures.ts
@@ -1,5 +1,8 @@
 import { test as base, expect, type Page } from "@playwright/test"
+import { ManagedRuntime } from "effect"
 import type { E2EWindow } from "../src/testing/terminal"
+import type { Item, Reply, Usage } from "../../opencode/test/lib/llm-server"
+import { TestLLMServer } from "../../opencode/test/lib/llm-server"
 import {
   healthPhase,
   cleanupSession,
@@ -13,6 +16,24 @@ import {
 } from "./actions"
 import { createSdk, dirSlug, getWorktree, sessionPath } from "./utils"
 
+type LLMFixture = {
+  url: string
+  push: (...input: (Item | Reply)[]) => Promise<void>
+  text: (value: string, opts?: { usage?: Usage }) => Promise<void>
+  tool: (name: string, input: unknown) => Promise<void>
+  toolHang: (name: string, input: unknown) => Promise<void>
+  reason: (value: string, opts?: { text?: string; usage?: Usage }) => Promise<void>
+  fail: (message?: unknown) => Promise<void>
+  error: (status: number, body: unknown) => Promise<void>
+  hang: () => Promise<void>
+  hold: (value: string, wait: PromiseLike<unknown>) => Promise<void>
+  hits: () => Promise<Array<{ url: URL; body: Record<string, unknown> }>>
+  calls: () => Promise<number>
+  wait: (count: number) => Promise<void>
+  inputs: () => Promise<Record<string, unknown>[]>
+  pending: () => Promise<number>
+}
+
 export const settingsKey = "settings.v3"
 
 const seedModel = (() => {
@@ -26,6 +47,7 @@ const seedModel = (() => {
 })()
 
 type TestFixtures = {
+  llm: LLMFixture
   sdk: ReturnType<typeof createSdk>
   gotoSession: (sessionID?: string) => Promise<void>
   withProject: <T>(
@@ -36,7 +58,11 @@ type TestFixtures = {
       trackSession: (sessionID: string, directory?: string) => void
       trackDirectory: (directory: string) => void
     }) => Promise<T>,
-    options?: { extra?: string[] },
+    options?: {
+      extra?: string[]
+      model?: { providerID: string; modelID: string }
+      setup?: (directory: string) => Promise<void>
+    },
   ) => Promise<T>
 }
 
@@ -46,6 +72,31 @@ type WorkerFixtures = {
 }
 
 export const test = base.extend<TestFixtures, WorkerFixtures>({
+  llm: async ({}, use) => {
+    const rt = ManagedRuntime.make(TestLLMServer.layer)
+    try {
+      const svc = await rt.runPromise(TestLLMServer.asEffect())
+      await use({
+        url: svc.url,
+        push: (...input) => rt.runPromise(svc.push(...input)),
+        text: (value, opts) => rt.runPromise(svc.text(value, opts)),
+        tool: (name, input) => rt.runPromise(svc.tool(name, input)),
+        toolHang: (name, input) => rt.runPromise(svc.toolHang(name, input)),
+        reason: (value, opts) => rt.runPromise(svc.reason(value, opts)),
+        fail: (message) => rt.runPromise(svc.fail(message)),
+        error: (status, body) => rt.runPromise(svc.error(status, body)),
+        hang: () => rt.runPromise(svc.hang),
+        hold: (value, wait) => rt.runPromise(svc.hold(value, wait)),
+        hits: () => rt.runPromise(svc.hits),
+        calls: () => rt.runPromise(svc.calls),
+        wait: (count) => rt.runPromise(svc.wait(count)),
+        inputs: () => rt.runPromise(svc.inputs),
+        pending: () => rt.runPromise(svc.pending),
+      })
+    } finally {
+      await rt.dispose()
+    }
+  },
   page: async ({ page }, use) => {
     let boundary: string | undefined
     setHealthPhase(page, "test")
@@ -99,7 +150,8 @@ export const test = base.extend<TestFixtures, WorkerFixtures>({
       const root = await createTestProject()
       const sessions = new Map<string, string>()
       const dirs = new Set<string>()
-      await seedStorage(page, { directory: root, extra: options?.extra })
+      await options?.setup?.(root)
+      await seedStorage(page, { directory: root, extra: options?.extra, model: options?.model })
 
       const gotoSession = async (sessionID?: string) => {
         await page.goto(sessionPath(root, sessionID))
@@ -133,7 +185,14 @@ export const test = base.extend<TestFixtures, WorkerFixtures>({
   },
 })
 
-async function seedStorage(page: Page, input: { directory: string; extra?: string[] }) {
+async function seedStorage(
+  page: Page,
+  input: {
+    directory: string
+    extra?: string[]
+    model?: { providerID: string; modelID: string }
+  },
+) {
   await seedProjects(page, input)
   await page.addInitScript((model: { providerID: string; modelID: string }) => {
     const win = window as E2EWindow
@@ -158,7 +217,7 @@ async function seedStorage(page: Page, input: { directory: string; extra?: strin
         variant: {},
       }),
     )
-  }, seedModel)
+  }, input.model ?? seedModel)
 }
 
 export { expect }
diff --git a/packages/app/e2e/prompt/prompt.spec.ts b/packages/app/e2e/prompt/prompt.spec.ts
@@ -1,8 +1,44 @@
+import fs from "node:fs/promises"
+import path from "node:path"
 import { test, expect } from "../fixtures"
 import { promptSelector } from "../selectors"
-import { cleanupSession, sessionIDFromUrl, withSession } from "../actions"
+import { sessionIDFromUrl } from "../actions"
+import { createSdk } from "../utils"
 
-test("can send a prompt and receive a reply", async ({ page, sdk, gotoSession }) => {
+async function config(dir: string, url: string) {
+  await fs.writeFile(
+    path.join(dir, "opencode.json"),
+    JSON.stringify({
+      $schema: "https://opencode.ai/config.json",
+      enabled_providers: ["e2e-llm"],
+      provider: {
+        "e2e-llm": {
+          name: "E2E LLM",
+          npm: "@ai-sdk/openai-compatible",
+          env: [],
+          models: {
+            "test-model": {
+              name: "Test Model",
+              tool_call: true,
+              limit: { context: 128000, output: 32000 },
+            },
+          },
+          options: {
+            apiKey: "test-key",
+            baseURL: url,
+          },
+        },
+      },
+      agent: {
+        build: {
+          model: "e2e-llm/test-model",
+        },
+      },
+    }),
+  )
+}
+
+test("can send a prompt and receive a reply", async ({ page, llm, withProject }) => {
   test.setTimeout(120_000)
 
   const pageErrors: string[] = []
@@ -11,42 +47,51 @@ test("can send a prompt and receive a reply", async ({ page, sdk, gotoSession })
   }
   page.on("pageerror", onPageError)
 
-  await gotoSession()
-
-  const token = `E2E_OK_${Date.now()}`
+  try {
+    await withProject(
+      async (project) => {
+        const sdk = createSdk(project.directory)
+        const token = `E2E_OK_${Date.now()}`
 
-  const prompt = page.locator(promptSelector)
-  await prompt.click()
-  await page.keyboard.type(`Reply with exactly: ${token}`)
-  await page.keyboard.press("Enter")
+        await llm.text(token)
+        await project.gotoSession()
 
-  await expect(page).toHaveURL(/\/session\/[^/?#]+/, { timeout: 30_000 })
+        const prompt = page.locator(promptSelector)
+        await prompt.click()
+        await page.keyboard.type(`Reply with exactly: ${token}`)
+        await page.keyboard.press("Enter")
 
-  const sessionID = (() => {
-    const id = sessionIDFromUrl(page.url())
-    if (!id) throw new Error(`Failed to parse session id from url: ${page.url()}`)
-    return id
-  })()
+        await expect(page).toHaveURL(/\/session\/[^/?#]+/, { timeout: 30_000 })
 
-  try {
-    await expect
-      .poll(
-        async () => {
-          const messages = await sdk.session.messages({ sessionID, limit: 50 }).then((r) => r.data ?? [])
-          return messages
-            .filter((m) => m.info.role === "assistant")
-            .flatMap((m) => m.parts)
-            .filter((p) => p.type === "text")
-            .map((p) => p.text)
-            .join("\n")
-        },
-        { timeout: 90_000 },
-      )
+        const sessionID = (() => {
+          const id = sessionIDFromUrl(page.url())
+          if (!id) throw new Error(`Failed to parse session id from url: ${page.url()}`)
+          return id
+        })()
+        project.trackSession(sessionID)
 
-      .toContain(token)
+        await expect
+          .poll(
+            async () => {
+              const messages = await sdk.session.messages({ sessionID, limit: 50 }).then((r) => r.data ?? [])
+              return messages
+                .filter((m) => m.info.role === "assistant")
+                .flatMap((m) => m.parts)
+                .filter((p) => p.type === "text")
+                .map((p) => p.text)
+                .join("\n")
+            },
+            { timeout: 30_000 },
+          )
+          .toContain(token)
+      },
+      {
+        model: { providerID: "e2e-llm", modelID: "test-model" },
+        setup: (dir) => config(dir, llm.url),
+      },
+    )
   } finally {
     page.off("pageerror", onPageError)
-    await cleanupSession({ sdk, sessionID })
   }
 
   if (pageErrors.length > 0) {
