fix(test): auto-acknowledge tool-result follow-ups in mock LLM server (#20528)

Author: kitlangton

.github/workflows/test.yml

@@ -100,9 +100,6 @@ jobs:
         run: bun --cwd packages/app test:e2e:local
         env:
           CI: true
-          OPENCODE_API_KEY: ${{ secrets.OPENCODE_API_KEY }}
-          OPENCODE_E2E_MODEL: opencode/claude-haiku-4-5
-          OPENCODE_E2E_REQUIRE_PAID: "true"
         timeout-minutes: 30
 
       - name: Upload Playwright artifacts

packages/app/e2e/fixtures.ts

@@ -15,6 +15,7 @@ import {
   waitSlug,
   waitSession,
 } from "./actions"
+import { openaiModel, withMockOpenAI } from "./prompt/mock"
 import { createSdk, dirSlug, getWorktree, sessionPath } from "./utils"
 
 type LLMFixture = {
@@ -47,6 +48,7 @@ type LLMFixture = {
   wait: (count: number) => Promise<void>
   inputs: () => Promise<Record<string, unknown>[]>
   pending: () => Promise<number>
+  misses: () => Promise<Array<{ url: URL; body: Record<string, unknown> }>>
 }
 
 export const settingsKey = "settings.v3"
@@ -83,6 +85,7 @@ type TestFixtures = {
   gotoSession: (sessionID?: string) => Promise<void>
   withProject: <T>(callback: (project: ProjectHandle) => Promise<T>, options?: ProjectOptions) => Promise<T>
   withBackendProject: <T>(callback: (project: ProjectHandle) => Promise<T>, options?: ProjectOptions) => Promise<T>
+  withMockProject: <T>(callback: (project: ProjectHandle) => Promise<T>, options?: ProjectOptions) => Promise<T>
 }
 
 type WorkerFixtures = {
@@ -132,6 +135,7 @@ export const test = base.extend<TestFixtures, WorkerFixtures>({
         wait: (count) => rt.runPromise(svc.wait(count)),
         inputs: () => rt.runPromise(svc.inputs),
         pending: () => rt.runPromise(svc.pending),
+        misses: () => rt.runPromise(svc.misses),
       })
     } finally {
       await rt.dispose()
@@ -193,6 +197,21 @@ export const test = base.extend<TestFixtures, WorkerFixtures>({
       runProject(page, callback, { ...options, serverUrl: backend.url, sdk: backend.sdk }),
     )
   },
+  withMockProject: async ({ page, llm, backend }, use) => {
+    await use((callback, options) =>
+      withMockOpenAI({
+        serverUrl: backend.url,
+        llmUrl: llm.url,
+        fn: () =>
+          runProject(page, callback, {
+            ...options,
+            model: options?.model ?? openaiModel,
+            serverUrl: backend.url,
+            sdk: backend.sdk,
+          }),
+      }),
+    )
+  },
 })
 
 async function runProject<T>(

packages/app/e2e/models/model-picker.spec.ts

@@ -2,7 +2,7 @@ import { test, expect } from "../fixtures"
 import { promptSelector } from "../selectors"
 import { clickListItem } from "../actions"
 
-test("smoke model selection updates prompt footer", async ({ page, gotoSession }) => {
+test.fixme("smoke model selection updates prompt footer", async ({ page, gotoSession }) => {
   await gotoSession()
 
   await page.locator(promptSelector).click()

packages/app/e2e/prompt/mock.ts

@@ -16,6 +16,16 @@ export function promptMatch(token: string) {
   return (hit: Hit) => bodyText(hit).includes(token)
 }
 
+/**
+ * Match requests whose body contains the exact serialized tool input.
+ * The seed prompts embed JSON.stringify(input) in the prompt text, which
+ * gets escaped again inside the JSON body — so we double-escape to match.
+ */
+export function inputMatch(input: unknown) {
+  const escaped = JSON.stringify(JSON.stringify(input)).slice(1, -1)
+  return (hit: Hit) => bodyText(hit).includes(escaped)
+}
+
 export async function withMockOpenAI<T>(input: { serverUrl: string; llmUrl: string; fn: () => Promise<T> }) {
   const sdk = createSdk(undefined, input.serverUrl)
   const prev = await sdk.global.config.get().then((res) => res.data ?? {})

packages/app/e2e/session/session-child-navigation.spec.ts

@@ -1,8 +1,9 @@
 import { seedSessionTask, withSession } from "../actions"
 import { test, expect } from "../fixtures"
+import { inputMatch } from "../prompt/mock"
 import { promptSelector } from "../selectors"
 
-test("task tool child-session link does not trigger stale show errors", async ({ page, withBackendProject }) => {
+test("task tool child-session link does not trigger stale show errors", async ({ page, llm, withMockProject }) => {
   test.setTimeout(120_000)
 
   const errs: string[] = []
@@ -12,12 +13,18 @@ test("task tool child-session link does not trigger stale show errors", async ({
   page.on("pageerror", onError)
 
   try {
-    await withBackendProject(async ({ gotoSession, trackSession, sdk }) => {
+    await withMockProject(async ({ gotoSession, trackSession, sdk }) => {
       await withSession(sdk, `e2e child nav ${Date.now()}`, async (session) => {
-        const child = await seedSessionTask(sdk, {
-          sessionID: session.id,
+        const taskInput = {
           description: "Open child session",
           prompt: "Search the repository for AssistantParts and then reply with exactly CHILD_OK.",
+          subagent_type: "general",
+        }
+        await llm.toolMatch(inputMatch(taskInput), "task", taskInput)
+        const child = await seedSessionTask(sdk, {
+          sessionID: session.id,
+          description: taskInput.description,
+          prompt: taskInput.prompt,
         })
         trackSession(child.sessionID)
 

packages/app/e2e/session/session-composer-dock.spec.ts

@@ -14,6 +14,7 @@ import {
   sessionTodoToggleButtonSelector,
 } from "../selectors"
 import { modKey } from "../utils"
+import { inputMatch } from "../prompt/mock"
 
 type Sdk = Parameters<typeof clearSessionDockSeed>[0]
 type PermissionRule = { permission: string; pattern: string; action: "allow" | "deny" | "ask" }
@@ -36,6 +37,17 @@ async function withDockSession<T>(
   }
 }
 
+const defaultQuestions = [
+  {
+    header: "Need input",
+    question: "Pick one option",
+    options: [
+      { label: "Continue", description: "Continue now" },
+      { label: "Stop", description: "Stop here" },
+    ],
+  },
+]
+
 test.setTimeout(120_000)
 
 async function withDockSeed<T>(sdk: Sdk, sessionID: string, fn: () => Promise<T>) {
@@ -291,27 +303,19 @@ test("auto-accept toggle works before first submit", async ({ page, withBackendP
   })
 })
 
-test("blocked question flow unblocks after submit", async ({ page, withBackendProject }) => {
-  await withBackendProject(async (project) => {
+test("blocked question flow unblocks after submit", async ({ page, llm, withMockProject }) => {
+  await withMockProject(async (project) => {
     await withDockSession(
       project.sdk,
       "e2e composer dock question",
       async (session) => {
         await withDockSeed(project.sdk, session.id, async () => {
           await project.gotoSession(session.id)
 
+          await llm.toolMatch(inputMatch({ questions: defaultQuestions }), "question", { questions: defaultQuestions })
           await seedSessionQuestion(project.sdk, {
             sessionID: session.id,
-            questions: [
-              {
-                header: "Need input",
-                question: "Pick one option",
-                options: [
-                  { label: "Continue", description: "Continue now" },
-                  { label: "Stop", description: "Stop here" },
-                ],
-              },
-            ],
+            questions: defaultQuestions,
           })
 
           const dock = page.locator(questionDockSelector)
@@ -328,27 +332,19 @@ test("blocked question flow unblocks after submit", async ({ page, withBackendPr
   })
 })
 
-test("blocked question flow supports keyboard shortcuts", async ({ page, withBackendProject }) => {
-  await withBackendProject(async (project) => {
+test("blocked question flow supports keyboard shortcuts", async ({ page, llm, withMockProject }) => {
+  await withMockProject(async (project) => {
     await withDockSession(
       project.sdk,
       "e2e composer dock question keyboard",
       async (session) => {
         await withDockSeed(project.sdk, session.id, async () => {
           await project.gotoSession(session.id)
 
+          await llm.toolMatch(inputMatch({ questions: defaultQuestions }), "question", { questions: defaultQuestions })
           await seedSessionQuestion(project.sdk, {
             sessionID: session.id,
-            questions: [
-              {
-                header: "Need input",
-                question: "Pick one option",
-                options: [
-                  { label: "Continue", description: "Continue now" },
-                  { label: "Stop", description: "Stop here" },
-                ],
-              },
-            ],
+            questions: defaultQuestions,
           })
 
           const dock = page.locator(questionDockSelector)
@@ -371,27 +367,19 @@ test("blocked question flow supports keyboard shortcuts", async ({ page, withBac
   })
 })
 
-test("blocked question flow supports escape dismiss", async ({ page, withBackendProject }) => {
-  await withBackendProject(async (project) => {
+test("blocked question flow supports escape dismiss", async ({ page, llm, withMockProject }) => {
+  await withMockProject(async (project) => {
     await withDockSession(
       project.sdk,
       "e2e composer dock question escape",
       async (session) => {
         await withDockSeed(project.sdk, session.id, async () => {
           await project.gotoSession(session.id)
 
+          await llm.toolMatch(inputMatch({ questions: defaultQuestions }), "question", { questions: defaultQuestions })
           await seedSessionQuestion(project.sdk, {
             sessionID: session.id,
-            questions: [
-              {
-                header: "Need input",
-                question: "Pick one option",
-                options: [
-                  { label: "Continue", description: "Continue now" },
-                  { label: "Stop", description: "Stop here" },
-                ],
-              },
-            ],
+            questions: defaultQuestions,
           })
 
           const dock = page.locator(questionDockSelector)
@@ -512,9 +500,20 @@ test("blocked permission flow supports allow always", async ({ page, withBackend
 
 test("child session question request blocks parent dock and unblocks after submit", async ({
   page,
-  withBackendProject,
+  llm,
+  withMockProject,
 }) => {
-  await withBackendProject(async (project) => {
+  const questions = [
+    {
+      header: "Child input",
+      question: "Pick one child option",
+      options: [
+        { label: "Continue", description: "Continue child" },
+        { label: "Stop", description: "Stop child" },
+      ],
+    },
+  ]
+  await withMockProject(async (project) => {
     await withDockSession(
       project.sdk,
       "e2e composer dock child question parent",
@@ -532,18 +531,10 @@ test("child session question request blocks parent dock and unblocks after submi
 
         try {
           await withDockSeed(project.sdk, child.id, async () => {
+            await llm.toolMatch(inputMatch({ questions }), "question", { questions })
             await seedSessionQuestion(project.sdk, {
               sessionID: child.id,
-              questions: [
-                {
-                  header: "Child input",
-                  question: "Pick one child option",
-                  options: [
-                    { label: "Continue", description: "Continue child" },
-                    { label: "Stop", description: "Stop child" },
-                  ],
-                },
-              ],
+              questions,
             })
 
             const dock = page.locator(questionDockSelector)
@@ -652,24 +643,26 @@ test("todo dock transitions and collapse behavior", async ({ page, withBackendPr
   })
 })
 
-test("keyboard focus stays off prompt while blocked", async ({ page, withBackendProject }) => {
-  await withBackendProject(async (project) => {
+test("keyboard focus stays off prompt while blocked", async ({ page, llm, withMockProject }) => {
+  const questions = [
+    {
+      header: "Need input",
+      question: "Pick one option",
+      options: [{ label: "Continue", description: "Continue now" }],
+    },
+  ]
+  await withMockProject(async (project) => {
     await withDockSession(
       project.sdk,
       "e2e composer dock keyboard",
       async (session) => {
         await withDockSeed(project.sdk, session.id, async () => {
           await project.gotoSession(session.id)
 
+          await llm.toolMatch(inputMatch({ questions }), "question", { questions })
           await seedSessionQuestion(project.sdk, {
             sessionID: session.id,
-            questions: [
-              {
-                header: "Need input",
-                question: "Pick one option",
-                options: [{ label: "Continue", description: "Continue now" }],
-              },
-            ],
+            questions,
           })
 
           await expectQuestionBlocked(page)