From 984a6cee3312fc5545f52643d8d81188895e8679 Mon Sep 17 00:00:00 2001 From: Frad LEE Date: Fri, 16 Oct 2026 10:33:49 +0800 Subject: [PATCH 1/4] test(e2e): support gemini in live runtime smoke tests --- .env.example | 3 +- e2e/README.md | 3 +- e2e/cases/public-api/runtime.spec.ts | 40 +++++++++++++++------- e2e/cases/ui/preview.spec.ts | 7 ++++ e2e/lib/env-preflight.ts | 21 ++++++++---- e2e/lib/setup-agent.ts | 41 +++++++++++++++++++++- scripts/e2e-env-preflight.test.ts | 51 ++++++++++++++++++++++++++++ 7 files changed, 144 insertions(+), 22 deletions(-) create mode 100644 scripts/e2e-env-preflight.test.ts diff --git a/.env.example b/.env.example index 8edcb94d..90d2c500 100644 --- a/.env.example +++ b/.env.example @@ -5,13 +5,14 @@ MOSOO_E2E_PROVIDER_API_KEY= MOSOO_E2E_OPENAI_API_KEY= MOSOO_E2E_ANTHROPIC_API_KEY= +MOSOO_E2E_GEMINI_API_KEY= MOSOO_E2E_OPENCODE_API_KEY= MOSOO_E2E_DEEPSEEK_API_KEY= OPENAI_API_KEY= ANTHROPIC_API_KEY= # Preview live smoke. -# Defaults to openai when unset; these cases accept openai or anthropic. +# Defaults to openai when unset; these cases accept openai, anthropic, or gemini. MOSOO_E2E_PROVIDER=openai # Public API runtime ACP overrides. diff --git a/e2e/README.md b/e2e/README.md index 47f3d9ca..f1a0ca72 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -12,12 +12,13 @@ The Playwright specs live in `e2e/cases`. Run them from the repository root with `ui/preview.spec.ts` and `public-api/runtime.spec.ts` call real models. Start the local stack with `just dev` first: Playwright on its own starts only the console, without the API. They sign in through the local `@mosoo.ai` development login, so they cannot target a deployed environment. -Choose the provider with `MOSOO_E2E_PROVIDER` and set its key, or the generic `MOSOO_E2E_PROVIDER_API_KEY`, which wins when both are set. Omitting the provider selects `openai`, so a DeepSeek or OpenCode key alone is not enough. +Choose the provider with `MOSOO_E2E_PROVIDER` and set its key, or the generic `MOSOO_E2E_PROVIDER_API_KEY`, which wins when both are set. Omitting the provider selects `openai`; a provider-specific key alone does not select that provider. | `MOSOO_E2E_PROVIDER` | Specs | Key | Notes | | -------------------- | -------------------------------------------------- | ----------------------------- | -------------------------------------------------------------------------------------------------------- | | `openai` (default) | `ui/preview.spec.ts`, `public-api/runtime.spec.ts` | `MOSOO_E2E_OPENAI_API_KEY` | | | `anthropic` | `ui/preview.spec.ts`, `public-api/runtime.spec.ts` | `MOSOO_E2E_ANTHROPIC_API_KEY` | | +| `gemini` | `ui/preview.spec.ts`, `public-api/runtime.spec.ts` | `MOSOO_E2E_GEMINI_API_KEY` | Uses OpenCode with the reviewed Gemini default and its public provider endpoint. | | `opencode` | `public-api/runtime.spec.ts` | `MOSOO_E2E_OPENCODE_API_KEY` | OpenCode Zen only | | `deepseek` | `public-api/runtime.spec.ts` | `MOSOO_E2E_DEEPSEEK_API_KEY` | Runs on OpenCode (`acp-fallback`); optional `MOSOO_E2E_DEEPSEEK_BASE_URL` and `MOSOO_E2E_DEEPSEEK_MODEL` | | `pi` | `public-api/runtime.spec.ts` | `MOSOO_E2E_PI_API_KEY` | Requires `MOSOO_E2E_PI_MODEL` and `MOSOO_E2E_PI_BASE_URL` for a custom Chat Completions endpoint | diff --git a/e2e/cases/public-api/runtime.spec.ts b/e2e/cases/public-api/runtime.spec.ts index 0c2d24b7..af2c03e4 100644 --- a/e2e/cases/public-api/runtime.spec.ts +++ b/e2e/cases/public-api/runtime.spec.ts @@ -1,14 +1,19 @@ import { PUBLIC_API_PREFIX } from "@mosoo/contracts/public-api"; -import { RUNTIME_CATALOG, listPresetModelsForVendor } from "@mosoo/runtime-catalog"; +import { + RUNTIME_CATALOG, + getDefaultModelIdForVendor, + listPresetModelsForVendor, +} from "@mosoo/runtime-catalog"; import { expect, test } from "@playwright/test"; import type { APIRequestContext } from "@playwright/test"; import { requirePiRuntimeSettings } from "../../lib/env-preflight"; -type ProviderId = "anthropic" | "deepseek" | "openai" | "opencode" | "pi"; +type ProviderId = "anthropic" | "deepseek" | "gemini" | "openai" | "opencode" | "pi"; type RuntimeCredentialVendorId = | "anthropic" | "deepseek" + | "gemini" | "openai" | "opencode" | "openai-compatible"; @@ -40,7 +45,12 @@ function readProviderId(): ProviderId { return "anthropic"; } - if (provider === "deepseek" || provider === "opencode" || provider === "pi") { + if ( + provider === "deepseek" || + provider === "gemini" || + provider === "opencode" || + provider === "pi" + ) { return provider; } @@ -58,9 +68,11 @@ function requireProviderApiKey(providerId: ProviderId): string { ? process.env["MOSOO_E2E_ANTHROPIC_API_KEY"]?.trim() : providerId === "deepseek" ? process.env["MOSOO_E2E_DEEPSEEK_API_KEY"]?.trim() - : providerId === "opencode" - ? process.env["MOSOO_E2E_OPENCODE_API_KEY"]?.trim() - : process.env["MOSOO_E2E_OPENAI_API_KEY"]?.trim()) || + : providerId === "gemini" + ? process.env["MOSOO_E2E_GEMINI_API_KEY"]?.trim() + : providerId === "opencode" + ? process.env["MOSOO_E2E_OPENCODE_API_KEY"]?.trim() + : process.env["MOSOO_E2E_OPENAI_API_KEY"]?.trim()) || ""; if (apiKey.length === 0) { @@ -124,7 +136,9 @@ function selectPresetRuntime(providerId: "anthropic" | "openai"): RuntimeSelecti }; } -function selectAcpFallbackRuntime(providerId: "deepseek" | "opencode"): RuntimeSelection { +function selectAcpFallbackRuntime( + providerId: "deepseek" | "gemini" | "opencode", +): RuntimeSelection { const runtimeId = readRuntimeIdOverride() ?? "acp-fallback"; const runtime = findPublicRuntime(runtimeId); @@ -139,10 +153,12 @@ function selectAcpFallbackRuntime(providerId: "deepseek" | "opencode"): RuntimeS const model = providerId === "deepseek" ? process.env["MOSOO_E2E_DEEPSEEK_MODEL"]?.trim() || DEFAULT_DEEPSEEK_MODEL - : process.env["MOSOO_E2E_OPENCODE_MODEL"]?.trim() || - listPresetModelsForVendor("opencode").find((entry) => - runtime.supportedModelIds.includes(entry.modelId), - )?.modelId; + : providerId === "gemini" + ? (getDefaultModelIdForVendor(providerId) ?? undefined) + : process.env["MOSOO_E2E_OPENCODE_MODEL"]?.trim() || + listPresetModelsForVendor("opencode").find((entry) => + runtime.supportedModelIds.includes(entry.modelId), + )?.modelId; if (model === undefined) { throw new Error( @@ -187,7 +203,7 @@ function getRuntimeSelection(providerId: ProviderId): RuntimeSelection { }; } - if (providerId === "deepseek" || providerId === "opencode") { + if (providerId === "deepseek" || providerId === "gemini" || providerId === "opencode") { return selectAcpFallbackRuntime(providerId); } diff --git a/e2e/cases/ui/preview.spec.ts b/e2e/cases/ui/preview.spec.ts index 6c07b459..ca2c8878 100644 --- a/e2e/cases/ui/preview.spec.ts +++ b/e2e/cases/ui/preview.spec.ts @@ -5,6 +5,7 @@ import { loginWithMosooAiBackdoor } from "../../lib/dev-auth"; import { requirePreviewRuntimeCredential } from "../../lib/env-preflight"; import { configureProviderCompanyKey, + configurePreviewRuntimeModel, createPreviewRuntimeAgent, createPreviewRunId, getPreviewSmokeEmail, @@ -79,6 +80,12 @@ test("Preview E2E smoke covers mosoo.ai login, blockers, stream, tool, pill, and providerId: runtimeCredential.providerId, runId, }); + if (runtimeCredential.modelId !== null) { + await configurePreviewRuntimeModel(page, agentId, { + modelId: runtimeCredential.modelId, + providerId: runtimeCredential.providerId, + }); + } await sendPreviewMessageAndVerifyRealStream(page, agentId); await verifyDiagnostics(page, agentId); }); diff --git a/e2e/lib/env-preflight.ts b/e2e/lib/env-preflight.ts index ef4cf88a..b2a645c0 100644 --- a/e2e/lib/env-preflight.ts +++ b/e2e/lib/env-preflight.ts @@ -1,4 +1,6 @@ -export type PreviewProviderId = "anthropic" | "openai"; +import { getDefaultModelIdForVendor } from "@mosoo/runtime-catalog"; + +export type PreviewProviderId = "anthropic" | "gemini" | "openai"; interface HarnessErrorInput { readonly fix: string; @@ -8,6 +10,7 @@ interface HarnessErrorInput { interface PreviewRuntimeCredential { apiKey: string; + modelId: string | null; providerId: PreviewProviderId; runtimeButtonName: string; } @@ -23,13 +26,13 @@ function readPreviewProviderId(): PreviewProviderId { return "openai"; } - if (provider === "anthropic") { - return "anthropic"; + if (provider === "anthropic" || provider === "gemini") { + return provider; } throw new Error( formatHarnessError({ - fix: "Set `MOSOO_E2E_PROVIDER=openai` or `MOSOO_E2E_PROVIDER=anthropic`.", + fix: "Set `MOSOO_E2E_PROVIDER=openai`, `MOSOO_E2E_PROVIDER=anthropic`, or `MOSOO_E2E_PROVIDER=gemini`.", what: `Preview live runtime smoke cannot start because MOSOO_E2E_PROVIDER=${provider} is unsupported.`, why: "The live harness must select a concrete public runtime provider before creating an agent.", }), @@ -41,7 +44,9 @@ function readProviderApiKey(providerId: PreviewProviderId): string { process.env["MOSOO_E2E_PROVIDER_API_KEY"]?.trim() || (providerId === "anthropic" ? process.env["MOSOO_E2E_ANTHROPIC_API_KEY"]?.trim() - : process.env["MOSOO_E2E_OPENAI_API_KEY"]?.trim()) || + : providerId === "gemini" + ? process.env["MOSOO_E2E_GEMINI_API_KEY"]?.trim() + : process.env["MOSOO_E2E_OPENAI_API_KEY"]?.trim()) || "" ); } @@ -53,7 +58,7 @@ export function requirePreviewRuntimeCredential(): PreviewRuntimeCredential { if (providerKey.length === 0) { throw new Error( formatHarnessError({ - fix: "Set `MOSOO_E2E_PROVIDER=anthropic MOSOO_E2E_PROVIDER_API_KEY=...`, or `MOSOO_E2E_PROVIDER=openai MOSOO_E2E_PROVIDER_API_KEY=...`.", + fix: "Set `MOSOO_E2E_PROVIDER` to the selected provider and supply `MOSOO_E2E_PROVIDER_API_KEY` or its provider-specific key.", what: "Preview live runtime smoke cannot start because the provider credential is missing.", why: "Live runtime smoke must run against the provider that owns the selected agent.", }), @@ -62,8 +67,10 @@ export function requirePreviewRuntimeCredential(): PreviewRuntimeCredential { return { apiKey: providerKey, + modelId: providerId === "gemini" ? getDefaultModelIdForVendor(providerId) : null, providerId, - runtimeButtonName: providerId === "anthropic" ? "Claude" : "OpenAI", + runtimeButtonName: + providerId === "anthropic" ? "Claude" : providerId === "gemini" ? "OpenCode" : "OpenAI", }; } diff --git a/e2e/lib/setup-agent.ts b/e2e/lib/setup-agent.ts index 63d7c8ed..ce34f94c 100644 --- a/e2e/lib/setup-agent.ts +++ b/e2e/lib/setup-agent.ts @@ -1,3 +1,4 @@ +import { getPresetModel } from "@mosoo/runtime-catalog"; import { expect } from "@playwright/test"; import type { Page } from "@playwright/test"; @@ -12,7 +13,45 @@ export function getPreviewSmokeEmail(runId: string): string { } function getProviderLabelPattern(providerId: PreviewProviderId): RegExp { - return providerId === "anthropic" ? /\bAnthropic\b/iu : /\bOpenAI\b/iu; + return providerId === "anthropic" + ? /\bAnthropic\b/iu + : providerId === "gemini" + ? /\bGemini\b/iu + : /\bOpenAI\b/iu; +} + +export async function configurePreviewRuntimeModel( + page: Page, + agentId: string, + input: { modelId: string; providerId: PreviewProviderId }, +): Promise { + const preset = getPresetModel({ modelId: input.modelId, vendorId: input.providerId }); + if (preset === null) throw new Error("Preview E2E requires an admitted preset model."); + + await page.goto(`/agent/${agentId}?tab=preview`); + const modelField = page.getByText("Model", { exact: true }).locator(".."); + await modelField.getByRole("button").click(); + const option = page.getByRole("menuitem").filter({ + hasText: `${preset.vendorLabel} ยท ${preset.modelId}`, + }); + await expect(option).toBeEnabled(); + const saved = page.waitForResponse((response) => { + if (!response.url().endsWith("/api/graphql")) return false; + const request = response.request().postDataJSON() as { + query?: string; + variables?: { input?: { model?: string } }; + }; + return ( + request.query?.includes("UpdateAgentConfig") === true && + request.variables?.input?.model === input.modelId + ); + }); + await option.click(); + const response = await saved; + expect(response.ok()).toBe(true); + expect(((await response.json()) as { errors?: unknown[] }).errors).toBeUndefined(); + await page.reload(); + await expect(modelField.getByRole("button")).toContainText(preset.displayName); } export async function createPreviewRuntimeAgent( diff --git a/scripts/e2e-env-preflight.test.ts b/scripts/e2e-env-preflight.test.ts new file mode 100644 index 00000000..67fd585d --- /dev/null +++ b/scripts/e2e-env-preflight.test.ts @@ -0,0 +1,51 @@ +import { afterEach, describe, expect, test } from "bun:test"; + +import { requirePreviewRuntimeCredential } from "../e2e/lib/env-preflight"; +import { getDefaultModelIdForVendor } from "../pkgs/runtime-catalog/src/runtime-catalog"; + +const keys = [ + "MOSOO_E2E_PROVIDER", + "MOSOO_E2E_PROVIDER_API_KEY", + "MOSOO_E2E_GEMINI_API_KEY", +] as const; +const original = new Map(keys.map((key) => [key, process.env[key]])); + +afterEach(() => { + for (const key of keys) { + const value = original.get(key); + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } +}); + +describe("Gemini live E2E preflight", () => { + test("selects the admitted Gemini model on OpenCode using its provider credential", () => { + // Given a Gemini credential and no generic override. + process.env.MOSOO_E2E_PROVIDER = "gemini"; + delete process.env.MOSOO_E2E_PROVIDER_API_KEY; + process.env.MOSOO_E2E_GEMINI_API_KEY = "fixture-credential"; + + // When Preview resolves its live runtime, then provider identity and + // the reviewed default are retained on the matching runtime. + expect(requirePreviewRuntimeCredential()).toMatchObject({ + apiKey: "fixture-credential", + modelId: getDefaultModelIdForVendor("gemini"), + providerId: "gemini", + runtimeButtonName: "OpenCode", + }); + }); + + test("keeps the generic credential override ahead of the Gemini-specific value", () => { + process.env.MOSOO_E2E_PROVIDER = "gemini"; + process.env.MOSOO_E2E_PROVIDER_API_KEY = "fixture-override"; + process.env.MOSOO_E2E_GEMINI_API_KEY = "fixture-credential"; + expect(requirePreviewRuntimeCredential().apiKey).toBe("fixture-override"); + }); + + test("rejects missing Gemini credentials before creating test resources", () => { + process.env.MOSOO_E2E_PROVIDER = "gemini"; + delete process.env.MOSOO_E2E_PROVIDER_API_KEY; + delete process.env.MOSOO_E2E_GEMINI_API_KEY; + expect(() => requirePreviewRuntimeCredential()).toThrow("provider credential is missing"); + }); +}); From ae259aeb9ad27b3927bff45a4f06d8de91ab559f Mon Sep 17 00:00:00 2001 From: Frad LEE Date: Tue, 20 Oct 2026 22:42:44 +0800 Subject: [PATCH 2/4] style(e2e): satisfy preview fixture lint rules --- e2e/cases/ui/preview-runtime-model.spec.ts | 3 ++- e2e/lib/setup-agent.ts | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/e2e/cases/ui/preview-runtime-model.spec.ts b/e2e/cases/ui/preview-runtime-model.spec.ts index 5aba0278..49fe78cf 100644 --- a/e2e/cases/ui/preview-runtime-model.spec.ts +++ b/e2e/cases/ui/preview-runtime-model.spec.ts @@ -56,7 +56,8 @@ for (const initial of [ const preset = getPresetModel(identity); if (preset === null) throw new Error("Expected a preset model."); return { - ...identity, + modelId: identity.modelId, + vendorId: identity.vendorId, available: true, displayName: preset.displayName, modelProtocol: preset.protocol, diff --git a/e2e/lib/setup-agent.ts b/e2e/lib/setup-agent.ts index 200b9157..9cdad1ab 100644 --- a/e2e/lib/setup-agent.ts +++ b/e2e/lib/setup-agent.ts @@ -63,7 +63,7 @@ export async function configurePreviewRuntimeModel( await expect(modelField.getByRole("button")).toContainText(preset.displayName); return; } - const [response] = await Promise.all([ + const [savedResponse] = await Promise.all([ page.waitForResponse((response) => { if (!response.url().endsWith("/api/graphql")) return false; const request = response.request().postDataJSON() as { @@ -79,8 +79,8 @@ export async function configurePreviewRuntimeModel( }), option.click(), ]); - expect(response.ok()).toBe(true); - expect(((await response.json()) as { errors?: unknown[] }).errors).toBeUndefined(); + expect(savedResponse.ok()).toBe(true); + expect(((await savedResponse.json()) as { errors?: unknown[] }).errors).toBeUndefined(); await page.reload(); await expect(modelField.getByRole("button")).toContainText(preset.displayName); } From c10fced0663ea9befbcc7fac0ff07490ba1bbf54 Mon Sep 17 00:00:00 2001 From: Frad LEE Date: Thu, 22 Oct 2026 17:13:37 +0800 Subject: [PATCH 3/4] test(e2e): verify preview model identity after reload --- e2e/README.md | 4 + e2e/cases/ui/preview-runtime-model.spec.ts | 221 ++++++++++++--------- e2e/lib/setup-agent.ts | 86 +++++--- 3 files changed, 191 insertions(+), 120 deletions(-) diff --git a/e2e/README.md b/e2e/README.md index f1a0ca72..1b33b8b0 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -8,12 +8,16 @@ The Playwright specs live in `e2e/cases`. Run them from the repository root with `deterministic/session-log.spec.ts` and every `ui` spec except `preview.spec.ts` need no provider keys or Worker bindings: they answer the console's GraphQL and auth-session requests with fixtures. Playwright reuses whatever already answers the base URL, `MOSOO_E2E_BASE_URL` or `http://127.0.0.1:$WEB_DEV_PORT` (port 5173) when it is unset, and otherwise starts only `@mosoo/web`. Screenshot specs write their review images under `.tmp/e2e/`. +`ui/preview-runtime-model.spec.ts` verifies the persisted Agent identity after reload, including rejected provider/model/Agent mismatches. Run it with `just e2e e2e/cases/ui/preview-runtime-model.spec.ts`. PR Check type-checks the E2E package; it does not execute Playwright. This fixture run needs no provider credentials. + ## Live specs `ui/preview.spec.ts` and `public-api/runtime.spec.ts` call real models. Start the local stack with `just dev` first: Playwright on its own starts only the console, without the API. They sign in through the local `@mosoo.ai` development login, so they cannot target a deployed environment. Choose the provider with `MOSOO_E2E_PROVIDER` and set its key, or the generic `MOSOO_E2E_PROVIDER_API_KEY`, which wins when both are set. Omitting the provider selects `openai`; a provider-specific key alone does not select that provider. +Live specs fail preflight when required credentials are missing. Their real model calls need a separate, explicitly configured run. + | `MOSOO_E2E_PROVIDER` | Specs | Key | Notes | | -------------------- | -------------------------------------------------- | ----------------------------- | -------------------------------------------------------------------------------------------------------- | | `openai` (default) | `ui/preview.spec.ts`, `public-api/runtime.spec.ts` | `MOSOO_E2E_OPENAI_API_KEY` | | diff --git a/e2e/cases/ui/preview-runtime-model.spec.ts b/e2e/cases/ui/preview-runtime-model.spec.ts index 49fe78cf..03086c5c 100644 --- a/e2e/cases/ui/preview-runtime-model.spec.ts +++ b/e2e/cases/ui/preview-runtime-model.spec.ts @@ -24,97 +24,140 @@ for (const initial of [ "already-selected", "same-model-other-provider", ] as const) { - test(`Preview model helper persists Gemini selection (${initial})`, async ({ page }) => { - test.setTimeout(30_000); - page.setDefaultTimeout(3_000); - await installConsoleFixtures(page, { locale: "en" }); - let agent = { - createdAt: now, - description: "Model selection fixture", - id: agentId, - liveVersion: null, - model: initial === "different-model" ? "deepseek-v4-pro" : modelId, - name: "Preview model selection", - owner: { id: CONSOLE_FIXTURE_IDS.accountId, imageUrl: null, name: "Fixture owner" }, - projectId: CONSOLE_FIXTURE_IDS.projectId, - prompt: "Use the selected model.", - provider: initial === "already-selected" ? "gemini" : "opencode", - runtimeId: "acp-fallback", - skills: [], - status: "draft", - tools: [], - updatedAt: now, - versions: [], - viewerRole: "owner", - visibility: "private", - }; - const writes: Record[] = []; - const entries = [ - { vendorId: "opencode", modelId: agent.model }, - { vendorId: "gemini", modelId }, - ].map((identity) => { - const preset = getPresetModel(identity); - if (preset === null) throw new Error("Expected a preset model."); - return { - modelId: identity.modelId, - vendorId: identity.vendorId, - available: true, - displayName: preset.displayName, - modelProtocol: preset.protocol, - reason: null, - source: "preset", - statusDetail: null, - statusLabel: "Available", - vendorLabel: preset.vendorLabel, + for (const persisted of [ + "matches", + "other-provider", + "missing-provider", + "other-model", + "other-agent", + ] as const) { + test(`Preview model helper checks persisted identity (${initial}, ${persisted})`, async ({ + page, + }) => { + test.setTimeout(60_000); + await installConsoleFixtures(page, { locale: "en" }); + let navigations = 0; + let reloadReads = 0; + page.on("request", (request) => { + if (request.isNavigationRequest() && request.frame() === page.mainFrame()) navigations += 1; + }); + let agent = { + createdAt: now, + description: "Model selection fixture", + id: agentId, + liveVersion: null, + model: initial === "different-model" ? "deepseek-v4-pro" : modelId, + name: "Preview model selection", + owner: { id: CONSOLE_FIXTURE_IDS.accountId, imageUrl: null, name: "Fixture owner" }, + projectId: CONSOLE_FIXTURE_IDS.projectId, + prompt: "Use the selected model.", + provider: initial === "already-selected" ? "gemini" : "opencode", + runtimeId: "acp-fallback", + skills: [], + status: "draft", + tools: [], + updatedAt: now, + versions: [], + viewerRole: "owner", + visibility: "private", }; - }); - await page.route("**/api/graphql", async (route) => { - const body = parseGraphQLRequestBody(route.request().postData()); - switch (getOperationName(body)) { - case "Agent": - await fulfillJson(route, { agent }); - return; - case "AgentEditorState": - await fulfillJson(route, { - agentEditorState: { - id: agentId, - builtInTools: [], - environment: { environmentId: null }, - mcpBindings: [], - packageResolution: null, - providerOptions: {}, - readiness: { checkedAt: now, issues: [], ready: true }, - }, - }); - return; - case "AgentSessionList": - await fulfillJson(route, { agentSessionList: { nodes: [] } }); - return; - case "AvailableAgentModels": - await fulfillJson(route, { availableAgentModels: entries }); - return; - case "UpdateAgentConfig": { - const input = body.variables?.["input"]; - if (!isRecord(input)) throw new Error("Expected an agent configuration input."); - writes.push(input); - agent = { - ...agent, - model: typeof input["model"] === "string" ? input["model"] : agent.model, - provider: typeof input["provider"] === "string" ? input["provider"] : agent.provider, - }; - await fulfillJson(route, { updateAgentConfig: agent }); - return; + const writes: Record[] = []; + const entries = [ + { vendorId: "opencode", modelId: agent.model }, + { vendorId: "gemini", modelId }, + ].map((identity) => { + const preset = getPresetModel(identity); + if (preset === null) throw new Error("Expected a preset model."); + return { + modelId: identity.modelId, + vendorId: identity.vendorId, + available: true, + displayName: preset.displayName, + modelProtocol: preset.protocol, + reason: null, + source: "preset", + statusDetail: null, + statusLabel: "Available", + vendorLabel: preset.vendorLabel, + }; + }); + await page.route("**/api/graphql", async (route) => { + const body = parseGraphQLRequestBody(route.request().postData()); + switch (getOperationName(body)) { + case "Agent": { + const reloaded = navigations > 1; + if (reloaded) reloadReads += 1; + // The mutation succeeds, but a fresh read can return a different + // identity. Gemini and OpenCode share the same model display name. + await fulfillJson(route, { + agent: reloaded + ? { + ...agent, + id: persisted === "other-agent" ? CONSOLE_FIXTURE_IDS.agentIds[1] : agent.id, + model: persisted === "other-model" ? "deepseek-v4-pro" : agent.model, + provider: + persisted === "missing-provider" + ? null + : persisted === "other-provider" + ? "opencode" + : agent.provider, + } + : agent, + }); + return; + } + case "AgentEditorState": + await fulfillJson(route, { + agentEditorState: { + id: agentId, + builtInTools: [], + environment: { environmentId: null }, + mcpBindings: [], + packageResolution: null, + providerOptions: {}, + readiness: { checkedAt: now, issues: [], ready: true }, + }, + }); + return; + case "AgentSessionList": + await fulfillJson(route, { agentSessionList: { nodes: [] } }); + return; + case "AvailableAgentModels": + await fulfillJson(route, { availableAgentModels: entries }); + return; + case "UpdateAgentConfig": { + const input = body.variables?.["input"]; + if (!isRecord(input)) throw new Error("Expected an agent configuration input."); + writes.push(input); + agent = { + ...agent, + model: typeof input["model"] === "string" ? input["model"] : agent.model, + provider: typeof input["provider"] === "string" ? input["provider"] : agent.provider, + }; + await fulfillJson(route, { updateAgentConfig: agent }); + return; + } + default: + await route.fallback(); } - default: - await route.fallback(); + }); + + const selection = configurePreviewRuntimeModel(page, agentId, { + modelId, + providerId: "gemini", + }); + if (persisted === "matches") { + await selection; + } else { + await expect(selection).rejects.toThrow( + "Reloaded Preview Agent must retain id, provider, and model", + ); } + expect(reloadReads).toBeGreaterThan(0); + expect(agent).toMatchObject({ model: modelId, provider: "gemini" }); + expect(writes).toHaveLength(initial === "already-selected" ? 0 : 1); + for (const input of writes) + expect(input).toMatchObject({ model: modelId, provider: "gemini" }); }); - - // Given the stored choice above, when the helper selects Gemini, then the - // exact provider/model persists through reload without a redundant write. - await configurePreviewRuntimeModel(page, agentId, { modelId, providerId: "gemini" }); - expect(agent).toMatchObject({ model: modelId, provider: "gemini" }); - expect(writes).toHaveLength(initial === "already-selected" ? 0 : 1); - for (const input of writes) expect(input).toMatchObject({ model: modelId, provider: "gemini" }); - }); + } } diff --git a/e2e/lib/setup-agent.ts b/e2e/lib/setup-agent.ts index 9cdad1ab..5d8c1756 100644 --- a/e2e/lib/setup-agent.ts +++ b/e2e/lib/setup-agent.ts @@ -28,17 +28,25 @@ export async function configurePreviewRuntimeModel( const preset = getPresetModel({ modelId: input.modelId, vendorId: input.providerId }); if (preset === null) throw new Error("Preview E2E requires an admitted preset model."); + const waitForAgentResponse = () => + page + .waitForRequest((request) => { + if (!request.url().endsWith("/api/graphql")) return false; + const body = request.postDataJSON() as { + query?: string; + variables?: { agentId?: string }; + }; + return ( + /\bquery\s+Agent\s*\(/u.test(body.query ?? "") && body.variables?.agentId === agentId + ); + }) + .then(async (request) => { + const response = await request.response(); + if (response === null) throw new Error("Preview Agent request did not receive a response."); + return response; + }); const [loaded] = await Promise.all([ - page.waitForResponse((response) => { - if (!response.url().endsWith("/api/graphql")) return false; - const request = response.request().postDataJSON() as { - query?: string; - variables?: { agentId?: string }; - }; - return ( - /\bquery\s+Agent\s*\(/u.test(request.query ?? "") && request.variables?.agentId === agentId - ); - }), + waitForAgentResponse(), page.goto(`/agent/${agentId}?tab=preview`), ]); expect(loaded.ok()).toBe(true); @@ -59,29 +67,45 @@ export async function configurePreviewRuntimeModel( initial.data.agent.provider === input.providerId ) { await page.keyboard.press("Escape"); - await page.reload(); - await expect(modelField.getByRole("button")).toContainText(preset.displayName); - return; + } else { + const [savedResponse] = await Promise.all([ + page.waitForResponse((response) => { + if (!response.url().endsWith("/api/graphql")) return false; + const request = response.request().postDataJSON() as { + query?: string; + variables?: { input?: { agentId?: string; model?: string; provider?: string } }; + }; + return ( + request.query?.includes("UpdateAgentConfig") === true && + request.variables?.input?.agentId === agentId && + request.variables.input.model === input.modelId && + request.variables.input.provider === input.providerId + ); + }), + option.click(), + ]); + expect(savedResponse.ok()).toBe(true); + expect(((await savedResponse.json()) as { errors?: unknown[] }).errors).toBeUndefined(); } - const [savedResponse] = await Promise.all([ - page.waitForResponse((response) => { - if (!response.url().endsWith("/api/graphql")) return false; - const request = response.request().postDataJSON() as { - query?: string; - variables?: { input?: { agentId?: string; model?: string; provider?: string } }; - }; - return ( - request.query?.includes("UpdateAgentConfig") === true && - request.variables?.input?.agentId === agentId && - request.variables.input.model === input.modelId && - request.variables.input.provider === input.providerId - ); - }), - option.click(), + // Watch requests from the new document so the mutation's refetch cannot + // satisfy the persistence check before navigation commits. + const [reloaded] = await Promise.all([ + page + .waitForEvent("framenavigated", (frame) => frame === page.mainFrame()) + .then(waitForAgentResponse), + page.reload(), ]); - expect(savedResponse.ok()).toBe(true); - expect(((await savedResponse.json()) as { errors?: unknown[] }).errors).toBeUndefined(); - await page.reload(); + expect(reloaded.ok()).toBe(true); + const persisted = (await reloaded.json()) as typeof initial; + expect(persisted.errors).toBeUndefined(); + expect( + persisted.data?.agent, + "Reloaded Preview Agent must retain id, provider, and model", + ).toMatchObject({ + id: agentId, + model: input.modelId, + provider: input.providerId, + }); await expect(modelField.getByRole("button")).toContainText(preset.displayName); } From 30dbdd16ccfbb32738d05605f6aa8d03c00264a7 Mon Sep 17 00:00:00 2001 From: Frad LEE Date: Sat, 24 Oct 2026 01:02:46 +0800 Subject: [PATCH 4/4] ci(e2e): run preview model persistence fixtures --- .github/workflows/pr-check.yml | 6 ++++++ e2e/README.md | 2 +- justfile | 4 ++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr-check.yml b/.github/workflows/pr-check.yml index 117f7d43..d7f1218e 100644 --- a/.github/workflows/pr-check.yml +++ b/.github/workflows/pr-check.yml @@ -46,5 +46,11 @@ jobs: - name: Run repository check run: just check + - name: Install fixture browser + run: just e2e-browser-install + + - name: Verify persisted Preview model selection + run: just e2e e2e/cases/ui/preview-runtime-model.spec.ts + - name: Verify native Host checkpoint recovery run: just test-native-host-checkpoint diff --git a/e2e/README.md b/e2e/README.md index 1b33b8b0..0b17a818 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -8,7 +8,7 @@ The Playwright specs live in `e2e/cases`. Run them from the repository root with `deterministic/session-log.spec.ts` and every `ui` spec except `preview.spec.ts` need no provider keys or Worker bindings: they answer the console's GraphQL and auth-session requests with fixtures. Playwright reuses whatever already answers the base URL, `MOSOO_E2E_BASE_URL` or `http://127.0.0.1:$WEB_DEV_PORT` (port 5173) when it is unset, and otherwise starts only `@mosoo/web`. Screenshot specs write their review images under `.tmp/e2e/`. -`ui/preview-runtime-model.spec.ts` verifies the persisted Agent identity after reload, including rejected provider/model/Agent mismatches. Run it with `just e2e e2e/cases/ui/preview-runtime-model.spec.ts`. PR Check type-checks the E2E package; it does not execute Playwright. This fixture run needs no provider credentials. +`ui/preview-runtime-model.spec.ts` verifies the persisted Agent identity after reload, including rejected provider/model/Agent mismatches. Run it with `just e2e e2e/cases/ui/preview-runtime-model.spec.ts`. PR Check installs Chromium with `just e2e-browser-install` and executes this fixture suite after the repository check. This run needs no provider credentials and creates no model Session. ## Live specs diff --git a/justfile b/justfile index 72c0e666..e94c7835 100644 --- a/justfile +++ b/justfile @@ -129,6 +129,10 @@ public-api-smoke: e2e *args: e2e/node_modules/.bin/playwright test --config e2e/playwright.config.ts {{args}} +# Install Chromium and its system dependencies for fixture browser checks. +e2e-browser-install: + e2e/node_modules/.bin/playwright install --with-deps chromium + # Update the Agent Driver submodule to upstream HEAD. driver-update: git submodule update --init --remote --checkout apps/driver