Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,14 @@
MOSOO_E2E_PROVIDER_API_KEY=
MOSOO_E2E_OPENAI_API_KEY=
MOSOO_E2E_ANTHROPIC_API_KEY=
MOSOO_E2E_GEMINI_API_KEY=
MOSOO_E2E_OPENCODE_API_KEY=
MOSOO_E2E_DEEPSEEK_API_KEY=
OPENAI_API_KEY=
ANTHROPIC_API_KEY=

# Preview live smoke.
# Defaults to openai when unset; these cases accept openai or anthropic.
# Defaults to openai when unset; these cases accept openai, anthropic, or gemini.
MOSOO_E2E_PROVIDER=openai

# Public API runtime ACP overrides.
Expand Down
6 changes: 6 additions & 0 deletions .github/workflows/pr-check.yml
Original file line number Diff line number Diff line change
Expand Up @@ -46,5 +46,11 @@ jobs:
- name: Run repository check
run: just check

- name: Install fixture browser
run: just e2e-browser-install

- name: Verify persisted Preview model selection
run: just e2e e2e/cases/ui/preview-runtime-model.spec.ts

- name: Verify native Host checkpoint recovery
run: just test-native-host-checkpoint
7 changes: 6 additions & 1 deletion e2e/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,16 +8,21 @@ The Playwright specs live in `e2e/cases`. Run them from the repository root with

`deterministic/session-log.spec.ts` and every `ui` spec except `preview.spec.ts` need no provider keys or Worker bindings: they answer the console's GraphQL and auth-session requests with fixtures. Playwright reuses whatever already answers the base URL, `MOSOO_E2E_BASE_URL` or `http://127.0.0.1:$WEB_DEV_PORT` (port 5173) when it is unset, and otherwise starts only `@mosoo/web`. Screenshot specs write their review images under `.tmp/e2e/`.

`ui/preview-runtime-model.spec.ts` verifies the persisted Agent identity after reload, including rejected provider/model/Agent mismatches. Run it with `just e2e e2e/cases/ui/preview-runtime-model.spec.ts`. PR Check installs Chromium with `just e2e-browser-install` and executes this fixture suite after the repository check. This run needs no provider credentials and creates no model Session.

## Live specs

`ui/preview.spec.ts` and `public-api/runtime.spec.ts` call real models. Start the local stack with `just dev` first: Playwright on its own starts only the console, without the API. They sign in through the local `@mosoo.ai` development login, so they cannot target a deployed environment.

Choose the provider with `MOSOO_E2E_PROVIDER` and set its key, or the generic `MOSOO_E2E_PROVIDER_API_KEY`, which wins when both are set. Omitting the provider selects `openai`, so a DeepSeek or OpenCode key alone is not enough.
Choose the provider with `MOSOO_E2E_PROVIDER` and set its key, or the generic `MOSOO_E2E_PROVIDER_API_KEY`, which wins when both are set. Omitting the provider selects `openai`; a provider-specific key alone does not select that provider.

Live specs fail preflight when required credentials are missing. Their real model calls need a separate, explicitly configured run.

| `MOSOO_E2E_PROVIDER` | Specs | Key | Notes |
| -------------------- | -------------------------------------------------- | ----------------------------- | -------------------------------------------------------------------------------------------------------- |
| `openai` (default) | `ui/preview.spec.ts`, `public-api/runtime.spec.ts` | `MOSOO_E2E_OPENAI_API_KEY` | |
| `anthropic` | `ui/preview.spec.ts`, `public-api/runtime.spec.ts` | `MOSOO_E2E_ANTHROPIC_API_KEY` | |
| `gemini` | `ui/preview.spec.ts`, `public-api/runtime.spec.ts` | `MOSOO_E2E_GEMINI_API_KEY` | Uses OpenCode with the reviewed Gemini default and its public provider endpoint. |
| `opencode` | `public-api/runtime.spec.ts` | `MOSOO_E2E_OPENCODE_API_KEY` | OpenCode Zen only |
| `deepseek` | `public-api/runtime.spec.ts` | `MOSOO_E2E_DEEPSEEK_API_KEY` | Runs on OpenCode (`acp-fallback`); optional `MOSOO_E2E_DEEPSEEK_BASE_URL` and `MOSOO_E2E_DEEPSEEK_MODEL` |
| `pi` | `public-api/runtime.spec.ts` | `MOSOO_E2E_PI_API_KEY` | Requires `MOSOO_E2E_PI_MODEL` and `MOSOO_E2E_PI_BASE_URL` for a custom Chat Completions endpoint |
Expand Down
40 changes: 28 additions & 12 deletions e2e/cases/public-api/runtime.spec.ts
Original file line number Diff line number Diff line change
@@ -1,14 +1,19 @@
import { PUBLIC_API_PREFIX } from "@mosoo/contracts/public-api";
import { RUNTIME_CATALOG, listPresetModelsForVendor } from "@mosoo/runtime-catalog";
import {
RUNTIME_CATALOG,
getDefaultModelIdForVendor,
listPresetModelsForVendor,
} from "@mosoo/runtime-catalog";
import { expect, test } from "@playwright/test";
import type { APIRequestContext } from "@playwright/test";

import { requirePiRuntimeSettings } from "../../lib/env-preflight";

type ProviderId = "anthropic" | "deepseek" | "openai" | "opencode" | "pi";
type ProviderId = "anthropic" | "deepseek" | "gemini" | "openai" | "opencode" | "pi";
type RuntimeCredentialVendorId =
| "anthropic"
| "deepseek"
| "gemini"
| "openai"
| "opencode"
| "openai-compatible";
Expand Down Expand Up @@ -40,7 +45,12 @@ function readProviderId(): ProviderId {
return "anthropic";
}

if (provider === "deepseek" || provider === "opencode" || provider === "pi") {
if (
provider === "deepseek" ||
provider === "gemini" ||
provider === "opencode" ||
provider === "pi"
) {
return provider;
}

Expand All @@ -58,9 +68,11 @@ function requireProviderApiKey(providerId: ProviderId): string {
? process.env["MOSOO_E2E_ANTHROPIC_API_KEY"]?.trim()
: providerId === "deepseek"
? process.env["MOSOO_E2E_DEEPSEEK_API_KEY"]?.trim()
: providerId === "opencode"
? process.env["MOSOO_E2E_OPENCODE_API_KEY"]?.trim()
: process.env["MOSOO_E2E_OPENAI_API_KEY"]?.trim()) ||
: providerId === "gemini"
? process.env["MOSOO_E2E_GEMINI_API_KEY"]?.trim()
: providerId === "opencode"
? process.env["MOSOO_E2E_OPENCODE_API_KEY"]?.trim()
: process.env["MOSOO_E2E_OPENAI_API_KEY"]?.trim()) ||
"";

if (apiKey.length === 0) {
Expand Down Expand Up @@ -124,7 +136,9 @@ function selectPresetRuntime(providerId: "anthropic" | "openai"): RuntimeSelecti
};
}

function selectAcpFallbackRuntime(providerId: "deepseek" | "opencode"): RuntimeSelection {
function selectAcpFallbackRuntime(
providerId: "deepseek" | "gemini" | "opencode",
): RuntimeSelection {
const runtimeId = readRuntimeIdOverride() ?? "acp-fallback";
const runtime = findPublicRuntime(runtimeId);

Expand All @@ -139,10 +153,12 @@ function selectAcpFallbackRuntime(providerId: "deepseek" | "opencode"): RuntimeS
const model =
providerId === "deepseek"
? process.env["MOSOO_E2E_DEEPSEEK_MODEL"]?.trim() || DEFAULT_DEEPSEEK_MODEL
: process.env["MOSOO_E2E_OPENCODE_MODEL"]?.trim() ||
listPresetModelsForVendor("opencode").find((entry) =>
runtime.supportedModelIds.includes(entry.modelId),
)?.modelId;
: providerId === "gemini"
? (getDefaultModelIdForVendor(providerId) ?? undefined)
: process.env["MOSOO_E2E_OPENCODE_MODEL"]?.trim() ||
listPresetModelsForVendor("opencode").find((entry) =>
runtime.supportedModelIds.includes(entry.modelId),
)?.modelId;

if (model === undefined) {
throw new Error(
Expand Down Expand Up @@ -187,7 +203,7 @@ function getRuntimeSelection(providerId: ProviderId): RuntimeSelection {
};
}

if (providerId === "deepseek" || providerId === "opencode") {
if (providerId === "deepseek" || providerId === "gemini" || providerId === "opencode") {
return selectAcpFallbackRuntime(providerId);
}

Expand Down
163 changes: 163 additions & 0 deletions e2e/cases/ui/preview-runtime-model.spec.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,163 @@
import { getDefaultModelIdForVendor, getPresetModel } from "@mosoo/runtime-catalog";
import { expect, test } from "@playwright/test";

import {
CONSOLE_FIXTURE_IDS,
fulfillJson,
getOperationName,
installConsoleFixtures,
parseGraphQLRequestBody,
} from "../../lib/console-fixtures";
import { configurePreviewRuntimeModel } from "../../lib/setup-agent";

const agentId = CONSOLE_FIXTURE_IDS.agentIds[0];
const now = "2026-10-20T00:00:00Z";
const modelId = getDefaultModelIdForVendor("gemini");
if (modelId === null) throw new Error("Gemini needs a reviewed default model.");

function isRecord(value: unknown): value is Record<string, unknown> {
return value !== null && typeof value === "object" && !Array.isArray(value);
}

for (const initial of [
"different-model",
"already-selected",
"same-model-other-provider",
] as const) {
for (const persisted of [
"matches",
"other-provider",
"missing-provider",
"other-model",
"other-agent",
] as const) {
test(`Preview model helper checks persisted identity (${initial}, ${persisted})`, async ({
page,
}) => {
test.setTimeout(60_000);
await installConsoleFixtures(page, { locale: "en" });
let navigations = 0;
let reloadReads = 0;
page.on("request", (request) => {
if (request.isNavigationRequest() && request.frame() === page.mainFrame()) navigations += 1;
});
let agent = {
createdAt: now,
description: "Model selection fixture",
id: agentId,
liveVersion: null,
model: initial === "different-model" ? "deepseek-v4-pro" : modelId,
name: "Preview model selection",
owner: { id: CONSOLE_FIXTURE_IDS.accountId, imageUrl: null, name: "Fixture owner" },
projectId: CONSOLE_FIXTURE_IDS.projectId,
prompt: "Use the selected model.",
provider: initial === "already-selected" ? "gemini" : "opencode",
runtimeId: "acp-fallback",
skills: [],
status: "draft",
tools: [],
updatedAt: now,
versions: [],
viewerRole: "owner",
visibility: "private",
};
const writes: Record<string, unknown>[] = [];
const entries = [
{ vendorId: "opencode", modelId: agent.model },
{ vendorId: "gemini", modelId },
].map((identity) => {
const preset = getPresetModel(identity);
if (preset === null) throw new Error("Expected a preset model.");
return {
modelId: identity.modelId,
vendorId: identity.vendorId,
available: true,
displayName: preset.displayName,
modelProtocol: preset.protocol,
reason: null,
source: "preset",
statusDetail: null,
statusLabel: "Available",
vendorLabel: preset.vendorLabel,
};
});
await page.route("**/api/graphql", async (route) => {
const body = parseGraphQLRequestBody(route.request().postData());
switch (getOperationName(body)) {
case "Agent": {
const reloaded = navigations > 1;
if (reloaded) reloadReads += 1;
// The mutation succeeds, but a fresh read can return a different
// identity. Gemini and OpenCode share the same model display name.
await fulfillJson(route, {
agent: reloaded
? {
...agent,
id: persisted === "other-agent" ? CONSOLE_FIXTURE_IDS.agentIds[1] : agent.id,
model: persisted === "other-model" ? "deepseek-v4-pro" : agent.model,
provider:
persisted === "missing-provider"
? null
: persisted === "other-provider"
? "opencode"
: agent.provider,
}
: agent,
});
return;
}
case "AgentEditorState":
await fulfillJson(route, {
agentEditorState: {
id: agentId,
builtInTools: [],
environment: { environmentId: null },
mcpBindings: [],
packageResolution: null,
providerOptions: {},
readiness: { checkedAt: now, issues: [], ready: true },
},
});
return;
case "AgentSessionList":
await fulfillJson(route, { agentSessionList: { nodes: [] } });
return;
case "AvailableAgentModels":
await fulfillJson(route, { availableAgentModels: entries });
return;
case "UpdateAgentConfig": {
const input = body.variables?.["input"];
if (!isRecord(input)) throw new Error("Expected an agent configuration input.");
writes.push(input);
agent = {
...agent,
model: typeof input["model"] === "string" ? input["model"] : agent.model,
provider: typeof input["provider"] === "string" ? input["provider"] : agent.provider,
};
await fulfillJson(route, { updateAgentConfig: agent });
return;
}
default:
await route.fallback();
}
});

const selection = configurePreviewRuntimeModel(page, agentId, {
modelId,
providerId: "gemini",
});
if (persisted === "matches") {
await selection;
} else {
await expect(selection).rejects.toThrow(
"Reloaded Preview Agent must retain id, provider, and model",
);
}
expect(reloadReads).toBeGreaterThan(0);
expect(agent).toMatchObject({ model: modelId, provider: "gemini" });
expect(writes).toHaveLength(initial === "already-selected" ? 0 : 1);
for (const input of writes)
expect(input).toMatchObject({ model: modelId, provider: "gemini" });
});
}
}
7 changes: 7 additions & 0 deletions e2e/cases/ui/preview.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import { loginWithMosooAiBackdoor } from "../../lib/dev-auth";
import { requirePreviewRuntimeCredential } from "../../lib/env-preflight";
import {
configureProviderCompanyKey,
configurePreviewRuntimeModel,
createPreviewRuntimeAgent,
createPreviewRunId,
getPreviewSmokeEmail,
Expand Down Expand Up @@ -79,6 +80,12 @@ test("Preview E2E smoke covers mosoo.ai login, blockers, stream, tool, pill, and
providerId: runtimeCredential.providerId,
runId,
});
if (runtimeCredential.modelId !== null) {
await configurePreviewRuntimeModel(page, agentId, {
modelId: runtimeCredential.modelId,
providerId: runtimeCredential.providerId,
});
}
await sendPreviewMessageAndVerifyRealStream(page, agentId);
await verifyDiagnostics(page, agentId);
});
21 changes: 14 additions & 7 deletions e2e/lib/env-preflight.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
export type PreviewProviderId = "anthropic" | "openai";
import { getDefaultModelIdForVendor } from "@mosoo/runtime-catalog";

export type PreviewProviderId = "anthropic" | "gemini" | "openai";

interface HarnessErrorInput {
readonly fix: string;
Expand All @@ -8,6 +10,7 @@ interface HarnessErrorInput {

interface PreviewRuntimeCredential {
apiKey: string;
modelId: string | null;
providerId: PreviewProviderId;
runtimeButtonName: string;
}
Expand All @@ -23,13 +26,13 @@ function readPreviewProviderId(): PreviewProviderId {
return "openai";
}

if (provider === "anthropic") {
return "anthropic";
if (provider === "anthropic" || provider === "gemini") {
return provider;
}

throw new Error(
formatHarnessError({
fix: "Set `MOSOO_E2E_PROVIDER=openai` or `MOSOO_E2E_PROVIDER=anthropic`.",
fix: "Set `MOSOO_E2E_PROVIDER=openai`, `MOSOO_E2E_PROVIDER=anthropic`, or `MOSOO_E2E_PROVIDER=gemini`.",
what: `Preview live runtime smoke cannot start because MOSOO_E2E_PROVIDER=${provider} is unsupported.`,
why: "The live harness must select a concrete public runtime provider before creating an agent.",
}),
Expand All @@ -41,7 +44,9 @@ function readProviderApiKey(providerId: PreviewProviderId): string {
process.env["MOSOO_E2E_PROVIDER_API_KEY"]?.trim() ||
(providerId === "anthropic"
? process.env["MOSOO_E2E_ANTHROPIC_API_KEY"]?.trim()
: process.env["MOSOO_E2E_OPENAI_API_KEY"]?.trim()) ||
: providerId === "gemini"
? process.env["MOSOO_E2E_GEMINI_API_KEY"]?.trim()
: process.env["MOSOO_E2E_OPENAI_API_KEY"]?.trim()) ||
""
);
}
Expand All @@ -53,7 +58,7 @@ export function requirePreviewRuntimeCredential(): PreviewRuntimeCredential {
if (providerKey.length === 0) {
throw new Error(
formatHarnessError({
fix: "Set `MOSOO_E2E_PROVIDER=anthropic MOSOO_E2E_PROVIDER_API_KEY=...`, or `MOSOO_E2E_PROVIDER=openai MOSOO_E2E_PROVIDER_API_KEY=...`.",
fix: "Set `MOSOO_E2E_PROVIDER` to the selected provider and supply `MOSOO_E2E_PROVIDER_API_KEY` or its provider-specific key.",
what: "Preview live runtime smoke cannot start because the provider credential is missing.",
why: "Live runtime smoke must run against the provider that owns the selected agent.",
}),
Expand All @@ -62,8 +67,10 @@ export function requirePreviewRuntimeCredential(): PreviewRuntimeCredential {

return {
apiKey: providerKey,
modelId: providerId === "gemini" ? getDefaultModelIdForVendor(providerId) : null,
providerId,
runtimeButtonName: providerId === "anthropic" ? "Claude" : "OpenAI",
runtimeButtonName:
providerId === "anthropic" ? "Claude" : providerId === "gemini" ? "OpenCode" : "OpenAI",
};
}

Expand Down
Loading
Loading