From 4e7301759179f1eba623cdc508f4cfef3ad27a4c Mon Sep 17 00:00:00 2001 From: FireTable Date: Sat, 25 Jul 2026 18:15:56 +0800 Subject: [PATCH 01/44] feat(eval): complete prompt A/B testing studio, trace logging, and LLM-as-a-Judge evaluation platform --- app/admin/admin-tabs.tsx | 1911 +----------- app/admin/eval/eval-dashboard-client.tsx | 1264 ++++++++ app/admin/eval/page.tsx | 30 + app/admin/providers/providers-client.tsx | 901 ++++++ app/admin/roles/roles-client.tsx | 320 ++ app/admin/types.ts | 75 + app/admin/users/users-client.tsx | 546 ++++ app/api/eval/assignments/route.ts | 79 + app/api/eval/feedback/route.ts | 36 + app/api/eval/judge/route.ts | 49 + app/api/eval/prompts/route.ts | 115 + app/api/eval/rubrics/route.ts | 82 + app/api/eval/runs/[id]/route.ts | 40 + app/api/eval/runs/compare/route.ts | 49 + backend/agent.ts | 5 +- backend/agent/chat-agent.ts | 6 +- backend/agent/code-agent.ts | 17 +- backend/agent/crypto-agent.ts | 17 +- backend/agent/eval-agent.ts | 114 + backend/agent/weather-agent.ts | 16 +- backend/callbacks.ts | 2 + backend/node/kb/chunk-alignment-node.ts | 5 +- backend/node/kb/chunk-extract-node.ts | 6 +- backend/node/kb/page-to-markdown-node.ts | 5 +- backend/node/prepare-data-node.ts | 22 +- backend/node/rename-thread-agent-node.ts | 11 +- backend/node/router-agent-node.ts | 6 +- backend/node/thread-summarize-node.ts | 8 +- backend/prompt/loader.ts | 50 + backend/state.ts | 4 + components/assistant-ui/thread.tsx | 49 + db/migrations/0017_rapid_kabuki.sql | 100 + db/migrations/0018_outstanding_gressill.sql | 1 + db/migrations/meta/0017_snapshot.json | 2927 ++++++++++++++++++ db/migrations/meta/0018_snapshot.json | 2934 +++++++++++++++++++ db/migrations/meta/_journal.json | 14 + db/schema.ts | 1 + docs/APIS.md | 30 + docs/DB.md | 41 +- docs/EVALUATION.md | 87 + langgraph.json | 4 +- lib/eval/callback.ts | 130 + lib/eval/queries.ts | 305 ++ lib/eval/schema.ts | 177 ++ lib/provider/model-registry.ts | 13 + lib/provider/schema.ts | 2 +- scripts/db-migrate.ts | 6 + tests/api/eval/feedback.test.ts | 74 + tests/lib/eval/queries.test.ts | 163 ++ 49 files changed, 10886 insertions(+), 1963 deletions(-) create mode 100644 app/admin/eval/eval-dashboard-client.tsx create mode 100644 app/admin/eval/page.tsx create mode 100644 app/admin/providers/providers-client.tsx create mode 100644 app/admin/roles/roles-client.tsx create mode 100644 app/admin/types.ts create mode 100644 app/admin/users/users-client.tsx create mode 100644 app/api/eval/assignments/route.ts create mode 100644 app/api/eval/feedback/route.ts create mode 100644 app/api/eval/judge/route.ts create mode 100644 app/api/eval/prompts/route.ts create mode 100644 app/api/eval/rubrics/route.ts create mode 100644 app/api/eval/runs/[id]/route.ts create mode 100644 app/api/eval/runs/compare/route.ts create mode 100644 backend/agent/eval-agent.ts create mode 100644 backend/prompt/loader.ts create mode 100644 db/migrations/0017_rapid_kabuki.sql create mode 100644 db/migrations/0018_outstanding_gressill.sql create mode 100644 db/migrations/meta/0017_snapshot.json create mode 100644 db/migrations/meta/0018_snapshot.json create mode 100644 docs/EVALUATION.md create mode 100644 lib/eval/callback.ts create mode 100644 lib/eval/queries.ts create mode 100644 lib/eval/schema.ts create mode 100644 tests/api/eval/feedback.test.ts create mode 100644 tests/lib/eval/queries.test.ts diff --git a/app/admin/admin-tabs.tsx b/app/admin/admin-tabs.tsx index ec11331e..b61c37d0 100644 --- a/app/admin/admin-tabs.tsx +++ b/app/admin/admin-tabs.tsx @@ -1,120 +1,14 @@ "use client"; -import { useEffect, useState, useTransition } from "react"; import { useRouter, useSearchParams } from "next/navigation"; - -import { Loader2 } from "lucide-react"; - import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; -import { Button } from "@/components/ui/button"; -import { Input } from "@/components/ui/input"; -import { Badge } from "@/components/ui/badge"; -import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; -import { Checkbox } from "@/components/ui/checkbox"; -import { Separator } from "@/components/ui/separator"; -import { Switch } from "@/components/ui/switch"; -import { - Dialog, - DialogContent, - DialogDescription, - DialogFooter, - DialogHeader, - DialogTitle, -} from "@/components/ui/dialog"; -import { FormDialog } from "@/components/ui/form-dialog"; -import { - Select, - SelectContent, - SelectItem, - SelectTrigger, - SelectValue, -} from "@/components/ui/select"; -import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip"; -import { Avatar, AvatarImage, AvatarFallback } from "@/components/ui/avatar"; -import { toast } from "sonner"; - -type PublicProviderApiKey = { name: string }; -type ModelKind = "chat" | "ocr" | "embed" | "extract" | "rerank"; -type PublicModel = { - name: string; - enabled: boolean; - inputPer1k: number; - outputPer1k: number; - // ponytail: server-side default ["chat"] means an older row may not - // carry the field at all. Fall back to chat at render time so the - // table shows a value for every model, regardless of seed-vs-new. - kind?: ModelKind[]; -}; -type PublicProviderRow = { - id: string; - name: string; - enabled: boolean; - baseUrl: string; - apiKeys: PublicProviderApiKey[]; - models: PublicModel[]; - createdAt: string; - updatedAt: string; -}; -type RoleRow = { - id: string; - name: string; - creditLimit: number | null; - windowHours: number; - createdAt: string; - updatedAt: string; -}; +import { ProvidersPanel } from "@/app/admin/providers/providers-client"; +import { RolesPanel } from "@/app/admin/roles/roles-client"; +import { UsersPanel } from "@/app/admin/users/users-client"; +import { EvalDashboardClient } from "@/app/admin/eval/eval-dashboard-client"; +import type { PublicProviderRow, RoleRow, UserRow } from "@/app/admin/types"; -// ponytail: module-scope constant so add-mode `initialKind ?? ["chat"]` -// keeps the same array reference across renders — `useEffect` dep -// comparison uses Object.is, and a fresh literal each render would loop -// forever (Next dev StrictMode catches it as "Maximum update depth"). -const DEFAULT_KIND: ModelKind[] = ["chat"]; -// ponytail: server-side join shape — user + role name inlined so the -// table can render "Admin" / "User" labels without a second round-trip. -// `roleName` is null when the FK points at a missing role (defensive — -// FK prevents it, but a leftJoin can still null it on race). -type UserRow = { - id: string; - name: string | null; - email: string; - image?: string | null; - emailVerified: boolean; - roleId: string; - roleName: string | null; - banned: boolean; - createdAt: string; - updatedAt: string; - todayCredits?: number; - todayInputTokens?: number; - todayOutputTokens?: number; - todayTokens?: number; - totalCredits?: number; - totalInputTokens?: number; - totalOutputTokens?: number; - totalTokens?: number; -}; - -async function jsonFetch( - url: string, - init?: RequestInit, -): Promise<{ ok: boolean; status: number; body: T | { code: string; message?: string } }> { - const res = await fetch(url, { - ...init, - headers: { "Content-Type": "application/json", ...init?.headers }, - }); - const body = (await res.json().catch(() => ({}))) as T | { code: string; message?: string }; - return { ok: res.ok, status: res.status, body }; -} - -function errMsg(body: unknown): string { - if (typeof body === "object" && body !== null) { - const m = (body as { message?: string }).message; - const c = (body as { code?: string }).code; - if (m) return m; - if (c) return c; - } - return "request failed"; -} +export type { PublicProviderRow, RoleRow, UserRow }; export function AdminTabs({ providers, @@ -129,7 +23,7 @@ export function AdminTabs({ const searchParams = useSearchParams(); const tabParam = searchParams.get("tab"); const activeTab = - tabParam && ["providers", "roles", "users"].includes(tabParam) ? tabParam : "providers"; + tabParam && ["providers", "roles", "users", "eval"].includes(tabParam) ? tabParam : "providers"; const handleTabChange = (val: string) => { const params = new URLSearchParams(searchParams.toString()); @@ -143,6 +37,7 @@ export function AdminTabs({ Providers Roles Users + Eval & A/B @@ -153,1791 +48,11 @@ export function AdminTabs({ - - ); -} - -function ProvidersPanel({ initial }: { initial: PublicProviderRow[] }) { - const router = useRouter(); - const [adding, setAdding] = useState(false); - - return ( -
-
-
-

Providers

-

- LLM providers, their base URL, models, and API keys. -

-
- -
- - {initial.length === 0 ? ( - - - No providers yet. - - - ) : ( - initial.map((p) => ) - )} - - setAdding(false)} - onSaved={() => { - setAdding(false); - router.refresh(); - }} - /> -
- ); -} - -function ProviderCard({ provider }: { provider: PublicProviderRow }) { - const router = useRouter(); - const [pending, start] = useTransition(); - const [confirmingDelete, setConfirmingDelete] = useState(false); - const [editing, setEditing] = useState(false); - - const confirmRemove = () => { - start(async () => { - const r = await jsonFetch(`/api/admin/providers/${encodeURIComponent(provider.id)}`, { - method: "DELETE", - }); - if (!r.ok) { - toast.error(errMsg(r.body)); - return; - } - setConfirmingDelete(false); - toast.success("provider deleted"); - router.refresh(); - }); - }; - - return ( - // ponytail: muted background for disabled cards so the list separates - // "live" from "kept around but off" at a glance. - - -
-
-
- {provider.name} - - {provider.enabled ? "Enabled" : "Disabled"} - -
- - {provider.id} - · - {new Date(provider.createdAt).toLocaleDateString("en-CA")} - -
-
- - {/* ponytail: the default provider is the seed row — at least - one provider must always exist for the system to boot, - so we hard-disable Delete on id="default". The backend - enforces the same rule (DELETE /[id]/route returns 409 - for default) so a tampered client can't slip past. */} - {provider.id === "default" ? ( - - - - - - - - Default provider — at least one is required. - - - ) : ( - - )} -
-
- - -
- - Base URL - - {provider.baseUrl} -
- - - - - - - setEditing(false)} - onSaved={() => { - setEditing(false); - router.refresh(); - }} - /> - - !open && setConfirmingDelete(false)}> - - - Delete this provider? - - “{provider.name}” and all of its models and API keys will be removed. This cannot be - undone. - - - - - - - - - - ); -} - -function ModelsSection({ providerId, models }: { providerId: string; models: PublicModel[] }) { - const router = useRouter(); - const [pending, start] = useTransition(); - const [pendingDelete, setPendingDelete] = useState(null); - const [editing, setEditing] = useState(null); - const [adding, setAdding] = useState(false); - - const confirmRemove = () => { - if (!pendingDelete) return; - const target = pendingDelete; - start(async () => { - const r = await jsonFetch( - `/api/admin/providers/${encodeURIComponent(providerId)}/models/${encodeURIComponent(target.name)}`, - { method: "DELETE" }, - ); - if (!r.ok) { - toast.error(errMsg(r.body)); - return; - } - setPendingDelete(null); - router.refresh(); - }); - }; - - return ( -
-
-

Models

- -
-
- - - - - - - - - - - - {models.length === 0 ? ( - - - - ) : ( - models.map((m) => ( - - - - - - - - - )) - )} - -
NameStatusKindInput / 1kOutput / 1k -
- No models configured. -
{m.name} - - {m.enabled ? "Enabled" : "Disabled"} - - -
- {(m.kind ?? ["chat"]).map((k) => ( - - {k.charAt(0).toUpperCase() + k.slice(1)} - - ))} -
- 		{m.inputPer1k}{m.outputPer1k} -
- - -
-
-
- - !open && setPendingDelete(null)} - > - - - Delete this model? - - “{pendingDelete?.name}” will be removed from this provider. This cannot be undone. - - - - - - - - - - {editing ? ( - setEditing(null)} - onSaved={() => { - setEditing(null); - router.refresh(); - }} - /> - ) : null} - - setAdding(false)} - onSaved={() => { - setAdding(false); - router.refresh(); - }} - /> -
- ); -} - -function ModelDialog( - props: - | { - mode: "add"; - open: boolean; - providerId: string; - pending: boolean; - onClose: () => void; - onSaved: () => void; - } - | { - mode: "edit"; - open?: boolean; - model: PublicModel; - providerId: string; - pending: boolean; - onClose: () => void; - onSaved: () => void; - }, -) { - const isEdit = props.mode === "edit"; - const initialName = isEdit ? props.model.name : ""; - const initialEnabled = isEdit ? props.model.enabled : true; - const initialIn = isEdit ? String(props.model.inputPer1k) : "0.001"; - const initialOut = isEdit ? String(props.model.outputPer1k) : "0.002"; - // ponytail: backend defaults kind to ["chat"] on POST when omitted, - // so the add-mode local seed mirrors the server contract and we don't - // need to send `kind` in the body. Edit-mode seeds from existing row. - const initialKind: ModelKind[] = isEdit ? (props.model.kind ?? DEFAULT_KIND) : DEFAULT_KIND; - - const [name, setName] = useState(initialName); - const [enabled, setEnabled] = useState(initialEnabled); - const [inputPer1k, setInputPer1k] = useState(initialIn); - const [outputPer1k, setOutputPer1k] = useState(initialOut); - const [kind, setKind] = useState(initialKind); - const [saving, start] = useTransition(); - - useEffect(() => { - setName(initialName); - setEnabled(initialEnabled); - setInputPer1k(initialIn); - setOutputPer1k(initialOut); - setKind(initialKind); - }, [initialName, initialEnabled, initialIn, initialOut, initialKind]); - - const toggleKind = (k: ModelKind, checked: boolean) => { - setKind((prev) => { - const next = checked ? Array.from(new Set([...prev, k])) : prev.filter((x) => x !== k); - // ponytail: at least one kind must stay selected — unchecking the - // last box is a no-op so the user can never save a model with - // kind = [] (which the API rejects with 400 anyway). - return next.length === 0 ? prev : next; - }); - }; - - const save = () => { - if (!name.trim()) { - toast.error("model name required"); - return; - } - const inp = Number(inputPer1k); - const out = Number(outputPer1k); - if (!Number.isFinite(inp) || inp < 0 || !Number.isFinite(out) || out < 0) { - toast.error("rates must be non-negative numbers"); - return; - } - const path = isEdit - ? `/api/admin/providers/${encodeURIComponent(props.providerId)}/models/${encodeURIComponent(props.model.name)}` - : `/api/admin/providers/${encodeURIComponent(props.providerId)}/models`; - const method = isEdit ? "PATCH" : "POST"; - const body = isEdit - ? { name: name.trim(), enabled, inputPer1k: inp, outputPer1k: out, kind } - : { name: name.trim(), enabled, inputPer1k: inp, outputPer1k: out, kind }; - start(async () => { - const r = await jsonFetch(path, { method, body: JSON.stringify(body) }); - if (!r.ok) { - toast.error(errMsg(r.body)); - return; - } - toast.success(isEdit ? "model updated" : "model added"); - props.onSaved(); - }); - }; - - return ( - !o && props.onClose()} - title={isEdit ? "Edit model" : "Add model"} - description={ - isEdit - ? `${props.model.name} — rename takes effect on Save; collision with another model on this provider is rejected.` - : "Register a new model for this provider. The name is the JSONB-array key; you can rename it later from the Edit dialog." - } - submitLabel={isEdit ? "Save" : "Add"} - pending={saving || props.pending} - onSubmit={save} - onCancel={props.onClose} - > -
- - - - -
- Kind -
- {(["chat", "ocr", "embed", "extract", "rerank"] as const).map((k) => ( - - ))} -
-

- At least one kind is required — chat for general inference, ocr for PDF vision, embed - for KB chunk vectors, extract for structured-output triples from chunks, rerank for - secondary retrieval reranking. -

-
-
- - ); -} - -function KeysSection({ providerId, keys }: { providerId: string; keys: PublicProviderApiKey[] }) { - const router = useRouter(); - const [pending, start] = useTransition(); - const [pendingDelete, setPendingDelete] = useState(null); - const [editing, setEditing] = useState(null); - const [adding, setAdding] = useState(false); - - const confirmRemove = () => { - if (!pendingDelete) return; - const target = pendingDelete; - start(async () => { - const r = await jsonFetch(`/api/admin/providers/${encodeURIComponent(providerId)}/keys`, { - method: "DELETE", - body: JSON.stringify({ name: target.name }), - }); - if (!r.ok) { - toast.error(errMsg(r.body)); - return; - } - setPendingDelete(null); - router.refresh(); - }); - }; - - return ( -
-
-

API keys

- -
-
- - - - - - - - {keys.length === 0 ? ( - - - - ) : ( - keys.map((k) => ( - - - - - )) - )} - -
Key -
- No API keys configured. -
{k.name} -
- - -
-
-
- - !open && setPendingDelete(null)} - > - - - Delete this API key? - - “{pendingDelete?.name}” will be removed from this provider. Requests using this key - will start failing immediately. This cannot be undone. - - - - - - - - - - {editing ? ( - setEditing(null)} - onSaved={() => { - setEditing(null); - router.refresh(); - }} - /> - ) : null} - - setAdding(false)} - onSaved={() => { - setAdding(false); - router.refresh(); - }} - /> -
- ); -} - -function KeyDialog( - props: - | { - mode: "add"; - open: boolean; - providerId: string; - pending: boolean; - onClose: () => void; - onSaved: () => void; - } - | { - mode: "edit"; - open?: boolean; - keyEntry: PublicProviderApiKey; - providerId: string; - pending: boolean; - onClose: () => void; - onSaved: () => void; - }, -) { - const isEdit = props.mode === "edit"; - - const [plain, setPlain] = useState(""); - const [saving, start] = useTransition(); - - // ponytail: re-seed `plain` on each dialog re-open so the new secret - // field starts blank (the edit-mode hint says "required to update", - // so a leftover from a previous attempt is misleading). - useEffect(() => { - setPlain(""); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [isEdit ? props.mode === "edit" && props.keyEntry?.name : props.mode === "add" && props.open]); - - const save = () => { - const trimmedPlain = plain.trim(); - if (!trimmedPlain) { - toast.error("key value required"); - return; - } - start(async () => { - // ponytail: both add and edit send plaintext only — the display - // name is derived server-side from the ciphertext tail via - // deriveKeyName (POST) or stays the same on PATCH (which keeps - // the path-bound keyName and only re-encrypts the blob). The - // backend's optional `name` field is left for SQL-direct edits - // and is intentionally not exposed in the UI. - const path = isEdit - ? `/api/admin/providers/${encodeURIComponent(props.providerId)}/keys/${encodeURIComponent(props.keyEntry.name)}` - : `/api/admin/providers/${encodeURIComponent(props.providerId)}/keys`; - const method = isEdit ? "PATCH" : "POST"; - const r = await jsonFetch(path, { - method, - body: JSON.stringify({ plaintext: trimmedPlain }), - }); - if (!r.ok) { - if (r.status === 409) { - toast.error("a key with this tail already exists"); - } else { - toast.error(errMsg(r.body)); - } - return; - } - toast.success(isEdit ? "key updated" : "key added"); - props.onSaved(); - }); - }; - - return ( - !o && props.onClose()} - title={isEdit ? "Edit API key" : "Add API key"} - description={ - isEdit - ? "Paste the new secret — the same key entry is re-encrypted in place. The display name (derived from the ciphertext) updates automatically." - : "Paste the key value to add it. The display name is derived from the ciphertext — both the value and the visible name are produced by the same secret." - } - submitLabel={isEdit ? "Save" : "Add"} - pending={saving || props.pending} - onSubmit={save} - onCancel={props.onClose} - > -
- - {isEdit ? ( -

- Display name {props.keyEntry.name} is derived from - the key value — paste the new secret to regenerate. -

- ) : null} -
- - ); -} - -function RolesPanel({ initial }: { initial: RoleRow[] }) { - const router = useRouter(); - const [adding, setAdding] = useState(false); - - return ( -
-
-
-

Roles

-

- Roles set the per-window credit cap for users. A blank credit limit means unlimited. -

-
- -
- -
- - - - - - - - - - - {initial.map((r) => ( - - ))} - -
IDNameCredit limitWindow (h) -
-
- - setAdding(false)} - onSaved={() => { - setAdding(false); - router.refresh(); - }} - /> -
- ); -} - -function RoleRowView({ role }: { role: RoleRow }) { - const router = useRouter(); - const [pending, start] = useTransition(); - const [editing, setEditing] = useState(false); - const [confirmingDelete, setConfirmingDelete] = useState(false); - - const remove = () => { - start(async () => { - const r = await jsonFetch(`/api/admin/roles/${encodeURIComponent(role.id)}`, { - method: "DELETE", - }); - if (!r.ok) { - toast.error(errMsg(r.body)); - return; - } - setConfirmingDelete(false); - toast.success("role deleted"); - router.refresh(); - }); - }; - - return ( - - {role.id} - {role.name} - - {role.creditLimit === null ? "unlimited" : role.creditLimit} - - {role.windowHours} - -
- - -
- - !open && setConfirmingDelete(false)} - > - - - Delete this role? - - “{role.name}” will be removed. Users on this role fall back to the default. This - cannot be undone. - - - - - - - - - - setEditing(false)} - onSaved={() => { - setEditing(false); - router.refresh(); - }} - /> - - ); -} - -function RoleDialog( - props: - | { - mode: "add"; - open: boolean; - onClose: () => void; - onSaved: () => void; - } - | { - mode: "edit"; - open?: boolean; - role: RoleRow; - onClose: () => void; - onSaved: () => void; - }, -) { - const isEdit = props.mode === "edit"; - const initialName = isEdit ? props.role.name : ""; - const initialLimit = isEdit - ? props.role.creditLimit === null - ? "" - : String(props.role.creditLimit) - : ""; - const initialHours = isEdit ? String(props.role.windowHours) : "24"; - - const [id, setId] = useState(isEdit ? props.role.id : ""); - const [name, setName] = useState(initialName); - const [limit, setLimit] = useState(initialLimit); - const [hours, setHours] = useState(initialHours); - const [saving, start] = useTransition(); - - useEffect(() => { - setName(initialName); - setLimit(initialLimit); - setHours(initialHours); - }, [initialName, initialLimit, initialHours]); - - const save = () => { - if (!name.trim()) { - toast.error("name required"); - return; - } - if (!isEdit && !id.trim()) { - toast.error("id required"); - return; - } - if (!isEdit && !/^[a-z][a-z0-9_-]*$/.test(id.trim())) { - toast.error("id must be lowercase alphanumeric / dash / underscore"); - return; - } - const h = Number(hours); - if (!Number.isInteger(h) || h < 1) { - toast.error("windowHours must be a positive integer"); - return; - } - const limitNum = limit.trim() === "" ? null : Number(limit); - if (limitNum !== null && (!Number.isFinite(limitNum) || limitNum < 0)) { - toast.error("creditLimit must be a non-negative number or blank for unlimited"); - return; - } - start(async () => { - let r; - if (isEdit) { - r = await jsonFetch(`/api/admin/roles/${encodeURIComponent(props.role.id)}`, { - method: "PATCH", - body: JSON.stringify({ name: name.trim(), creditLimit: limitNum, windowHours: h }), - }); - } else { - r = await jsonFetch("/api/admin/roles", { - method: "POST", - body: JSON.stringify({ - id: id.trim(), - name: name.trim(), - creditLimit: limitNum, - windowHours: h, - }), - }); - } - if (!r.ok) { - toast.error(errMsg(r.body)); - return; - } - toast.success(isEdit ? "role updated" : "role created"); - props.onSaved(); - }); - }; - - return ( - !o && props.onClose()} - title={isEdit ? `Edit role: ${props.role.id}` : "Add role"} - description={ - isEdit - ? "ID is the FK identifier and can’t be changed here. Delete + recreate to rename." - : "Roles set the per-window credit cap for users. A blank credit limit means unlimited." - } - submitLabel={isEdit ? "Save" : "Add"} - pending={saving} - onSubmit={save} - onCancel={props.onClose} - > -
- - - - -
- - ); -} - -function UsersPanel({ initial, roles }: { initial: UserRow[]; roles: RoleRow[] }) { - const total = initial.length; - const admins = initial.filter((u) => u.roleId === "admin").length; - const banned = initial.filter((u) => u.banned).length; - - const todayCredits = initial.reduce((sum, u) => sum + (u.todayCredits ?? 0), 0); - const todayInputTokens = initial.reduce((sum, u) => sum + (u.todayInputTokens ?? 0), 0); - const todayOutputTokens = initial.reduce((sum, u) => sum + (u.todayOutputTokens ?? 0), 0); - - const totalCredits = initial.reduce((sum, u) => sum + (u.totalCredits ?? 0), 0); - const totalInputTokens = initial.reduce((sum, u) => sum + (u.totalInputTokens ?? 0), 0); - const totalOutputTokens = initial.reduce((sum, u) => sum + (u.totalOutputTokens ?? 0), 0); - - return ( -
-
-

Users

-

- Registered accounts, their role, and ban status. Banning immediately revokes every active - session for that user — they’re signed out on the next request. -

-
- -
- - - -
- -
- - - - - - - - - - - - {initial.length === 0 ? ( - - - - ) : ( - initial.map((u) => ) - )} - -
UserRoleStatusToday's UsageTotal Usage -
- No users yet. -
-
-
- ); -} - -function UserBreakdownCard({ - total, - admins, - banned, -}: { - total: number; - admins: number; - banned: number; -}) { - const regular = Math.max(0, total - admins - banned); - const adminPct = total > 0 ? (admins / total) * 100 : 0; - const regularPct = total > 0 ? (regular / total) * 100 : 0; - const bannedPct = total > 0 ? (banned / total) * 100 : 0; - - return ( - - -
-
- Users -
- {total} registered -
- -
- {total} - users -
- -
- - - -
- {total > 0 ? ( - <> -
-
-
- - ) : ( -
- )} -
- - -
- admins: {admins} ({adminPct.toFixed(1)}%) -
-
- active: {regular} ({regularPct.toFixed(1)}%) -
-
- banned: {banned} ({bannedPct.toFixed(1)}%) -
- - - - -
-
- - Admin: {admins} -
-
- - Active: {regular} -
-
- - Banned: {banned} -
-
-
- - - ); -} - -function UsageStatCard({ - title, - credits, - inputTokens, - outputTokens, -}: { - title: string; - credits: number; - inputTokens: number; - outputTokens: number; -}) { - const totalTokens = inputTokens + outputTokens; - const inputPct = totalTokens > 0 ? (inputTokens / totalTokens) * 100 : 0; - const outputPct = totalTokens > 0 ? 100 - inputPct : 0; - - return ( - - -
-
- {title} -
- - {totalTokens.toLocaleString()} tok - + +
+
- -
- - {credits.toLocaleString(undefined, { - minimumFractionDigits: 2, - maximumFractionDigits: 2, - })} - - credit -
- -
- - - -
- {totalTokens > 0 ? ( - <> -
-
- - ) : ( -
- )} -
- - -
- input: {inputTokens.toLocaleString()} tok ({inputPct.toFixed(1)}%) -
-
- output: {outputTokens.toLocaleString()} tok ({outputPct.toFixed(1)}%) -
- - - - -
-
- - In: {inputTokens.toLocaleString()} -
-
- - Out: {outputTokens.toLocaleString()} -
-
-
- - - ); -} - -function UserRowView({ user, roles }: { user: UserRow; roles: RoleRow[] }) { - const router = useRouter(); - const [pending, start] = useTransition(); - const [editing, setEditing] = useState(false); - const [confirmingDelete, setConfirmingDelete] = useState(false); - - const remove = () => { - start(async () => { - const r = await jsonFetch(`/api/admin/users/${encodeURIComponent(user.id)}`, { - method: "DELETE", - }); - if (!r.ok) { - if (r.status === 409) { - toast.error("cannot delete the last admin"); - } else { - toast.error(errMsg(r.body)); - } - return; - } - setConfirmingDelete(false); - toast.success("user deleted"); - router.refresh(); - }); - }; - - return ( - - -
- - {user.image ? : null} - - {(user.name || user.email).slice(0, 2).toUpperCase()} - - -
- - {user.name ?? "—"} - - - {user.email} - -
-
- - - - {user.roleName ?? user.roleId} - - - - {user.banned ? ( - Banned - ) : user.emailVerified ? ( - Verified - ) : ( - Unverified - )} - - -
- - {(user.todayCredits ?? 0).toLocaleString(undefined, { - minimumFractionDigits: 2, - maximumFractionDigits: 2, - })}{" "} - credit - - - - - - {(user.todayTokens ?? 0).toLocaleString()}{" "} - tok - - - -
input: {(user.todayInputTokens ?? 0).toLocaleString()} tok
-
output: {(user.todayOutputTokens ?? 0).toLocaleString()} tok
- - - -
- - -
- - {(user.totalCredits ?? 0).toLocaleString(undefined, { - minimumFractionDigits: 2, - maximumFractionDigits: 2, - })}{" "} - credit - - - - - - {(user.totalTokens ?? 0).toLocaleString()}{" "} - tok - - - -
input: {(user.totalInputTokens ?? 0).toLocaleString()} tok
-
output: {(user.totalOutputTokens ?? 0).toLocaleString()} tok
- - - -
- - -
- - -
- - !open && setConfirmingDelete(false)} - > - - - Delete this user? - - “{user.email}” and all of their sessions, accounts, and threads will be removed. - This cannot be undone. - - - - - - - - - - setEditing(false)} - onSaved={() => { - setEditing(false); - router.refresh(); - }} - /> - - - ); -} - -type UserDialogProps = { - mode: "edit"; - open: boolean; - user: UserRow; - roles: RoleRow[]; - onClose: () => void; - onSaved: () => void; -}; - -// ponytail: role is a Select (dropdown), never a free-text input — the -// FK target list comes from the roles prop, so the dropdown is the -// authoritative surface and there's no risk of typing a typo that -// 404s on save. Server still validates (404 ROLE_NOT_FOUND) as a -// defense-in-depth in case the role list goes stale. -function UserDialog(props: UserDialogProps) { - const initialRoleId = props.user.roleId; - const initialBanned = props.user.banned; - - const [roleId, setRoleId] = useState(initialRoleId); - const [banned, setBanned] = useState(initialBanned); - const [saving, start] = useTransition(); - - useEffect(() => { - setRoleId(initialRoleId); - setBanned(initialBanned); - }, [initialRoleId, initialBanned]); - - const save = () => { - if (!roleId) { - toast.error("role required"); - return; - } - start(async () => { - const r = await jsonFetch(`/api/admin/users/${encodeURIComponent(props.user.id)}`, { - method: "PATCH", - body: JSON.stringify({ roleId, banned }), - }); - if (!r.ok) { - if (r.status === 409) { - toast.error("cannot demote or ban the last admin"); - } else if (r.status === 404) { - toast.error("role not found"); - } else { - toast.error(errMsg(r.body)); - } - return; - } - toast.success("user updated"); - props.onSaved(); - }); - }; - - return ( - !o && props.onClose()} - title={`Edit user: ${props.user.email}`} - description="Pick a role from the list — FK is validated server-side. Banning immediately revokes every active session for this user." - submitLabel="Save" - pending={saving} - onSubmit={save} - onCancel={props.onClose} - > -
-
- - Email - - {props.user.email} -
- - -
- - ); -} - -// ponytail: provider add + edit share fields (display name, enabled, -// baseUrl). Mode flips the title, submit label, and whether the auto- -// generated id is shown. `id` is the DB primary key — PATCH doesn't -// accept a rename, so the edit dialog locks it. Add mode hides it -// entirely (server picks one from `crypto.randomUUID()`). -type ProviderDialogProps = - | { - mode: "add"; - open: boolean; - onClose: () => void; - onSaved: () => void; - } - | { - mode: "edit"; - open?: boolean; - provider: PublicProviderRow; - pending: boolean; - onClose: () => void; - onSaved: () => void; - }; - -function ProviderDialog(props: ProviderDialogProps) { - const isEdit = props.mode === "edit"; - const initialName = isEdit ? props.provider.name : ""; - const initialEnabled = isEdit ? props.provider.enabled : true; - const initialBaseUrl = isEdit ? props.provider.baseUrl : ""; - - const [name, setName] = useState(initialName); - const [enabled, setEnabled] = useState(initialEnabled); - const [baseUrl, setBaseUrl] = useState(initialBaseUrl); - const [saving, start] = useTransition(); - - useEffect(() => { - setName(initialName); - setEnabled(initialEnabled); - setBaseUrl(initialBaseUrl); - }, [initialName, initialEnabled, initialBaseUrl]); - - const save = () => { - if (!name.trim()) { - toast.error("display name is required"); - return; - } - if (!baseUrl.trim()) { - toast.error("base URL is required"); - return; - } - try { - // ponytail: trust the user but validate the URL parses — admin - // will hit 4xx from the upstream if they paste a typo, but a - // frontend check keeps the error next to the field. - // eslint-disable-next-line no-new - new URL(baseUrl.trim()); - } catch { - toast.error("base URL must be a valid URL"); - return; - } - start(async () => { - let r; - if (isEdit) { - r = await jsonFetch(`/api/admin/providers/${encodeURIComponent(props.provider.id)}`, { - method: "PATCH", - body: JSON.stringify({ - name: name.trim(), - enabled, - baseUrl: baseUrl.trim(), - }), - }); - } else { - // ponytail: id is auto-generated server-side — we send a UUID - // with a "prov_" prefix so the URL space is unmistakable. - // Collisions are astronomically unlikely but POST returns 409 - // with a "DUPLICATE" code if it ever happens; the toast - // surfaces that as a fallback. - const newId = `prov_${crypto.randomUUID().replace(/-/g, "").slice(0, 12)}`; - r = await jsonFetch("/api/admin/providers", { - method: "POST", - body: JSON.stringify({ - id: newId, - name: name.trim(), - enabled, - baseUrl: baseUrl.trim(), - }), - }); - } - if (!r.ok) { - if (r.status === 409) { - toast.error("a provider with this id already exists"); - } else { - toast.error(errMsg(r.body)); - } - return; - } - toast.success(isEdit ? "provider updated" : "provider created"); - props.onSaved(); - }); - }; - - return ( - !o && props.onClose()} - title={isEdit ? "Edit provider" : "Add provider"} - description={ - isEdit - ? `ID ${props.provider.id} is the FK identifier and can’t be changed here. Delete + recreate to rename.` - : "Register a new LLM provider. The id is auto-generated; provide a display name, enabled toggle, and base URL." - } - submitLabel={isEdit ? "Save" : "Add"} - pending={saving || (isEdit ? props.pending : false)} - onSubmit={save} - onCancel={props.onClose} - > -
- {isEdit ? ( -
- - ID - - {props.provider.id} -
- ) : null} - - - -
- + + ); } diff --git a/app/admin/eval/eval-dashboard-client.tsx b/app/admin/eval/eval-dashboard-client.tsx new file mode 100644 index 00000000..a4fe56e4 --- /dev/null +++ b/app/admin/eval/eval-dashboard-client.tsx @@ -0,0 +1,1264 @@ +"use client"; + +import React, { useEffect, useState } from "react"; +import { + Activity, + Bot, + CheckCircle2, + ChevronDown, + ChevronRight, + ChevronUp, + Clock, + Code2, + Copy, + Cpu, + Database, + GitBranch, + Layers, + Plus, + RefreshCw, + Scale, + Search, + Sliders, + Sparkles, + ThumbsDown, + ThumbsUp, + UserCheck, + Workflow, + XCircle, +} from "lucide-react"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; +import { Input } from "@/components/ui/input"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { Separator } from "@/components/ui/separator"; +import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs"; +import { toast } from "sonner"; + +type Template = { + id: string; + group: string; + agent: string; + content: string; + notes?: string; + createdAt: string; +}; + +type Variant = { + id: string; + templateId: string; + label: string; + trafficWeight: number; + enabled: boolean; +}; + +type VariantStat = { + variantId: string; + label: string; + totalRuns: number; + avgTotalMs: number; + avgRating: number; +}; + +type RecentRun = { + id: string; + agent: string; + variantId: string; + label: string; + totalMs: number; + status: string; + createdAt: string; + userRating?: number; + threadId?: string; + parentMessageId?: string; + inputTokens?: number; + outputTokens?: number; +}; + +type UserAssignment = { + userId: string; + userName?: string; + userEmail?: string; + variantId: string; + variantLabel?: string; + templateId?: string; + agent?: string; + assignedAt: string; +}; + +type Rubric = { + id: string; + name: string; + criteria: Array<{ name: string; description: string; weight: number }>; +}; + +type Judgment = { + id: string; + runId: string; + rubricId: string; + scores: Record; + reasoning: string; + totalCostTokens?: number; + createdAt: string; + agent?: string; + variantId?: string; +}; + +type RunTraceDetail = { + run: RecentRun; + feedback?: { rating: number; reason?: string }; + judgment?: Judgment; + spans?: Array<{ id: string; name: string; durationMs: number; error?: string }>; +}; + +const LANGGRAPH_GROUPS = [ + { + id: "agent", + label: "Main Chat Graph (agent)", + entrypoint: "./backend/agent.ts", + icon: Cpu, + description: "Core interactive chat, intent router, and domain sub-agents", + agents: [ + { + id: "chatAgent", + name: "Chat Agent", + desc: "General conversational LLM & tool orchestrator", + }, + { id: "routerAgent", name: "Router Agent", desc: "Intent classification & node dispatcher" }, + { id: "weatherAgent", name: "Weather Sub-Agent", desc: "Geocoding & forecast fetching" }, + { + id: "cryptoAgent", + name: "Crypto Sub-Agent", + desc: "Ticker price & mock DEX order execution", + }, + { id: "codeAgent", name: "Code Sub-Agent", desc: "TypeScript & Python Firecracker Sandbox" }, + ], + }, + { + id: "background_agent", + label: "Background Graph (background_agent)", + entrypoint: "./backend/background-agent.ts", + icon: Workflow, + description: "Asynchronous background processing and thread summarization", + agents: [ + { + id: "threadSummarizeAgent", + name: "Thread Summarizer", + desc: "Context window compression & history summarization", + }, + { + id: "renameThreadAgent", + name: "Title Generator", + desc: "Automatic thread title generation", + }, + ], + }, + { + id: "kbAgent", + label: "Knowledge Base Graph (kbAgent)", + entrypoint: "./backend/agent/kb-agent.ts", + icon: Database, + description: "PDF OCR vision parsing, GraphRAG entity extraction, and canonical alignment", + agents: [ + { id: "kbOcrAgent", name: "KB OCR Digitizer", desc: "PDF page vision to clean Markdown" }, + { + id: "kbEntityExtractAgent", + name: "GraphRAG Extract", + desc: "Entity, relation & theme triple extraction", + }, + { + id: "kbEntityAlignAgent", + name: "GraphRAG Align", + desc: "Cross-chunk canonical entity deduplication", + }, + ], + }, + { + id: "evalAgent", + label: "Judge Eval Graph (evalAgent)", + entrypoint: "./backend/agent/eval-agent.ts", + icon: Scale, + description: "LLM-as-a-Judge offline/online scoring agent", + agents: [ + { id: "evalJudgeAgent", name: "Eval Judge Agent", desc: "Automated scoring against rubric" }, + ], + }, +]; + +export function EvalDashboardClient() { + const [templates, setTemplates] = useState([]); + const [variants, setVariants] = useState([]); + const [stats, setStats] = useState([]); + const [recentRuns, setRecentRuns] = useState([]); + const [assignments, setAssignments] = useState([]); + const [rubrics, setRubrics] = useState([]); + const [judgments, setJudgments] = useState([]); + const [loading, setLoading] = useState(true); + + // Active Sub-Tab + const [subTab, setSubTab] = useState("prompts"); + + // Trace Detail Drawer / Modal + const [selectedRunId, setSelectedRunId] = useState(null); + const [traceDetail, setTraceDetail] = useState(null); + const [loadingTrace, setLoadingTrace] = useState(false); + const [evaluating, setEvaluating] = useState(false); + + // New Prompt Variant Dialog + const [addingVariantTmplId, setAddingVariantTmplId] = useState(null); + const [newVarLabel, setNewVarLabel] = useState(""); + const [newVarWeight, setNewVarWeight] = useState("50"); + + // Deploy Prompt Version Dialog State + const [deployModalOpen, setDeployModalOpen] = useState(false); + const [targetAgentForNew, setTargetAgentForNew] = useState("chatAgent"); + const [newPromptContent, setNewPromptContent] = useState(""); + const [newPromptNotes, setNewPromptNotes] = useState(""); + const [creatingTmpl, setCreatingTmpl] = useState(false); + + // User Filter for Assignments + const [userSearch, setUserSearch] = useState(""); + + // Collapsible Groups State + const [collapsedGroups, setCollapsedGroups] = useState>({}); + + const toggleGroupCollapse = (groupId: string) => { + setCollapsedGroups((prev) => ({ ...prev, [groupId]: !prev[groupId] })); + }; + + const fetchData = async () => { + try { + setLoading(true); + const [promptsRes, compareRes, assignRes, rubricsRes] = await Promise.all([ + fetch("/api/eval/prompts"), + fetch("/api/eval/runs/compare"), + fetch("/api/eval/assignments"), + fetch("/api/eval/rubrics"), + ]); + + if (promptsRes.ok) { + const pData = await promptsRes.json(); + setTemplates(pData.templates || []); + setVariants(pData.variants || []); + } + + if (compareRes.ok) { + const cData = await compareRes.json(); + setStats(cData.stats || []); + setRecentRuns(cData.recentRuns || []); + } + + if (assignRes.ok) { + const aData = await assignRes.json(); + setAssignments(aData.assignments || []); + } + + if (rubricsRes.ok) { + const rData = await rubricsRes.json(); + setRubrics(rData.rubrics || []); + setJudgments(rData.judgments || []); + } + } catch (err) { + console.error("Failed to load eval dashboard data:", err); + toast.error("Failed to fetch evaluation data"); + } finally { + setLoading(false); + } + }; + + useEffect(() => { + fetchData(); + }, []); + + const handleUpdateWeight = async (variantId: string, weight: number) => { + try { + const res = await fetch("/api/eval/prompts", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + action: "update_variant_weight", + variantId, + trafficWeight: weight, + }), + }); + if (res.ok) { + toast.success("Traffic weight updated"); + fetchData(); + } else { + toast.error("Failed to update weight"); + } + } catch (err) { + console.error("Failed to update weight:", err); + } + }; + + const handleAddVariant = async () => { + if (!addingVariantTmplId || !newVarLabel.trim()) return; + try { + const res = await fetch("/api/eval/prompts", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + action: "create_variant", + templateId: addingVariantTmplId, + label: newVarLabel.trim(), + trafficWeight: parseInt(newVarWeight) || 50, + }), + }); + if (res.ok) { + toast.success("New variant added"); + setAddingVariantTmplId(null); + setNewVarLabel(""); + fetchData(); + } else { + toast.error("Failed to create variant"); + } + } catch (err) { + console.error("Failed to create variant:", err); + } + }; + + const handleCreateTemplate = async () => { + if (!newPromptContent.trim()) { + toast.error("Prompt content cannot be empty"); + return; + } + try { + setCreatingTmpl(true); + const res = await fetch("/api/eval/prompts", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + action: "create_template", + agent: targetAgentForNew, + content: newPromptContent, + notes: newPromptNotes.trim() || undefined, + }), + }); + if (res.ok) { + toast.success(`Deployed prompt template for ${targetAgentForNew}`); + setNewPromptContent(""); + setNewPromptNotes(""); + setDeployModalOpen(false); + fetchData(); + } else { + toast.error("Failed to deploy template"); + } + } catch (err) { + console.error("Failed to create template:", err); + } finally { + setCreatingTmpl(false); + } + }; + + const openDeployDialogForAgent = (agentId: string) => { + setTargetAgentForNew(agentId); + setNewPromptContent(""); + setNewPromptNotes(""); + setDeployModalOpen(true); + }; + + const openTraceDetail = async (runId: string) => { + setSelectedRunId(runId); + setLoadingTrace(true); + try { + const res = await fetch(`/api/eval/runs/${runId}`); + if (res.ok) { + const data = await res.json(); + setTraceDetail(data); + } + } catch (err) { + console.error("Failed to load run trace detail:", err); + } finally { + setLoadingTrace(false); + } + }; + + const triggerAIJudge = async (runId: string) => { + setEvaluating(true); + try { + const res = await fetch("/api/eval/judge", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ runId }), + }); + if (res.ok) { + toast.success("AI Judge scoring complete!"); + if (selectedRunId === runId) { + openTraceDetail(runId); + } + fetchData(); + } else { + toast.error("AI Judge scoring failed"); + } + } catch (err) { + console.error("Failed to trigger AI Judge:", err); + } finally { + setEvaluating(false); + } + }; + + const copyToClipboard = (text: string) => { + navigator.clipboard.writeText(text); + toast.success("Copied to clipboard"); + }; + + if (loading) { + return ( +
+ + Loading Evaluation Platform... +
+ ); + } + + const filteredAssignments = assignments.filter( + (a) => + !userSearch || + (a.userEmail && a.userEmail.toLowerCase().includes(userSearch.toLowerCase())) || + (a.userId && a.userId.toLowerCase().includes(userSearch.toLowerCase())) || + (a.agent && a.agent.toLowerCase().includes(userSearch.toLowerCase())), + ); + + return ( +
+ {/* Top Header Bar with Standard Action Button */} +
+
+
+

Eval & A/B Platform

+

+ System prompts, A/B traffic variants, sticky user assignments, execution traces, and + LLM-as-a-Judge scoring. +

+
+ +
+ + {/* 4 Main Sub-Tabs Navigation */} + + + + Prompts & A/B Studio + + {templates.length} + + + + User Assignments + + {assignments.length} + + + + Execution Logs & Traces + + {recentRuns.length} + + + + Rubric & AI Judgments + + {judgments.length} + + + + +
+ + {/* ========================================================================= */} + {/* TAB 1: Prompts & A/B Studio (targetGroupName Cards with Prompts Sub-Table) */} + {/* ========================================================================= */} + {subTab === "prompts" && ( +
+ {LANGGRAPH_GROUPS.map((group) => { + const Icon = group.icon; + const isCollapsed = collapsedGroups[group.id]; + const groupTemplates = templates.filter( + (t) => + t.group === group.id || + (group.id === "agent" && (!t.group || t.group === "Main Assistant")), + ); + + return ( + + +
+
+
+ + {group.label} + + {group.id} + +
+ + {group.entrypoint} + · + {group.description} + +
+
+ + +
+
+ + +
+
+ +
+ + Prompts ({groupTemplates.length} Template versions) + +
+ +
+ + + + + + + + + + + + {group.agents.map((agentObj) => { + const agentId = agentObj.id; + const agentTemplates = groupTemplates.filter( + (t) => t.agent === agentId, + ); + + if (agentTemplates.length === 0) { + return ( + + + + + + ); + } + + return agentTemplates.map((tmpl, index) => { + const tmplVariants = variants.filter( + (v) => v.templateId === tmpl.id, + ); + const totalW = tmplVariants.reduce( + (s, v) => s + Math.max(0, v.trafficWeight), + 0, + ); + + return ( + + {index === 0 ? ( + + ) : null} + + + + + + ); + }); + })} + +
Target Node (targetName)Template IDNotes / RationaleActive Variants & Traffic WeightsActions
+
+ + {agentId} +
+ + {agentObj.name} + + 		+ No prompt template deployed yet. + + +
+
+ + {agentId} +
+ + {agentObj.name} + + 		+ {tmpl.id} + + {tmpl.notes || "—"} + +
+ {tmplVariants.map((v) => { + const pct = + totalW > 0 + ? Math.round((v.trafficWeight / totalW) * 100) + : 0; + return ( +
+
+ + {v.label} ({v.id}) + + + {pct}% Traffic + +
+
+ + handleUpdateWeight( + v.id, + parseInt(e.target.value), + ) + } + className="h-1.5 flex-1 cursor-pointer accent-primary rounded-lg bg-muted" + /> + + {v.trafficWeight} + +
+
+ ); + })} + +
+ 		+
+ + +
+
+
+ +
+
+ + ); + })} +
+ )} + + {/* ========================================================================= */} + {/* TAB 2: User A/B Variant Assignments (prompt_variant_assignment Table) */} + {/* ========================================================================= */} + {subTab === "assignments" && ( +
+
+
+

User Sticky A/B Assignments

+

+ Inspect which sticky prompt variant is assigned to each registered user. +

+
+
+ + setUserSearch(e.target.value)} + placeholder="Search user or email..." + className="pl-8 h-9 text-xs" + /> +
+
+ +
+ + + + + + + + + + + + {filteredAssignments.map((a, idx) => ( + + + + + + + + ))} + {filteredAssignments.length === 0 && ( + + + + )} + +
UserTarget AgentAssigned VariantTemplate IDAssigned Date
+
+ {a.userName || "User"} + + {a.userEmail || a.userId} + +
+ 		+ {a.agent || "—"} + + + {a.variantLabel || a.variantId} + + + {a.templateId || "—"} + + {new Date(a.assignedAt).toLocaleDateString("en-CA")} +
+ No sticky user assignments recorded yet. Invocations will automatically bind + users to weighted variants. +
+
+
+ )} + + {/* ========================================================================= */} + {/* TAB 3: Execution Logs & Traces (eval_run + Trace Drawer + Trigger AI Judge) */} + {/* ========================================================================= */} + {subTab === "logs" && ( +
+
+
+

Execution Traces & AI Judge Trigger

+

+ Inspect live evaluation run traces, input/output messages, token consumption, and + trigger LLM-as-a-Judge scoring. +

+
+ + {recentRuns.length} Total Run Logs + +
+ +
+ + + + + + + + + + + + + + + {recentRuns.map((run) => ( + + + + + + + + + + + ))} + {recentRuns.length === 0 && ( + + + + )} + +
Run IDAgent NodeVariantStatusLatencyUser RatingAI Judge StatusActions
+ {run.id} + {run.agent} + + {run.label || run.variantId} + + + {run.status === "success" ? ( + + success + + ) : ( + + {run.status} + + )} + + {run.totalMs} ms + + {run.userRating === 5 ? ( + + 5 (Up) + + ) : run.userRating === 1 ? ( + + 1 (Down) + + ) : run.userRating ? ( + {run.userRating} ★ + ) : ( + — + )} + + + + +
+ No evaluation execution logs recorded yet. Start a chat session to generate + live runs! +
+
+
+ )} + + {/* ========================================================================= */} + {/* TAB 4: Rubric & AI Judgments (eval_rubric & eval_judgment Tables) */} + {/* ========================================================================= */} + {subTab === "rubric" && ( +
+ {/* Section 1: Active Rubric Criteria */} +
+
+

+ Evaluation Rubric Criteria (eval_rubric) +

+ + {rubrics.length} Rubric(s) Configured + +
+ +
+ {rubrics.map((rubric) => ( + + +
+ {rubric.name} + + {rubric.id} + +
+ + + + Evaluation Criteria & Weights + +
+ {rubric.criteria.map((c) => ( +
+
+ {c.name} + + {c.description} + +
+ + {Math.round(c.weight * 100)}% + +
+ ))} +
+ + + ))} +
+
+ + + + {/* Section 2: AI Judgment Scoring History */} +
+
+

+ AI Judge Scoring History (eval_judgment) +

+ + {judgments.length} Judgment(s) + +
+ +
+ + + + + + + + + + + + + {judgments.map((jg) => ( + + + + + + + + + ))} + {judgments.length === 0 && ( + + + + )} + +
Judgment IDRun IDAgent NodeScores BreakdownReasoning SummaryCost Tokens
{jg.id}{jg.runId}{jg.agent || "—"} +
+ {Object.entries(jg.scores || {}).map(([key, val]) => ( + + {key}: {val} + + ))} +
+ 		+ {jg.reasoning} + + {jg.totalCostTokens ?? 0} tok +
+ No AI Judge evaluation history recorded yet. Trigger AI Judge on an + execution run to generate scores! +
+
+
+
+ )} + + {/* Deploy Prompt Version Modal */} + + + + Add prompt version + + Create a new System Prompt template version for node{" "} + {targetAgentForNew}. + + + +
+ + + + +